Cache all articles in IndexedDB

apiserver/feeds/reddit.py

@@ -14,7 +14,7 @@ from prawcore.exceptions import PrawcoreException
 
 from utils import render_md
 
-SUBREDDITS = 'Economics+Foodforthought+Futurology+TrueReddit+business+science+technology'
+SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+technology'
 
 SITE_LINK = lambda x : 'https://old.reddit.com/{}'.format(x)
 SITE_AUTHOR_LINK = lambda x : 'https://old.reddit.com/u/{}'.format(x)

apiserver/requirements.txt

@@ -0,0 +1,25 @@
+beautifulsoup4==4.8.0
+certifi==2019.6.16
+chardet==3.0.4
+Click==7.0
+commonmark==0.9.0
+feedparser==5.2.1
+Flask==1.1.1
+Flask-Cors==3.0.8
+future==0.17.1
+idna==2.8
+itsdangerous==1.1.0
+Jinja2==2.10.1
+MarkupSafe==1.1.1
+pkg-resources==0.0.0
+praw==6.3.1
+prawcore==1.0.1
+requests==2.22.0
+six==1.12.0
+soupsieve==1.9.3
+update-checker==0.16
+urllib3==1.25.3
+webencodings==0.5.1
+websocket-client==0.56.0
+Werkzeug==0.15.5
+Whoosh==2.7.4

apiserver/server.py

@@ -43,10 +43,11 @@ cors = CORS(flask_app)
 @flask_app.route('/api')
 def api():
     front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list]
-    front_page = [x for x in front_page if 'title' in x and x['title']]
+    front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']]
     front_page = front_page[:100]
-    to_remove = ['text', 'comments']
+    for story in front_page:
-    front_page = [{k:v for k,v in s.items() if k not in to_remove} for s in front_page]
+        story.pop('text', None)
+        story.pop('comments', None)
 
     return {'stories': front_page}
 

webclient/package.json

@@ -3,6 +3,7 @@
   "version": "0.1.0",
   "private": true,
   "dependencies": {
+    "localforage": "^1.7.3",
     "moment": "^2.24.0",
     "query-string": "^6.8.3",
     "react": "^16.9.0",

webclient/src/Article.js

@@ -1,16 +1,15 @@
 import React from 'react';
 import { Link } from 'react-router-dom';
 import { Helmet } from 'react-helmet';
+import localForage from 'localforage';
 import { sourceLink, infoLine, ToggleDot } from './utils.js';
 
 class Article extends React.Component {
 	constructor(props) {
 		super(props);
 
-		const id = this.props.match.params.id;
-
 		this.state = {
-			story: JSON.parse(localStorage.getItem(id)) || false,
+			story: false,
 			error: false,
 		};
 	}
@@ -18,12 +17,19 @@ class Article extends React.Component {
 	componentDidMount() {
 		const id = this.props.match.params.id;
 
+		localForage.getItem(id)
+			.then(
+				(value) => {
+					this.setState({ story: value });
+				}
+			);
+
 		fetch('/api/' + id)
 			.then(res => res.json())
 			.then(
 				(result) => {
 					this.setState({ story: result.story });
-					localStorage.setItem(id, JSON.stringify(result.story));
+					localForage.setItem(id, result.story);
 				},
 				(error) => {
 					this.setState({ error: true });

webclient/src/Comments.js

@@ -3,16 +3,15 @@ import { Link } from 'react-router-dom';
 import { HashLink } from 'react-router-hash-link';
 import { Helmet } from 'react-helmet';
 import moment from 'moment';
+import localForage from 'localforage';
 import { sourceLink, infoLine, ToggleDot } from './utils.js';
 
 class Article extends React.Component {
 	constructor(props) {
 		super(props);
 
-		const id = this.props.match.params.id;
-
 		this.state = {
-			story: JSON.parse(localStorage.getItem(id)) || false,
+			story: false,
 			error: false,
 		};
 	}
@@ -20,17 +19,24 @@ class Article extends React.Component {
 	componentDidMount() {
 		const id = this.props.match.params.id;
 
+		localForage.getItem(id)
+			.then(
+				(value) => {
+					this.setState({ story: value });
+				}
+			);
+
 		fetch('/api/' + id)
 			.then(res => res.json())
 			.then(
 				(result) => {
-					localStorage.setItem(id, JSON.stringify(result.story));
 					this.setState({ story: result.story }, () => {
 						const hash = window.location.hash.substring(1);
 						if (hash) {
 							document.getElementById(hash).scrollIntoView();
 						}
 					});
+					localForage.setItem(id, result.story);
 				},
 				(error) => {
 					this.setState({ error: true });

webclient/src/Feed.js

@@ -1,8 +1,8 @@
 import React from 'react';
 import { Link } from 'react-router-dom';
 import { Helmet } from 'react-helmet';
+import localForage from 'localforage';
 import { siteLogo, sourceLink, infoLine } from './utils.js';
-import { clearStorage } from './utils.js';
 
 class Feed extends React.Component {
 	constructor(props) {
@@ -19,18 +19,24 @@ class Feed extends React.Component {
 			.then(res => res.json())
 			.then(
 				(result) => {
+					const updated = !this.state.stories || this.state.stories[0].id !== result.stories[0].id;
+					console.log('updated:', updated);
+
 					this.setState({ stories: result.stories });
-					clearStorage();
 					localStorage.setItem('stories', JSON.stringify(result.stories));
-					result.stories.filter(x => x.score >= 20).slice(0, 25).forEach(x => {
+
-						fetch('/api/' + x.id)
+					if (updated) {
-							.then(res => res.json())
+						localForage.clear();
-							.then(result => {
+						result.stories.forEach(x => {
-								localStorage.setItem(x.id, JSON.stringify(result.story));
+							fetch('/api/' + x.id)
-								console.log('Preloaded story', x.id, x.title);
+								.then(res => res.json())
-							}, error => {}
+								.then(result => {
-						);
+									localForage.setItem(x.id, result.story)
-					});
+										.then(console.log('preloaded', x.id, x.title));
+								}, error => {}
+							);
+						});
+					}
 				},
 				(error) => {
 					this.setState({ error: true });

webclient/src/Style-light.css

@@ -109,7 +109,7 @@ span.source {
 	height: auto;
 }
 
-.article figure {
+.article figure, .article video {
 	width: 100%;
 	height: auto;
 	margin: 0;

webclient/src/utils.js

@@ -29,12 +29,6 @@ export const infoLine = (story) =>
 	</div>
 ;
 
-export const clearStorage = () => {
-	const themeSetting = localStorage.getItem('theme');
-	localStorage.clear();
-	localStorage.setItem('theme', themeSetting);
-};
-
 export class ToggleDot extends React.Component {
 	render() {
 		const id = this.props.id;

webclient/yarn.lock

@@ -4629,6 +4629,11 @@ ignore@^4.0.6:
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc"
   integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==
 
+immediate@~3.0.5:
+  version "3.0.6"
+  resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
+  integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=
+
 immer@1.10.0:
   version "1.10.0"
   resolved "https://registry.yarnpkg.com/immer/-/immer-1.10.0.tgz#bad67605ba9c810275d91e1c2a47d4582e98286d"
@@ -5769,6 +5774,13 @@ levn@^0.3.0, levn@~0.3.0:
     prelude-ls "~1.1.2"
     type-check "~0.3.2"
 
+lie@3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
+  integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=
+  dependencies:
+    immediate "~3.0.5"
+
 load-json-file@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-2.0.0.tgz#7947e42149af80d696cbf797bcaabcfe1fe29ca8"
@@ -5811,6 +5823,13 @@ loader-utils@1.2.3, loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.
     emojis-list "^2.0.0"
     json5 "^1.0.1"
 
+localforage@^1.7.3:
+  version "1.7.3"
+  resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.7.3.tgz#0082b3ca9734679e1bd534995bdd3b24cf10f204"
+  integrity sha512-1TulyYfc4udS7ECSBT2vwJksWbkwwTX8BzeUIiq8Y07Riy7bDAAnxDaPU/tWyOVmQAcWJIEIFP9lPfBGqVoPgQ==
+  dependencies:
+    lie "3.1.1"
+
 locate-path@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e"