diff --git a/apiserver/feeds/reddit.py b/apiserver/feeds/reddit.py index 86e1a4b..b7ee104 100644 --- a/apiserver/feeds/reddit.py +++ b/apiserver/feeds/reddit.py @@ -14,7 +14,7 @@ from prawcore.exceptions import PrawcoreException from utils import render_md -SUBREDDITS = 'Economics+Foodforthought+Futurology+TrueReddit+business+science+technology' +SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+technology' SITE_LINK = lambda x : 'https://old.reddit.com/{}'.format(x) SITE_AUTHOR_LINK = lambda x : 'https://old.reddit.com/u/{}'.format(x) diff --git a/apiserver/requirements.txt b/apiserver/requirements.txt new file mode 100644 index 0000000..23d2f77 --- /dev/null +++ b/apiserver/requirements.txt @@ -0,0 +1,25 @@ +beautifulsoup4==4.8.0 +certifi==2019.6.16 +chardet==3.0.4 +Click==7.0 +commonmark==0.9.0 +feedparser==5.2.1 +Flask==1.1.1 +Flask-Cors==3.0.8 +future==0.17.1 +idna==2.8 +itsdangerous==1.1.0 +Jinja2==2.10.1 +MarkupSafe==1.1.1 +pkg-resources==0.0.0 +praw==6.3.1 +prawcore==1.0.1 +requests==2.22.0 +six==1.12.0 +soupsieve==1.9.3 +update-checker==0.16 +urllib3==1.25.3 +webencodings==0.5.1 +websocket-client==0.56.0 +Werkzeug==0.15.5 +Whoosh==2.7.4 diff --git a/apiserver/server.py b/apiserver/server.py index 3c1ac09..ed94fe6 100644 --- a/apiserver/server.py +++ b/apiserver/server.py @@ -43,10 +43,11 @@ cors = CORS(flask_app) @flask_app.route('/api') def api(): front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list] - front_page = [x for x in front_page if 'title' in x and x['title']] + front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']] front_page = front_page[:100] - to_remove = ['text', 'comments'] - front_page = [{k:v for k,v in s.items() if k not in to_remove} for s in front_page] + for story in front_page: + story.pop('text', None) + story.pop('comments', None) return {'stories': front_page} diff --git a/webclient/package.json b/webclient/package.json index 9a0b3a3..99e5370 100644 --- a/webclient/package.json +++ b/webclient/package.json @@ -3,6 +3,7 @@ "version": "0.1.0", "private": true, "dependencies": { + "localforage": "^1.7.3", "moment": "^2.24.0", "query-string": "^6.8.3", "react": "^16.9.0", diff --git a/webclient/src/Article.js b/webclient/src/Article.js index 6d9e3a1..2cbcd80 100644 --- a/webclient/src/Article.js +++ b/webclient/src/Article.js @@ -1,16 +1,15 @@ import React from 'react'; import { Link } from 'react-router-dom'; import { Helmet } from 'react-helmet'; +import localForage from 'localforage'; import { sourceLink, infoLine, ToggleDot } from './utils.js'; class Article extends React.Component { constructor(props) { super(props); - const id = this.props.match.params.id; - this.state = { - story: JSON.parse(localStorage.getItem(id)) || false, + story: false, error: false, }; } @@ -18,12 +17,19 @@ class Article extends React.Component { componentDidMount() { const id = this.props.match.params.id; + localForage.getItem(id) + .then( + (value) => { + this.setState({ story: value }); + } + ); + fetch('/api/' + id) .then(res => res.json()) .then( (result) => { this.setState({ story: result.story }); - localStorage.setItem(id, JSON.stringify(result.story)); + localForage.setItem(id, result.story); }, (error) => { this.setState({ error: true }); diff --git a/webclient/src/Comments.js b/webclient/src/Comments.js index 8bb98ac..5997cbc 100644 --- a/webclient/src/Comments.js +++ b/webclient/src/Comments.js @@ -3,16 +3,15 @@ import { Link } from 'react-router-dom'; import { HashLink } from 'react-router-hash-link'; import { Helmet } from 'react-helmet'; import moment from 'moment'; +import localForage from 'localforage'; import { sourceLink, infoLine, ToggleDot } from './utils.js'; class Article extends React.Component { constructor(props) { super(props); - const id = this.props.match.params.id; - this.state = { - story: JSON.parse(localStorage.getItem(id)) || false, + story: false, error: false, }; } @@ -20,17 +19,24 @@ class Article extends React.Component { componentDidMount() { const id = this.props.match.params.id; + localForage.getItem(id) + .then( + (value) => { + this.setState({ story: value }); + } + ); + fetch('/api/' + id) .then(res => res.json()) .then( (result) => { - localStorage.setItem(id, JSON.stringify(result.story)); this.setState({ story: result.story }, () => { const hash = window.location.hash.substring(1); if (hash) { document.getElementById(hash).scrollIntoView(); } }); + localForage.setItem(id, result.story); }, (error) => { this.setState({ error: true }); diff --git a/webclient/src/Feed.js b/webclient/src/Feed.js index 7da5e9c..310cf72 100644 --- a/webclient/src/Feed.js +++ b/webclient/src/Feed.js @@ -1,8 +1,8 @@ import React from 'react'; import { Link } from 'react-router-dom'; import { Helmet } from 'react-helmet'; +import localForage from 'localforage'; import { siteLogo, sourceLink, infoLine } from './utils.js'; -import { clearStorage } from './utils.js'; class Feed extends React.Component { constructor(props) { @@ -19,18 +19,24 @@ class Feed extends React.Component { .then(res => res.json()) .then( (result) => { + const updated = !this.state.stories || this.state.stories[0].id !== result.stories[0].id; + console.log('updated:', updated); + this.setState({ stories: result.stories }); - clearStorage(); localStorage.setItem('stories', JSON.stringify(result.stories)); - result.stories.filter(x => x.score >= 20).slice(0, 25).forEach(x => { - fetch('/api/' + x.id) - .then(res => res.json()) - .then(result => { - localStorage.setItem(x.id, JSON.stringify(result.story)); - console.log('Preloaded story', x.id, x.title); - }, error => {} - ); - }); + + if (updated) { + localForage.clear(); + result.stories.forEach(x => { + fetch('/api/' + x.id) + .then(res => res.json()) + .then(result => { + localForage.setItem(x.id, result.story) + .then(console.log('preloaded', x.id, x.title)); + }, error => {} + ); + }); + } }, (error) => { this.setState({ error: true }); diff --git a/webclient/src/Style-light.css b/webclient/src/Style-light.css index d29804f..9528f77 100644 --- a/webclient/src/Style-light.css +++ b/webclient/src/Style-light.css @@ -109,7 +109,7 @@ span.source { height: auto; } -.article figure { +.article figure, .article video { width: 100%; height: auto; margin: 0; diff --git a/webclient/src/utils.js b/webclient/src/utils.js index 64e12ee..8e20263 100644 --- a/webclient/src/utils.js +++ b/webclient/src/utils.js @@ -29,12 +29,6 @@ export const infoLine = (story) => ; -export const clearStorage = () => { - const themeSetting = localStorage.getItem('theme'); - localStorage.clear(); - localStorage.setItem('theme', themeSetting); -}; - export class ToggleDot extends React.Component { render() { const id = this.props.id; diff --git a/webclient/yarn.lock b/webclient/yarn.lock index 25c2558..32e764e 100644 --- a/webclient/yarn.lock +++ b/webclient/yarn.lock @@ -4629,6 +4629,11 @@ ignore@^4.0.6: resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc" integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg== +immediate@~3.0.5: + version "3.0.6" + resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" + integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps= + immer@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/immer/-/immer-1.10.0.tgz#bad67605ba9c810275d91e1c2a47d4582e98286d" @@ -5769,6 +5774,13 @@ levn@^0.3.0, levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" +lie@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e" + integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4= + dependencies: + immediate "~3.0.5" + load-json-file@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-2.0.0.tgz#7947e42149af80d696cbf797bcaabcfe1fe29ca8" @@ -5811,6 +5823,13 @@ loader-utils@1.2.3, loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1. emojis-list "^2.0.0" json5 "^1.0.1" +localforage@^1.7.3: + version "1.7.3" + resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.7.3.tgz#0082b3ca9734679e1bd534995bdd3b24cf10f204" + integrity sha512-1TulyYfc4udS7ECSBT2vwJksWbkwwTX8BzeUIiq8Y07Riy7bDAAnxDaPU/tWyOVmQAcWJIEIFP9lPfBGqVoPgQ== + dependencies: + lie "3.1.1" + locate-path@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e"