tanner/qotnews
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Cache all articles in IndexedDB

Browse Source
This commit is contained in:
Tanner Collin 2019-10-12 23:41:31 +00:00
parent 7cb87b59fe
commit 0f5b2a5ff9
10 changed files with 88 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apiserver/feeds/reddit.py
View File

@ -14,7 +14,7 @@ from prawcore.exceptions import PrawcoreException
from utils import render_md
SUBREDDITS = 'Economics+Foodforthought+Futurology+TrueReddit+business+science+technology'
SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+technology'
SITE_LINK = lambda x : 'https://old.reddit.com/{}'.format(x)
SITE_AUTHOR_LINK = lambda x : 'https://old.reddit.com/u/{}'.format(x)

25
apiserver/requirements.txt Normal file
View File

@ -0,0 +1,25 @@
beautifulsoup4==4.8.0
certifi==2019.6.16
chardet==3.0.4
Click==7.0
commonmark==0.9.0
feedparser==5.2.1
Flask==1.1.1
Flask-Cors==3.0.8
future==0.17.1
idna==2.8
itsdangerous==1.1.0
Jinja2==2.10.1
MarkupSafe==1.1.1
pkg-resources==0.0.0
praw==6.3.1
prawcore==1.0.1
requests==2.22.0
six==1.12.0
soupsieve==1.9.3
update-checker==0.16
urllib3==1.25.3
webencodings==0.5.1
websocket-client==0.56.0
Werkzeug==0.15.5
Whoosh==2.7.4

7
apiserver/server.py
View File

@ -43,10 +43,11 @@ cors = CORS(flask_app)
@flask_app.route('/api')
def api():
front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list]
front_page = [x for x in front_page if 'title' in x and x['title']]
front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']]
front_page = front_page[:100]
to_remove = ['text', 'comments']
front_page = [{k:v for k,v in s.items() if k not in to_remove} for s in front_page]
for story in front_page:
story.pop('text', None)
story.pop('comments', None)
return {'stories': front_page}

1
webclient/package.json
View File

@ -3,6 +3,7 @@
"version": "0.1.0",
"private": true,
"dependencies": {
"localforage": "^1.7.3",
"moment": "^2.24.0",
"query-string": "^6.8.3",
"react": "^16.9.0",

14
webclient/src/Article.js
View File

@ -1,16 +1,15 @@
import React from 'react';
import { Link } from 'react-router-dom';
import { Helmet } from 'react-helmet';
import localForage from 'localforage';
import { sourceLink, infoLine, ToggleDot } from './utils.js';
class Article extends React.Component {
constructor(props) {
super(props);
const id = this.props.match.params.id;
this.state = {
story: JSON.parse(localStorage.getItem(id)) || false,
story: false,
error: false,
};
}
@ -18,12 +17,19 @@ class Article extends React.Component {
componentDidMount() {
const id = this.props.match.params.id;
localForage.getItem(id)
.then(
(value) => {
this.setState({ story: value });
}
);
fetch('/api/' + id)
.then(res => res.json())
.then(
(result) => {
this.setState({ story: result.story });
localStorage.setItem(id, JSON.stringify(result.story));
localForage.setItem(id, result.story);
},
(error) => {
this.setState({ error: true });

14
webclient/src/Comments.js
View File

@ -3,16 +3,15 @@ import { Link } from 'react-router-dom';
import { HashLink } from 'react-router-hash-link';
import { Helmet } from 'react-helmet';
import moment from 'moment';
import localForage from 'localforage';
import { sourceLink, infoLine, ToggleDot } from './utils.js';
class Article extends React.Component {
constructor(props) {
super(props);
const id = this.props.match.params.id;
this.state = {
story: JSON.parse(localStorage.getItem(id)) || false,
story: false,
error: false,
};
}
@ -20,17 +19,24 @@ class Article extends React.Component {
componentDidMount() {
const id = this.props.match.params.id;
localForage.getItem(id)
.then(
(value) => {
this.setState({ story: value });
}
);
fetch('/api/' + id)
.then(res => res.json())
.then(
(result) => {
localStorage.setItem(id, JSON.stringify(result.story));
this.setState({ story: result.story }, () => {
const hash = window.location.hash.substring(1);
if (hash) {
document.getElementById(hash).scrollIntoView();
}
});
localForage.setItem(id, result.story);
},
(error) => {
this.setState({ error: true });

28
webclient/src/Feed.js
View File

@ -1,8 +1,8 @@
import React from 'react';
import { Link } from 'react-router-dom';
import { Helmet } from 'react-helmet';
import localForage from 'localforage';
import { siteLogo, sourceLink, infoLine } from './utils.js';
import { clearStorage } from './utils.js';
class Feed extends React.Component {
constructor(props) {
@ -19,18 +19,24 @@ class Feed extends React.Component {
.then(res => res.json())
.then(
(result) => {
const updated = !this.state.stories || this.state.stories[0].id !== result.stories[0].id;
console.log('updated:', updated);
this.setState({ stories: result.stories });
clearStorage();
localStorage.setItem('stories', JSON.stringify(result.stories));
result.stories.filter(x => x.score >= 20).slice(0, 25).forEach(x => {
fetch('/api/' + x.id)
.then(res => res.json())
.then(result => {
localStorage.setItem(x.id, JSON.stringify(result.story));
console.log('Preloaded story', x.id, x.title);
}, error => {}
);
});
if (updated) {
localForage.clear();
result.stories.forEach(x => {
fetch('/api/' + x.id)
.then(res => res.json())
.then(result => {
localForage.setItem(x.id, result.story)
.then(console.log('preloaded', x.id, x.title));
}, error => {}
);
});
}
},
(error) => {
this.setState({ error: true });

2
webclient/src/Style-light.css
View File

@ -109,7 +109,7 @@ span.source {
height: auto;
}
.article figure {
.article figure, .article video {
width: 100%;
height: auto;
margin: 0;

6
webclient/src/utils.js
View File

@ -29,12 +29,6 @@ export const infoLine = (story) =>
</div>
;
export const clearStorage = () => {
const themeSetting = localStorage.getItem('theme');
localStorage.clear();
localStorage.setItem('theme', themeSetting);
};
export class ToggleDot extends React.Component {
render() {
const id = this.props.id;

19
webclient/yarn.lock
View File

@ -4629,6 +4629,11 @@ ignore@^4.0.6:
resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc"
integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==
immediate@~3.0.5:
version "3.0.6"
resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=
immer@1.10.0:
version "1.10.0"
resolved "https://registry.yarnpkg.com/immer/-/immer-1.10.0.tgz#bad67605ba9c810275d91e1c2a47d4582e98286d"
@ -5769,6 +5774,13 @@ levn@^0.3.0, levn@~0.3.0:
prelude-ls "~1.1.2"
type-check "~0.3.2"
lie@3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=
dependencies:
immediate "~3.0.5"
load-json-file@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-2.0.0.tgz#7947e42149af80d696cbf797bcaabcfe1fe29ca8"
@ -5811,6 +5823,13 @@ loader-utils@1.2.3, loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.
emojis-list "^2.0.0"
json5 "^1.0.1"
localforage@^1.7.3:
version "1.7.3"
resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.7.3.tgz#0082b3ca9734679e1bd534995bdd3b24cf10f204"
integrity sha512-1TulyYfc4udS7ECSBT2vwJksWbkwwTX8BzeUIiq8Y07Riy7bDAAnxDaPU/tWyOVmQAcWJIEIFP9lPfBGqVoPgQ==
dependencies:
lie "3.1.1"
locate-path@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e"