From 2edb3ceba7b4e0df480412915039f30bbfa010ab Mon Sep 17 00:00:00 2001 From: Tanner Collin Date: Fri, 8 Nov 2019 05:55:30 +0000 Subject: [PATCH] Allow manual submission of articles --- apiserver/feed.py | 4 ++- apiserver/feeds/manual.py | 46 +++++++++++++++++++++++++++ apiserver/server.py | 58 ++++++++++++++++++++++++++--------- webclient/src/App.js | 2 ++ webclient/src/Search.js | 22 ++++++------- webclient/src/Style-light.css | 5 +++ webclient/src/Submit.js | 54 ++++++++++++++++++++++++++++++++ 7 files changed, 163 insertions(+), 28 deletions(-) create mode 100644 apiserver/feeds/manual.py create mode 100644 webclient/src/Submit.js diff --git a/apiserver/feed.py b/apiserver/feed.py index a56e685..0cb8e42 100644 --- a/apiserver/feed.py +++ b/apiserver/feed.py @@ -7,7 +7,7 @@ import requests import time from bs4 import BeautifulSoup -from feeds import hackernews, reddit, tildes +from feeds import hackernews, reddit, tildes, manual OUTLINE_API = 'https://outlineapi.com/article' ARCHIVE_API = 'https://archive.fo/submit/' @@ -99,6 +99,8 @@ def update_story(story): res = reddit.story(story['ref']) elif story['source'] == 'tildes': res = tildes.story(story['ref']) + elif story['source'] == 'manual': + res = manual.story(story['ref']) if res: story.update(res) # join dicts diff --git a/apiserver/feeds/manual.py b/apiserver/feeds/manual.py new file mode 100644 index 0000000..fef3cd0 --- /dev/null +++ b/apiserver/feeds/manual.py @@ -0,0 +1,46 @@ +import logging +logging.basicConfig( + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + level=logging.DEBUG) + +import requests +import time +from bs4 import BeautifulSoup + +USER_AGENT = 'Twitterbot/1.0' + +def api(route): + try: + headers = {'User-Agent': USER_AGENT} + r = requests.get(route, headers=headers, timeout=5) + if r.status_code != 200: + raise Exception('Bad response code ' + str(r.status_code)) + return r.text + except KeyboardInterrupt: + raise + except BaseException as e: + logging.error('Problem hitting manual website: {}'.format(str(e))) + return False + +def story(ref): + html = api(ref) + if not html: return False + + soup = BeautifulSoup(html, features='html.parser') + + s = {} + s['author'] = 'manual submission' + s['author_link'] = 'https://news.t0.vc' + s['score'] = 0 + s['date'] = int(time.time()) + s['title'] = str(soup.title.string) + s['link'] = ref + s['url'] = ref + s['comments'] = [] + s['num_comments'] = 0 + + return s + +# scratchpad so I can quickly develop the parser +if __name__ == '__main__': + print(story('https://www.backblaze.com/blog/what-smart-stats-indicate-hard-drive-failures/')) diff --git a/apiserver/server.py b/apiserver/server.py index 44db571..0d7b254 100644 --- a/apiserver/server.py +++ b/apiserver/server.py @@ -13,7 +13,7 @@ import archive import feed from utils import gen_rand_id -from flask import abort, Flask, request, render_template +from flask import abort, Flask, request, render_template, stream_with_context, Response from werkzeug.exceptions import NotFound from flask_cors import CORS @@ -36,12 +36,37 @@ with shelve.open(DATA_FILE) as db: news_ref_to_id = db.get('news_ref_to_id', {}) news_cache = db.get('news_cache', {}) + # clean cache if broken + try: + for ref in news_list: + nid = news_ref_to_id[ref] + _ = news_cache[nid] + except KeyError as e: + logging.error('Unable to find key: ' + str(e)) + logging.info('Clearing caches...') + news_list = [] + news_ref_to_id = {} + news_cache = {} + def get_story(sid): if sid in news_cache: return news_cache[sid] else: return archive.get_story(sid) +def new_id(): + nid = gen_rand_id() + while nid in news_cache or archive.get_story(nid): + nid = gen_rand_id() + return nid + +def remove_ref(old_ref): + while old_ref in news_list: + news_list.remove(old_ref) + old_story = news_cache.pop(news_ref_to_id[old_ref]) + old_id = news_ref_to_id.pop(old_ref) + logging.info('Removed ref {} id {}.'.format(old_ref, old_id)) + build_folder = '../webclient/build' flask_app = Flask(__name__, template_folder=build_folder, static_folder=build_folder, static_url_path='') cors = CORS(flask_app) @@ -66,6 +91,20 @@ def search(): res = [] return {'results': res} +@flask_app.route('/api/submit', methods=['POST'], strict_slashes=False) +def submit(): + url = request.form['url'] + nid = new_id() + news_story = dict(id=nid, ref=url, source='manual') + news_cache[nid] = news_story + valid = feed.update_story(news_story) + if valid: + archive.update(news_story) + return {'nid': nid} + else: + news_cache.pop(nid, '') + abort(400) + @flask_app.route('/api/') def story(sid): story = get_story(sid) @@ -105,19 +144,6 @@ def static_story(sid): url=url, description=description) -def new_id(): - nid = gen_rand_id() - while nid in news_cache or archive.get_story(nid): - nid = gen_rand_id() - return nid - -def remove_ref(old_ref): - while old_ref in news_list: - news_list.remove(old_ref) - old_story = news_cache.pop(news_ref_to_id[old_ref]) - old_id = news_ref_to_id.pop(old_ref) - logging.info('Removed ref {} id {}.'.format(old_ref, old_id)) - http_server = WSGIServer(('', 33842), flask_app) def feed_thread(): @@ -159,7 +185,9 @@ def feed_thread(): news_index += 1 if news_index == CACHE_LENGTH: news_index = 0 - except BaseException as e: + except KeyboardInterrupt: + logging.info('Ending feed thread...') + except ValueError as e: logging.error('feed_thread error: {} {}'.format(e.__class__.__name__, e)) http_server.stop() diff --git a/webclient/src/App.js b/webclient/src/App.js index 7b4d8da..f977f31 100644 --- a/webclient/src/App.js +++ b/webclient/src/App.js @@ -8,6 +8,7 @@ import Feed from './Feed.js'; import Article from './Article.js'; import Comments from './Comments.js'; import Search from './Search.js'; +import Submit from './Submit.js'; import Results from './Results.js'; import ScrollToTop from './ScrollToTop.js'; @@ -60,6 +61,7 @@ class App extends React.Component { Reddit, Hacker News, and Tildes combined, then pre-rendered in reader mode.

+ } /> diff --git a/webclient/src/Search.js b/webclient/src/Search.js index 52cbbe4..80396c0 100644 --- a/webclient/src/Search.js +++ b/webclient/src/Search.js @@ -34,18 +34,16 @@ class Search extends Component { const search = this.state.search; return ( -
-
-
- -
-
-
+ +
+ +
+ ); } } diff --git a/webclient/src/Style-light.css b/webclient/src/Style-light.css index b00011e..1c91c5b 100644 --- a/webclient/src/Style-light.css +++ b/webclient/src/Style-light.css @@ -15,6 +15,7 @@ input { font-size: 1.05rem; background-color: transparent; border: 1px solid #828282; + margin: 0.25rem; padding: 6px; border-radius: 4px; } @@ -173,3 +174,7 @@ span.source { top: 0.1rem; left: 0.1rem; } + +.search form { + display: inline; +} diff --git a/webclient/src/Submit.js b/webclient/src/Submit.js new file mode 100644 index 0000000..eed87d7 --- /dev/null +++ b/webclient/src/Submit.js @@ -0,0 +1,54 @@ +import React, { Component } from 'react'; +import { withRouter } from 'react-router-dom'; + +class Submit extends Component { + constructor(props) { + super(props); + + this.state = { + progress: null, + }; + + this.inputRef = React.createRef(); + } + + submitArticle = (event) => { + event.preventDefault(); + const url = event.target[0].value; + this.inputRef.current.blur(); + + this.setState({ progress: 'Submitting...' }); + + let data = new FormData(); + data.append('url', url); + + fetch('/api/submit', { method: 'POST', body: data }) + .then(res => res.json()) + .then( + (result) => { + this.props.history.replace('/' + result.nid); + }, + (error) => { + this.setState({ progress: 'Error' }); + } + ); + } + + render() { + const progress = this.state.progress; + + return ( + +
+ +
+ {progress ? progress : ''} + + ); + } +} + +export default withRouter(Submit);