apiserver/.gitignore

@@ -109,4 +109,3 @@ praw.ini
 data.db
 data.db.bak
 data/archive/*
-qotnews.sqlite

apiserver/database.py

@@ -1,106 +0,0 @@
-import json
-
-from sqlalchemy import create_engine, Column, String, ForeignKey, Integer
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.exc import IntegrityError
-
-engine = create_engine('sqlite:///data/qotnews.sqlite')
-Session = sessionmaker(bind=engine)
-
-Base = declarative_base()
-
-class Story(Base):
-    __tablename__ = 'stories'
-
-    sid = Column(String(16), primary_key=True)
-    ref = Column(String(16), unique=True)
-    meta_json = Column(String)
-    full_json = Column(String)
-    title = Column(String)
-
-class Reflist(Base):
-    __tablename__ = 'reflist'
-
-    rid = Column(Integer, primary_key=True)
-    ref = Column(String(16), unique=True)
-    sid = Column(String, ForeignKey('stories.sid'), unique=True)
-    source = Column(String(16))
-
-def init():
-    Base.metadata.create_all(engine)
-
-def get_story(sid):
-    session = Session()
-    return session.query(Story).get(sid)
-
-def put_story(story):
-    story = story.copy()
-    full_json = json.dumps(story)
-
-    story.pop('text', None)
-    story.pop('comments', None)
-    meta_json = json.dumps(story)
-
-    try:
-        session = Session()
-        s = Story(
-            sid=story['id'],
-            ref=story['ref'],
-            full_json=full_json,
-            meta_json=meta_json,
-            title=story.get('title', None),
-        )
-        session.merge(s)
-        session.commit()
-    except:
-        session.rollback()
-        raise
-    finally:
-        session.close() 
-
-def get_story_by_ref(ref):
-    session = Session()
-    return session.query(Story).filter(Story.ref==ref).first()
-
-def get_reflist(amount):
-    session = Session()
-    q = session.query(Reflist).order_by(Reflist.rid.desc()).limit(amount)
-    return [dict(ref=x.ref, sid=x.sid, source=x.source) for x in q.all()]
-
-def get_stories(amount):
-    session = Session()
-    q = session.query(Reflist, Story.meta_json).\
-            order_by(Reflist.rid.desc()).\
-            join(Story).\
-            filter(Story.title != None).\
-            limit(amount)
-    return [x[1] for x in q]
-
-def put_ref(ref, sid, source):
-    try:
-        session = Session()
-        r = Reflist(ref=ref, sid=sid, source=source)
-        session.add(r)
-        session.commit()
-    except:
-        session.rollback()
-        raise
-    finally:
-        session.close()
-
-def del_ref(ref):
-    try:
-        session = Session()
-        session.query(Reflist).filter(Reflist.ref==ref).delete()
-        session.commit()
-    except:
-        session.rollback()
-        raise
-    finally:
-        session.close()
-
-if __name__ == '__main__':
-    init()
-
-    print(get_story_by_ref('hgi3sy'))

apiserver/feed.py

@@ -9,10 +9,11 @@ from bs4 import BeautifulSoup
 
 from feeds import hackernews, reddit, tildes, manual
 
-OUTLINE_API = 'https://api.outline.com/v3/parse_article'
+OUTLINE_API = 'https://outlineapi.com/article'
 ARCHIVE_API = 'https://archive.fo/submit/'
 READ_API = 'http://127.0.0.1:33843'
 
+INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif']
 INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com']
 TWO_DAYS = 60*60*24*2
 
@@ -56,18 +57,18 @@ def get_article(url):
         logging.error('Problem getting article: {}'.format(str(e)))
         return ''
 
-def get_content_type(url):
+def get_first_image(text):
-    try:
+    soup = BeautifulSoup(text, features='html.parser')
-        headers = {'User-Agent': 'Twitterbot/1.0'}
-        return requests.get(url, headers=headers, timeout=2).headers['content-type']
-    except:
-        pass
 
     try:
-        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0'}
+        first_img = soup.find('img')
-        return requests.get(url, headers=headers, timeout=2).headers['content-type']
+        url = first_img['src']
+        headers = {'User-Agent': 'Twitterbot/1.0'}
+        length = requests.get(url, headers=headers).headers['Content-length']
+        if int(length) > 1000000: raise
+        return url
     except:
-        return 'text/'
+        return ''
 
 def update_story(story, is_manual=False):
     res = {}
@@ -86,29 +87,28 @@ def update_story(story, is_manual=False):
     if res:
         story.update(res) # join dicts
     else:
-        logging.info('Story not ready yet')
+        logging.info('Article not ready yet')
         return False
 
     if story['date'] and not is_manual and story['date'] + TWO_DAYS < time.time():
-        logging.info('Story too old, removing')
+        logging.info('Article too old, removing')
         return False
 
     if story.get('url', '') and not story.get('text', ''):
-        logging.info('inside if')
+        if any([story['url'].endswith(ext) for ext in INVALID_FILES]):
-        if not get_content_type(story['url']).startswith('text/'):
+            logging.info('URL invalid file type')
-            logging.info('URL invalid file type / content type:')
-            logging.info(story['url'])
             return False
 
         if any([domain in story['url'] for domain in INVALID_DOMAINS]):
-            logging.info('URL invalid domain:')
+            logging.info('URL invalid domain')
-            logging.info(story['url'])
             return False
 
         logging.info('Getting article ' + story['url'])
         story['text'] = get_article(story['url'])
         if not story['text']: return False
 
+        story['img'] = get_first_image(story['text'])
+
     return True
 
 if __name__ == '__main__':
@@ -124,5 +124,7 @@ if __name__ == '__main__':
 
     a = get_article('https://blog.joinmastodon.org/2019/10/mastodon-3.0/')
     print(a)
+    u = get_first_image(a)
+    print(u)
 
     print('done')

apiserver/feeds/hackernews.py

@@ -30,7 +30,7 @@ def api(route, ref=None):
         return False
 
 def feed():
-    return [str(x) for x in api(API_TOPSTORIES) or []]
+    return api(API_TOPSTORIES) or []
 
 def comment(i):
     if 'author' not in i:
@@ -77,6 +77,6 @@ def story(ref):
 
 # scratchpad so I can quickly develop the parser
 if __name__ == '__main__':
-    print(feed())
+    #print(feed())
     #print(story(20763961))
-    #print(story(20802050))
+    print(story(20802050))

apiserver/feeds/manual.py

@@ -33,7 +33,7 @@ def story(ref):
     s['author_link'] = 'https://news.t0.vc'
     s['score'] = 0
     s['date'] = int(time.time())
-    s['title'] = str(soup.title.string) if soup.title else ref
+    s['title'] = str(soup.title.string)
     s['link'] = ref
     s['url'] = ref
     s['comments'] = []

apiserver/feeds/reddit.py

@@ -14,9 +14,9 @@ from prawcore.exceptions import PrawcoreException
 
 from utils import render_md, clean
 
-SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+privacy'
+SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+technology+privacy'
 
-SITE_LINK = lambda x : 'https://old.reddit.com{}'.format(x)
+SITE_LINK = lambda x : 'https://old.reddit.com/{}'.format(x)
 SITE_AUTHOR_LINK = lambda x : 'https://old.reddit.com/u/{}'.format(x)
 
 reddit = praw.Reddit('bot')

apiserver/feeds/tildes.py

@@ -91,7 +91,7 @@ def story(ref):
     s['score'] = int(h.find('span', class_='topic-voting-votes').string)
     s['date'] = unix(h.find('time')['datetime'])
     s['title'] = str(h.h1.string)
-    s['group'] = str(soup.find('div', class_='site-header-context').a.string)
+    s['group'] = str(soup.find('a', class_='site-header-context').string)
     group_lookup[ref] = s['group']
     s['link'] = SITE_LINK(s['group'], ref)
     ud = a.find('div', class_='topic-full-link')
@@ -122,7 +122,7 @@ if __name__ == '__main__':
     #print(self_post)
     #li_comment = story('gqx')
     #print(li_comment)
-    broken = story('q4y')
+    broken = story('l11')
     print(broken)
 
     # make sure there's no self-reference

apiserver/migrate-whoosh-to-sqlite.py

@@ -1,74 +0,0 @@
-import archive
-import database
-import search
-
-import json
-import requests
-
-database.init()
-archive.init()
-search.init()
-
-count = 0
-
-def database_del_story_by_ref(ref):
-    try:
-        session = database.Session()
-        session.query(database.Story).filter(database.Story.ref==ref).delete()
-        session.commit()
-    except:
-        session.rollback()
-        raise
-    finally:
-        session.close()
-
-def search_del_story(sid):
-    try:
-        r = requests.delete(search.MEILI_URL + 'indexes/qotnews/documents/'+sid, timeout=2)
-        if r.status_code != 202:
-            raise Exception('Bad response code ' + str(r.status_code))
-        return r.json()
-    except KeyboardInterrupt:
-        raise
-    except BaseException as e:
-        logging.error('Problem deleting MeiliSearch story: {}'.format(str(e)))
-        return False
-
-with archive.ix.searcher() as searcher:
-    print('count all', searcher.doc_count_all())
-    print('count', searcher.doc_count())
-
-    for doc in searcher.documents():
-        try:
-            print('num', count, 'id', doc['id'])
-            count += 1
-
-            story = doc['story']
-            story.pop('img', None)
-
-            if 'reddit.com/r/technology' in story['link']:
-                print('skipping r/technology')
-                continue
-
-            try:
-                database.put_story(story)
-            except database.IntegrityError:
-                print('collision!')
-                old_story = database.get_story_by_ref(story['ref'])
-                old_story = json.loads(old_story.full_json)
-                if story['num_comments'] > old_story['num_comments']:
-                    print('more comments, replacing')
-                    database_del_story_by_ref(story['ref'])
-                    database.put_story(story)
-                    search_del_story(old_story['id'])
-                else:
-                    print('fewer comments, skipping')
-                    continue
-
-            search.put_story(story)
-            print()
-        except KeyboardInterrupt:
-            break
-        except BaseException as e:
-            print('skipping', doc['id'])
-            print('reason:', e)

apiserver/search.py

@@ -1,85 +0,0 @@
-import logging
-logging.basicConfig(
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-        level=logging.DEBUG)
-
-import requests
-
-MEILI_URL = 'http://127.0.0.1:7700/'
-
-def create_index():
-    try:
-        json = dict(name='qotnews', uid='qotnews')
-        r = requests.post(MEILI_URL + 'indexes', json=json, timeout=2)
-        if r.status_code != 201:
-            raise Exception('Bad response code ' + str(r.status_code))
-        return r.json()
-    except KeyboardInterrupt:
-        raise
-    except BaseException as e:
-        logging.error('Problem creating MeiliSearch index: {}'.format(str(e)))
-        return False
-
-def update_rankings():
-    try:
-        json = ['typo', 'words', 'proximity', 'attribute', 'desc(date)', 'wordsPosition', 'exactness']
-        r = requests.post(MEILI_URL + 'indexes/qotnews/settings/ranking-rules', json=json, timeout=2)
-        if r.status_code != 202:
-            raise Exception('Bad response code ' + str(r.status_code))
-        return r.json()
-    except KeyboardInterrupt:
-        raise
-    except BaseException as e:
-        logging.error('Problem setting MeiliSearch ranking rules: {}'.format(str(e)))
-        return False
-
-def update_attributes():
-    try:
-        json = ['title', 'url', 'author', 'link', 'id']
-        r = requests.post(MEILI_URL + 'indexes/qotnews/settings/searchable-attributes', json=json, timeout=2)
-        if r.status_code != 202:
-            raise Exception('Bad response code ' + str(r.status_code))
-        return r.json()
-    except KeyboardInterrupt:
-        raise
-    except BaseException as e:
-        logging.error('Problem setting MeiliSearch searchable attributes: {}'.format(str(e)))
-        return False
-
-def init():
-    create_index()
-    update_rankings()
-    update_attributes()
-
-def put_story(story):
-    story = story.copy()
-    story.pop('text', None)
-    story.pop('comments', None)
-    try:
-        r = requests.post(MEILI_URL + 'indexes/qotnews/documents', json=[story], timeout=2)
-        if r.status_code != 202:
-            raise Exception('Bad response code ' + str(r.status_code))
-        return r.json()
-    except KeyboardInterrupt:
-        raise
-    except BaseException as e:
-        logging.error('Problem putting MeiliSearch story: {}'.format(str(e)))
-        return False
-
-def search(q):
-    try:
-        params = dict(q=q, limit=250)
-        r = requests.get(MEILI_URL + 'indexes/qotnews/search', params=params, timeout=2)
-        if r.status_code != 200:
-            raise Exception('Bad response code ' + str(r.status_code))
-        return r.json()['hits']
-    except KeyboardInterrupt:
-        raise
-    except BaseException as e:
-        logging.error('Problem searching MeiliSearch: {}'.format(str(e)))
-        return False
-    
-if __name__ == '__main__':
-    create_index()
-
-    print(search('the'))

apiserver/server.py

@@ -4,14 +4,12 @@ logging.basicConfig(
         level=logging.INFO)
 
 import copy
-import json
 import threading
-import traceback
 import time
+import shelve
 from urllib.parse import urlparse, parse_qs
 
-import database
+import archive
-import search
 import feed
 from utils import gen_rand_id
 
@@ -25,38 +23,75 @@ from gevent.pywsgi import WSGIServer
 
 monkey.patch_all()
 
-database.init()
+archive.init()
-search.init()
+
+CACHE_LENGTH = 150
+DATA_FILE = 'data/data'
 
-FEED_LENGTH = 75
 news_index = 0
 
+with shelve.open(DATA_FILE) as db:
+    logging.info('Reading caches from disk...')
+    news_list = db.get('news_list', [])
+    news_ref_to_id = db.get('news_ref_to_id', {})
+    news_cache = db.get('news_cache', {})
+
+    # clean cache if broken
+    try:
+        for ref in news_list:
+            nid = news_ref_to_id[ref]
+            _ = news_cache[nid]
+    except KeyError as e:
+        logging.error('Unable to find key {}. Trying to remove...'.format(str(e)))
+        news_list.remove(str(e))
+
+def get_story(sid):
+    if sid in news_cache:
+        return news_cache[sid]
+    else:
+        return archive.get_story(sid)
+
 def new_id():
     nid = gen_rand_id()
-    while database.get_story(nid):
+    while nid in news_cache or archive.get_story(nid):
         nid = gen_rand_id()
     return nid
 
+def remove_ref(old_ref):
+    while old_ref in news_list:
+        news_list.remove(old_ref)
+    old_story = news_cache.pop(news_ref_to_id[old_ref])
+    old_id = news_ref_to_id.pop(old_ref)
+    logging.info('Removed ref {} id {}.'.format(old_ref, old_id))
+
 build_folder = '../webclient/build'
 flask_app = Flask(__name__, template_folder=build_folder, static_folder=build_folder, static_url_path='')
 cors = CORS(flask_app)
 
 @flask_app.route('/api')
 def api():
-    stories = database.get_stories(FEED_LENGTH)
+    try:
-    # hacky nested json
+        front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list]
-    res = Response('{"stories":[' + ','.join(stories) + ']}')
+    except KeyError as e:
-    res.headers['content-type'] = 'application/json'
+        logging.error('Unable to find key {}. Trying to remove...'.format(str(e)))
-    return res
+        news_list.remove(str(e))
+
+    front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']]
+    front_page = front_page[:60]
+    for story in front_page:
+        story.pop('text', None)
+        story.pop('comments', None)
+
+    return {'stories': front_page}
 
 @flask_app.route('/api/search', strict_slashes=False)
-def apisearch():
+def search():
-    q = request.args.get('q', '')
+    search = request.args.get('q', '')
-    if len(q) >= 3:
+    if len(search) >= 3:
-        results = search.search(q)
+        res = archive.search(search)
     else:
-        results = []
+        res = []
-    return dict(results=results)
+    return {'results': res}
 
 @flask_app.route('/api/submit', methods=['POST'], strict_slashes=False)
 def submit():
@@ -74,41 +109,29 @@ def submit():
         elif 'reddit.com' in parse.hostname and 'comments' in url:
             source = 'reddit'
             ref = parse.path.split('/')[4]
-        elif 'news.t0.vc' in parse.hostname:
-            raise Exception('Invalid article')
         else:
             source = 'manual'
             ref = url
 
-        existing = database.get_story_by_ref(ref)
+        news_story = dict(id=nid, ref=ref, source=source)
-        if existing:
+        news_cache[nid] = news_story
-            return {'nid': existing.sid}
+        valid = feed.update_story(news_story, is_manual=True)
+        if valid:
+            archive.update(news_story)
+            return {'nid': nid}
         else:
-            story = dict(id=nid, ref=ref, source=source)
+            news_cache.pop(nid, '')
-            valid = feed.update_story(story, is_manual=True)
+            raise Exception('Invalid article')
-            if valid:
-                database.put_story(story)
-                search.put_story(story)
-                return {'nid': nid}
-            else:
-                raise Exception('Invalid article')
 
     except BaseException as e:
         logging.error('Problem with article submission: {} - {}'.format(e.__class__.__name__, str(e)))
-        print(traceback.format_exc())
         abort(400)
 
 
 @flask_app.route('/api/<sid>')
 def story(sid):
-    story = database.get_story(sid)
+    story = get_story(sid)
-    if story:
+    return dict(story=story) if story else abort(404)
-        # hacky nested json
-        res = Response('{"story":' + story.full_json + '}')
-        res.headers['content-type'] = 'application/json'
-        return res
-    else:
-        return abort(404)
 
 @flask_app.route('/')
 @flask_app.route('/search')
@@ -126,9 +149,8 @@ def static_story(sid):
     except NotFound:
         pass
 
-    story = database.get_story(sid)
+    story = get_story(sid)
     if not story: return abort(404)
-    story = json.loads(story.full_json)
 
     score = story['score']
     num_comments = story['num_comments']
@@ -145,7 +167,7 @@ def static_story(sid):
             url=url,
             description=description)
 
-http_server = WSGIServer(('', 43842), flask_app)
+http_server = WSGIServer(('', 33842), flask_app)
 
 def feed_thread():
     global news_index
@@ -154,42 +176,39 @@ def feed_thread():
         while True:
             # onboard new stories
             if news_index == 0:
-                for ref, source in feed.list():
+                feed_list = feed.list()
-                    if database.get_story_by_ref(ref):
+                new_items = [(ref, source) for ref, source in feed_list if ref not in news_list]
-                        continue
+                for ref, source in new_items:
-                    try:
+                    news_list.insert(0, ref)
-                        nid = new_id()
+                    nid = new_id()
-                        database.put_ref(ref, nid, source)
+                    news_ref_to_id[ref] = nid
-                        logging.info('Added ref ' + ref)
+                    news_cache[nid] = dict(id=nid, ref=ref, source=source)
-                    except database.IntegrityError:
-                        continue
 
-            ref_list = database.get_reflist(FEED_LENGTH)
+                if len(new_items):
+                    logging.info('Added {} new refs.'.format(len(new_items)))
+
+                # drop old ones
+                while len(news_list) > CACHE_LENGTH:
+                    old_ref = news_list[-1]
+                    remove_ref(old_ref)
 
             # update current stories
-            if news_index < len(ref_list):
+            if news_index < len(news_list):
-                item = ref_list[news_index]
+                update_ref = news_list[news_index]
-
+                update_id = news_ref_to_id[update_ref]
-                try:
+                news_story = news_cache[update_id]
-                    story_json = database.get_story(item['sid']).full_json
+                valid = feed.update_story(news_story)
-                    story = json.loads(story_json)
-                except AttributeError:
-                    story = dict(id=item['sid'], ref=item['ref'], source=item['source'])
-
-                valid = feed.update_story(story)
                 if valid:
-                    database.put_story(story)
+                    archive.update(news_story)
-                    search.put_story(story)
                 else:
-                    database.del_ref(item['ref'])
+                    remove_ref(update_ref)
-                    logging.info('Removed ref {}'.format(item['ref']))
             else:
-                logging.info('Skipping index')
+                logging.info('Skipping update - no story #' + str(news_index+1))
 
             gevent.sleep(6)
 
             news_index += 1
-            if news_index == FEED_LENGTH: news_index = 0
+            if news_index == CACHE_LENGTH: news_index = 0
 
     except KeyboardInterrupt:
         logging.info('Ending feed thread...')
@@ -205,3 +224,9 @@ try:
     http_server.serve_forever()
 except KeyboardInterrupt:
     logging.info('Exiting...')
+finally:
+    with shelve.open(DATA_FILE) as db:
+        logging.info('Writing caches to disk...')
+        db['news_list'] = news_list
+        db['news_ref_to_id'] = news_ref_to_id
+        db['news_cache'] = news_cache

webclient/src/Feed.js

@@ -36,6 +36,12 @@ class Feed extends React.Component {
 									this.props.updateCache(x.id, result.story);
 								}, error => {}
 							);
+
+							if (i < 20 && x.img) {
+								const img = new Image();
+								img.src = x.img;
+								console.log('prefetched image', x.img);
+							}
 						});
 					}
 				},
@@ -59,6 +65,10 @@ class Feed extends React.Component {
 					<div>
 						{stories.map((x, i) =>
 							<div className='item' key={i}>
+								<div className='num'>
+									{i+1}.
+								</div>
+
 								<div className='title'>
 									<Link className='link' to={'/' + x.id}>
 										<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}

webclient/src/Results.js

@@ -64,6 +64,10 @@ class Results extends React.Component {
 						{stories.length ?
 							stories.map((x, i) =>
 								<div className='item' key={i}>
+									<div className='num'>
+										{i+1}.
+									</div>
+
 									<div className='title'>
 										<Link className='link' to={'/' + x.id}>
 											<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}

webclient/src/Search.js

@@ -37,7 +37,7 @@ class Search extends Component {
 			<span className='search'>
 				<form onSubmit={this.searchAgain}>
 					<input
-						placeholder='Search... (fixed)'
+						placeholder='Search...'
 						value={search}
 						onChange={this.searchArticles}
 						ref={this.inputRef}

webclient/src/Style-light.css

@@ -46,7 +46,7 @@ pre {
 .item {
 	display: table;
 	color: #828282;
-	margin-bottom: 0.7rem;
+	margin-bottom: 0.6rem;
 }
 
 .item .source-logo {
@@ -61,6 +61,11 @@ pre {
 	text-decoration: underline;
 }
 
+.item .num {
+	display: table-cell;
+	width: 2em;
+}
+
 .item a.link {
 	font-size: 1.1rem;
 	color: #000000;