Remove extra logging

apiserver/.gitignore

@@ -109,3 +109,4 @@ praw.ini
 data.db
 data.db.bak
 data/archive/*
+qotnews.sqlite

apiserver/database.py

@@ -0,0 +1,106 @@
+import json
+
+from sqlalchemy import create_engine, Column, String, ForeignKey, Integer
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.exc import IntegrityError
+
+engine = create_engine('sqlite:///data/qotnews.sqlite')
+Session = sessionmaker(bind=engine)
+
+Base = declarative_base()
+
+class Story(Base):
+    __tablename__ = 'stories'
+
+    sid = Column(String(16), primary_key=True)
+    ref = Column(String(16), unique=True)
+    meta_json = Column(String)
+    full_json = Column(String)
+    title = Column(String)
+
+class Reflist(Base):
+    __tablename__ = 'reflist'
+
+    rid = Column(Integer, primary_key=True)
+    ref = Column(String(16), unique=True)
+    sid = Column(String, ForeignKey('stories.sid'), unique=True)
+    source = Column(String(16))
+
+def init():
+    Base.metadata.create_all(engine)
+
+def get_story(sid):
+    session = Session()
+    return session.query(Story).get(sid)
+
+def put_story(story):
+    story = story.copy()
+    full_json = json.dumps(story)
+
+    story.pop('text', None)
+    story.pop('comments', None)
+    meta_json = json.dumps(story)
+
+    try:
+        session = Session()
+        s = Story(
+            sid=story['id'],
+            ref=story['ref'],
+            full_json=full_json,
+            meta_json=meta_json,
+            title=story.get('title', None),
+        )
+        session.merge(s)
+        session.commit()
+    except:
+        session.rollback()
+        raise
+    finally:
+        session.close() 
+
+def get_story_by_ref(ref):
+    session = Session()
+    return session.query(Story).filter(Story.ref==ref).first()
+
+def get_reflist(amount):
+    session = Session()
+    q = session.query(Reflist).order_by(Reflist.rid.desc()).limit(amount)
+    return [dict(ref=x.ref, sid=x.sid, source=x.source) for x in q.all()]
+
+def get_stories(amount):
+    session = Session()
+    q = session.query(Reflist, Story.meta_json).\
+            order_by(Reflist.rid.desc()).\
+            join(Story).\
+            filter(Story.title != None).\
+            limit(amount)
+    return [x[1] for x in q]
+
+def put_ref(ref, sid, source):
+    try:
+        session = Session()
+        r = Reflist(ref=ref, sid=sid, source=source)
+        session.add(r)
+        session.commit()
+    except:
+        session.rollback()
+        raise
+    finally:
+        session.close()
+
+def del_ref(ref):
+    try:
+        session = Session()
+        session.query(Reflist).filter(Reflist.ref==ref).delete()
+        session.commit()
+    except:
+        session.rollback()
+        raise
+    finally:
+        session.close()
+
+if __name__ == '__main__':
+    init()
+
+    print(get_story_by_ref('hgi3sy'))

apiserver/feed.py

@@ -9,11 +9,10 @@ from bs4 import BeautifulSoup
 
 from feeds import hackernews, reddit, tildes, manual
 
-OUTLINE_API = 'https://outlineapi.com/article'
+OUTLINE_API = 'https://api.outline.com/v3/parse_article'
 ARCHIVE_API = 'https://archive.fo/submit/'
 READ_API = 'http://127.0.0.1:33843'
 
-INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif']
 INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com']
 TWO_DAYS = 60*60*24*2
 
@@ -57,18 +56,18 @@ def get_article(url):
         logging.error('Problem getting article: {}'.format(str(e)))
         return ''
 
-def get_first_image(text):
+def get_content_type(url):
-    soup = BeautifulSoup(text, features='html.parser')
+    try:
+        headers = {'User-Agent': 'Twitterbot/1.0'}
+        return requests.get(url, headers=headers, timeout=2).headers['content-type']
+    except:
+        pass
 
     try:
-        first_img = soup.find('img')
+        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0'}
-        url = first_img['src']
+        return requests.get(url, headers=headers, timeout=2).headers['content-type']
-        headers = {'User-Agent': 'Twitterbot/1.0'}
-        length = requests.get(url, headers=headers).headers['Content-length']
-        if int(length) > 1000000: raise
-        return url
     except:
-        return ''
+        return 'text/'
 
 def update_story(story, is_manual=False):
     res = {}
@@ -87,28 +86,29 @@ def update_story(story, is_manual=False):
     if res:
         story.update(res) # join dicts
     else:
-        logging.info('Article not ready yet')
+        logging.info('Story not ready yet')
         return False
 
     if story['date'] and not is_manual and story['date'] + TWO_DAYS < time.time():
-        logging.info('Article too old, removing')
+        logging.info('Story too old, removing')
         return False
 
     if story.get('url', '') and not story.get('text', ''):
-        if any([story['url'].endswith(ext) for ext in INVALID_FILES]):
+        logging.info('inside if')
-            logging.info('URL invalid file type')
+        if not get_content_type(story['url']).startswith('text/'):
+            logging.info('URL invalid file type / content type:')
+            logging.info(story['url'])
             return False
 
         if any([domain in story['url'] for domain in INVALID_DOMAINS]):
-            logging.info('URL invalid domain')
+            logging.info('URL invalid domain:')
+            logging.info(story['url'])
             return False
 
         logging.info('Getting article ' + story['url'])
         story['text'] = get_article(story['url'])
         if not story['text']: return False
 
-        story['img'] = get_first_image(story['text'])
-
     return True
 
 if __name__ == '__main__':
@@ -124,7 +124,5 @@ if __name__ == '__main__':
 
     a = get_article('https://blog.joinmastodon.org/2019/10/mastodon-3.0/')
     print(a)
-    u = get_first_image(a)
-    print(u)
 
     print('done')

apiserver/feeds/hackernews.py

@@ -30,7 +30,7 @@ def api(route, ref=None):
         return False
 
 def feed():
-    return api(API_TOPSTORIES) or []
+    return [str(x) for x in api(API_TOPSTORIES) or []]
 
 def comment(i):
     if 'author' not in i:
@@ -77,6 +77,6 @@ def story(ref):
 
 # scratchpad so I can quickly develop the parser
 if __name__ == '__main__':
-    #print(feed())
+    print(feed())
     #print(story(20763961))
-    print(story(20802050))
+    #print(story(20802050))

apiserver/feeds/manual.py

@@ -33,7 +33,7 @@ def story(ref):
     s['author_link'] = 'https://news.t0.vc'
     s['score'] = 0
     s['date'] = int(time.time())
-    s['title'] = str(soup.title.string)
+    s['title'] = str(soup.title.string) if soup.title else ref
     s['link'] = ref
     s['url'] = ref
     s['comments'] = []

apiserver/feeds/reddit.py

@@ -14,9 +14,9 @@ from prawcore.exceptions import PrawcoreException
 
 from utils import render_md, clean
 
-SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+technology+privacy'
+SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+privacy'
 
-SITE_LINK = lambda x : 'https://old.reddit.com/{}'.format(x)
+SITE_LINK = lambda x : 'https://old.reddit.com{}'.format(x)
 SITE_AUTHOR_LINK = lambda x : 'https://old.reddit.com/u/{}'.format(x)
 
 reddit = praw.Reddit('bot')

apiserver/feeds/tildes.py

@@ -91,7 +91,7 @@ def story(ref):
     s['score'] = int(h.find('span', class_='topic-voting-votes').string)
     s['date'] = unix(h.find('time')['datetime'])
     s['title'] = str(h.h1.string)
-    s['group'] = str(soup.find('a', class_='site-header-context').string)
+    s['group'] = str(soup.find('div', class_='site-header-context').a.string)
     group_lookup[ref] = s['group']
     s['link'] = SITE_LINK(s['group'], ref)
     ud = a.find('div', class_='topic-full-link')
@@ -122,7 +122,7 @@ if __name__ == '__main__':
     #print(self_post)
     #li_comment = story('gqx')
     #print(li_comment)
-    broken = story('l11')
+    broken = story('q4y')
     print(broken)
 
     # make sure there's no self-reference

apiserver/migrate-whoosh-to-sqlite.py

@@ -0,0 +1,74 @@
+import archive
+import database
+import search
+
+import json
+import requests
+
+database.init()
+archive.init()
+search.init()
+
+count = 0
+
+def database_del_story_by_ref(ref):
+    try:
+        session = database.Session()
+        session.query(database.Story).filter(database.Story.ref==ref).delete()
+        session.commit()
+    except:
+        session.rollback()
+        raise
+    finally:
+        session.close()
+
+def search_del_story(sid):
+    try:
+        r = requests.delete(search.MEILI_URL + 'indexes/qotnews/documents/'+sid, timeout=2)
+        if r.status_code != 202:
+            raise Exception('Bad response code ' + str(r.status_code))
+        return r.json()
+    except KeyboardInterrupt:
+        raise
+    except BaseException as e:
+        logging.error('Problem deleting MeiliSearch story: {}'.format(str(e)))
+        return False
+
+with archive.ix.searcher() as searcher:
+    print('count all', searcher.doc_count_all())
+    print('count', searcher.doc_count())
+
+    for doc in searcher.documents():
+        try:
+            print('num', count, 'id', doc['id'])
+            count += 1
+
+            story = doc['story']
+            story.pop('img', None)
+
+            if 'reddit.com/r/technology' in story['link']:
+                print('skipping r/technology')
+                continue
+
+            try:
+                database.put_story(story)
+            except database.IntegrityError:
+                print('collision!')
+                old_story = database.get_story_by_ref(story['ref'])
+                old_story = json.loads(old_story.full_json)
+                if story['num_comments'] > old_story['num_comments']:
+                    print('more comments, replacing')
+                    database_del_story_by_ref(story['ref'])
+                    database.put_story(story)
+                    search_del_story(old_story['id'])
+                else:
+                    print('fewer comments, skipping')
+                    continue
+
+            search.put_story(story)
+            print()
+        except KeyboardInterrupt:
+            break
+        except BaseException as e:
+            print('skipping', doc['id'])
+            print('reason:', e)

apiserver/search.py

@@ -0,0 +1,85 @@
+import logging
+logging.basicConfig(
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        level=logging.DEBUG)
+
+import requests
+
+MEILI_URL = 'http://127.0.0.1:7700/'
+
+def create_index():
+    try:
+        json = dict(name='qotnews', uid='qotnews')
+        r = requests.post(MEILI_URL + 'indexes', json=json, timeout=2)
+        if r.status_code != 201:
+            raise Exception('Bad response code ' + str(r.status_code))
+        return r.json()
+    except KeyboardInterrupt:
+        raise
+    except BaseException as e:
+        logging.error('Problem creating MeiliSearch index: {}'.format(str(e)))
+        return False
+
+def update_rankings():
+    try:
+        json = ['typo', 'words', 'proximity', 'attribute', 'desc(date)', 'wordsPosition', 'exactness']
+        r = requests.post(MEILI_URL + 'indexes/qotnews/settings/ranking-rules', json=json, timeout=2)
+        if r.status_code != 202:
+            raise Exception('Bad response code ' + str(r.status_code))
+        return r.json()
+    except KeyboardInterrupt:
+        raise
+    except BaseException as e:
+        logging.error('Problem setting MeiliSearch ranking rules: {}'.format(str(e)))
+        return False
+
+def update_attributes():
+    try:
+        json = ['title', 'url', 'author', 'link', 'id']
+        r = requests.post(MEILI_URL + 'indexes/qotnews/settings/searchable-attributes', json=json, timeout=2)
+        if r.status_code != 202:
+            raise Exception('Bad response code ' + str(r.status_code))
+        return r.json()
+    except KeyboardInterrupt:
+        raise
+    except BaseException as e:
+        logging.error('Problem setting MeiliSearch searchable attributes: {}'.format(str(e)))
+        return False
+
+def init():
+    create_index()
+    update_rankings()
+    update_attributes()
+
+def put_story(story):
+    story = story.copy()
+    story.pop('text', None)
+    story.pop('comments', None)
+    try:
+        r = requests.post(MEILI_URL + 'indexes/qotnews/documents', json=[story], timeout=2)
+        if r.status_code != 202:
+            raise Exception('Bad response code ' + str(r.status_code))
+        return r.json()
+    except KeyboardInterrupt:
+        raise
+    except BaseException as e:
+        logging.error('Problem putting MeiliSearch story: {}'.format(str(e)))
+        return False
+
+def search(q):
+    try:
+        params = dict(q=q, limit=250)
+        r = requests.get(MEILI_URL + 'indexes/qotnews/search', params=params, timeout=2)
+        if r.status_code != 200:
+            raise Exception('Bad response code ' + str(r.status_code))
+        return r.json()['hits']
+    except KeyboardInterrupt:
+        raise
+    except BaseException as e:
+        logging.error('Problem searching MeiliSearch: {}'.format(str(e)))
+        return False
+    
+if __name__ == '__main__':
+    create_index()
+
+    print(search('the'))

apiserver/server.py

@@ -4,12 +4,14 @@ logging.basicConfig(
         level=logging.INFO)
 
 import copy
+import json
 import threading
+import traceback
 import time
-import shelve
 from urllib.parse import urlparse, parse_qs
 
-import archive
+import database
+import search
 import feed
 from utils import gen_rand_id
 
@@ -23,75 +25,38 @@ from gevent.pywsgi import WSGIServer
 
 monkey.patch_all()
 
-archive.init()
+database.init()
-
+search.init()
-CACHE_LENGTH = 150
-DATA_FILE = 'data/data'
 
+FEED_LENGTH = 75
 news_index = 0
 
-with shelve.open(DATA_FILE) as db:
-    logging.info('Reading caches from disk...')
-    news_list = db.get('news_list', [])
-    news_ref_to_id = db.get('news_ref_to_id', {})
-    news_cache = db.get('news_cache', {})
-
-    # clean cache if broken
-    try:
-        for ref in news_list:
-            nid = news_ref_to_id[ref]
-            _ = news_cache[nid]
-    except KeyError as e:
-        logging.error('Unable to find key {}. Trying to remove...'.format(str(e)))
-        news_list.remove(str(e))
-
-def get_story(sid):
-    if sid in news_cache:
-        return news_cache[sid]
-    else:
-        return archive.get_story(sid)
-
 def new_id():
     nid = gen_rand_id()
-    while nid in news_cache or archive.get_story(nid):
+    while database.get_story(nid):
         nid = gen_rand_id()
     return nid
 
-def remove_ref(old_ref):
-    while old_ref in news_list:
-        news_list.remove(old_ref)
-    old_story = news_cache.pop(news_ref_to_id[old_ref])
-    old_id = news_ref_to_id.pop(old_ref)
-    logging.info('Removed ref {} id {}.'.format(old_ref, old_id))
-
 build_folder = '../webclient/build'
 flask_app = Flask(__name__, template_folder=build_folder, static_folder=build_folder, static_url_path='')
 cors = CORS(flask_app)
 
 @flask_app.route('/api')
 def api():
-    try:
+    stories = database.get_stories(FEED_LENGTH)
-        front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list]
+    # hacky nested json
-    except KeyError as e:
+    res = Response('{"stories":[' + ','.join(stories) + ']}')
-        logging.error('Unable to find key {}. Trying to remove...'.format(str(e)))
+    res.headers['content-type'] = 'application/json'
-        news_list.remove(str(e))
+    return res
-
-    front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']]
-    front_page = front_page[:60]
-    for story in front_page:
-        story.pop('text', None)
-        story.pop('comments', None)
-
-    return {'stories': front_page}
 
 @flask_app.route('/api/search', strict_slashes=False)
-def search():
+def apisearch():
-    search = request.args.get('q', '')
+    q = request.args.get('q', '')
-    if len(search) >= 3:
+    if len(q) >= 3:
-        res = archive.search(search)
+        results = search.search(q)
     else:
-        res = []
+        results = []
-    return {'results': res}
+    return dict(results=results)
 
 @flask_app.route('/api/submit', methods=['POST'], strict_slashes=False)
 def submit():
@@ -109,29 +74,41 @@ def submit():
         elif 'reddit.com' in parse.hostname and 'comments' in url:
             source = 'reddit'
             ref = parse.path.split('/')[4]
+        elif 'news.t0.vc' in parse.hostname:
+            raise Exception('Invalid article')
         else:
             source = 'manual'
             ref = url
 
-        news_story = dict(id=nid, ref=ref, source=source)
+        existing = database.get_story_by_ref(ref)
-        news_cache[nid] = news_story
+        if existing:
-        valid = feed.update_story(news_story, is_manual=True)
+            return {'nid': existing.sid}
+        else:
+            story = dict(id=nid, ref=ref, source=source)
+            valid = feed.update_story(story, is_manual=True)
             if valid:
-            archive.update(news_story)
+                database.put_story(story)
+                search.put_story(story)
                 return {'nid': nid}
             else:
-            news_cache.pop(nid, '')
                 raise Exception('Invalid article')
 
     except BaseException as e:
         logging.error('Problem with article submission: {} - {}'.format(e.__class__.__name__, str(e)))
+        print(traceback.format_exc())
         abort(400)
 
 
 @flask_app.route('/api/<sid>')
 def story(sid):
-    story = get_story(sid)
+    story = database.get_story(sid)
-    return dict(story=story) if story else abort(404)
+    if story:
+        # hacky nested json
+        res = Response('{"story":' + story.full_json + '}')
+        res.headers['content-type'] = 'application/json'
+        return res
+    else:
+        return abort(404)
 
 @flask_app.route('/')
 @flask_app.route('/search')
@@ -149,8 +126,9 @@ def static_story(sid):
     except NotFound:
         pass
 
-    story = get_story(sid)
+    story = database.get_story(sid)
     if not story: return abort(404)
+    story = json.loads(story.full_json)
 
     score = story['score']
     num_comments = story['num_comments']
@@ -167,7 +145,7 @@ def static_story(sid):
             url=url,
             description=description)
 
-http_server = WSGIServer(('', 33842), flask_app)
+http_server = WSGIServer(('', 43842), flask_app)
 
 def feed_thread():
     global news_index
@@ -176,39 +154,42 @@ def feed_thread():
         while True:
             # onboard new stories
             if news_index == 0:
-                feed_list = feed.list()
+                for ref, source in feed.list():
-                new_items = [(ref, source) for ref, source in feed_list if ref not in news_list]
+                    if database.get_story_by_ref(ref):
-                for ref, source in new_items:
+                        continue
-                    news_list.insert(0, ref)
+                    try:
                         nid = new_id()
-                    news_ref_to_id[ref] = nid
+                        database.put_ref(ref, nid, source)
-                    news_cache[nid] = dict(id=nid, ref=ref, source=source)
+                        logging.info('Added ref ' + ref)
+                    except database.IntegrityError:
+                        continue
 
-                if len(new_items):
+            ref_list = database.get_reflist(FEED_LENGTH)
-                    logging.info('Added {} new refs.'.format(len(new_items)))
-
-                # drop old ones
-                while len(news_list) > CACHE_LENGTH:
-                    old_ref = news_list[-1]
-                    remove_ref(old_ref)
 
             # update current stories
-            if news_index < len(news_list):
+            if news_index < len(ref_list):
-                update_ref = news_list[news_index]
+                item = ref_list[news_index]
-                update_id = news_ref_to_id[update_ref]
+
-                news_story = news_cache[update_id]
+                try:
-                valid = feed.update_story(news_story)
+                    story_json = database.get_story(item['sid']).full_json
+                    story = json.loads(story_json)
+                except AttributeError:
+                    story = dict(id=item['sid'], ref=item['ref'], source=item['source'])
+
+                valid = feed.update_story(story)
                 if valid:
-                    archive.update(news_story)
+                    database.put_story(story)
+                    search.put_story(story)
                 else:
-                    remove_ref(update_ref)
+                    database.del_ref(item['ref'])
+                    logging.info('Removed ref {}'.format(item['ref']))
             else:
-                logging.info('Skipping update - no story #' + str(news_index+1))
+                logging.info('Skipping index')
 
             gevent.sleep(6)
 
             news_index += 1
-            if news_index == CACHE_LENGTH: news_index = 0
+            if news_index == FEED_LENGTH: news_index = 0
 
     except KeyboardInterrupt:
         logging.info('Ending feed thread...')
@@ -224,9 +205,3 @@ try:
     http_server.serve_forever()
 except KeyboardInterrupt:
     logging.info('Exiting...')
-finally:
-    with shelve.open(DATA_FILE) as db:
-        logging.info('Writing caches to disk...')
-        db['news_list'] = news_list
-        db['news_ref_to_id'] = news_ref_to_id
-        db['news_cache'] = news_cache

webclient/src/Feed.js

@@ -36,12 +36,6 @@ class Feed extends React.Component {
 									this.props.updateCache(x.id, result.story);
 								}, error => {}
 							);
-
-							if (i < 20 && x.img) {
-								const img = new Image();
-								img.src = x.img;
-								console.log('prefetched image', x.img);
-							}
 						});
 					}
 				},
@@ -65,10 +59,6 @@ class Feed extends React.Component {
 					<div>
 						{stories.map((x, i) =>
 							<div className='item' key={i}>
-								<div className='num'>
-									{i+1}.
-								</div>
-
 								<div className='title'>
 									<Link className='link' to={'/' + x.id}>
 										<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}

webclient/src/Results.js

@@ -64,10 +64,6 @@ class Results extends React.Component {
 						{stories.length ?
 							stories.map((x, i) =>
 								<div className='item' key={i}>
-									<div className='num'>
-										{i+1}.
-									</div>
-
 									<div className='title'>
 										<Link className='link' to={'/' + x.id}>
 											<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}

webclient/src/Search.js

@@ -37,7 +37,7 @@ class Search extends Component {
 			<span className='search'>
 				<form onSubmit={this.searchAgain}>
 					<input
-						placeholder='Search...'
+						placeholder='Search... (fixed)'
 						value={search}
 						onChange={this.searchArticles}
 						ref={this.inputRef}

webclient/src/Style-light.css

@@ -46,7 +46,7 @@ pre {
 .item {
 	display: table;
 	color: #828282;
-	margin-bottom: 0.6rem;
+	margin-bottom: 0.7rem;
 }
 
 .item .source-logo {
@@ -61,11 +61,6 @@ pre {
 	text-decoration: underline;
 }
 
-.item .num {
-	display: table-cell;
-	width: 2em;
-}
-
 .item a.link {
 	font-size: 1.1rem;
 	color: #000000;