apiserver/.gitignore

@@ -109,5 +109,4 @@ settings.py
 data.db
 data.db.bak
 data/archive/*
-data/backup/*
 qotnews.sqlite

apiserver/database.py

@@ -5,7 +5,7 @@ from sqlalchemy.orm import sessionmaker
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.types import JSON
 
-engine = create_engine('sqlite:///data/qotnews.sqlite', connect_args={'timeout': 120})
+engine = create_engine('sqlite:///data/qotnews.sqlite')
 Session = sessionmaker(bind=engine)
 
 Base = declarative_base()

apiserver/feed.py

@@ -9,13 +9,13 @@ from bs4 import BeautifulSoup
 import itertools
 
 import settings
-from feeds import hackernews, reddit, tildes, substack, manual, lobsters
+from feeds import hackernews, reddit, tildes, substack, manual
 from feeds.sitemap import Sitemap
 from feeds.category import Category
 from scrapers import outline
 from scrapers.declutter import declutter, headless, simple
 
-INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com', 'sec.gov']
+INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com']
 
 substacks = {}
 for key, value in settings.SUBSTACK.items():
@@ -33,9 +33,6 @@ def get_list():
     if settings.NUM_HACKERNEWS:
         feeds['hackernews'] = [(x, 'hackernews', x) for x in hackernews.feed()[:settings.NUM_HACKERNEWS]]
 
-    if settings.NUM_LOBSTERS:
-        feed += [(x, 'lobsters', x) for x in lobsters.feed()[:settings.NUM_LOBSTERS]]
-
     if settings.NUM_REDDIT:
         feeds['reddit'] = [(x, 'reddit', x) for x in reddit.feed()[:settings.NUM_REDDIT]]
 
@@ -110,8 +107,6 @@ def update_story(story, is_manual=False, urlref=None):
 
     if story['source'] == 'hackernews':
         res = hackernews.story(story['ref'])
-    elif story['source'] == 'lobsters':
-        res = lobsters.story(story['ref'])
     elif story['source'] == 'reddit':
         res = reddit.story(story['ref'])
     elif story['source'] == 'tildes':

apiserver/feeds/lobsters.py

@@ -1,113 +0,0 @@
-import logging
-logging.basicConfig(
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-        level=logging.DEBUG)
-
-if __name__ == '__main__':
-    import sys
-    sys.path.insert(0,'.')
-
-import requests
-from datetime import datetime
-
-from utils import clean
-
-API_HOTTEST = lambda x: 'https://lobste.rs/hottest.json'
-API_ITEM = lambda x : 'https://lobste.rs/s/{}.json'.format(x)
-
-SITE_LINK = lambda x : 'https://lobste.rs/s/{}'.format(x)
-SITE_AUTHOR_LINK = lambda x : 'https://lobste.rs/u/{}'.format(x)
-
-def api(route, ref=None):
-    try:
-        r = requests.get(route(ref), timeout=5)
-        if r.status_code != 200:
-            raise Exception('Bad response code ' + str(r.status_code))
-        return r.json()
-    except KeyboardInterrupt:
-        raise
-    except BaseException as e:
-        logging.error('Problem hitting lobsters API: {}, trying again'.format(str(e)))
-
-    try:
-        r = requests.get(route(ref), timeout=15)
-        if r.status_code != 200:
-            raise Exception('Bad response code ' + str(r.status_code))
-        return r.json()
-    except KeyboardInterrupt:
-        raise
-    except BaseException as e:
-        logging.error('Problem hitting lobsters API: {}'.format(str(e)))
-        return False
-
-def feed():
-    return [x['short_id'] for x in api(API_HOTTEST) or []]
-
-def unix(date_str):
-    return int(datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%f%z').timestamp())
-
-def make_comment(i):
-    c = {}
-    try:
-        c['author'] = i['commenting_user']['username']
-    except KeyError:
-        c['author'] = ''
-    c['score'] = i.get('score', 0)
-    try:
-        c['date'] = unix(i['created_at'])
-    except KeyError:
-        c['date'] = 0
-    c['text'] = clean(i.get('comment', '') or '')
-    c['comments'] = []
-    return c
-
-def iter_comments(flat_comments):
-    nested_comments = []
-    parent_stack = []
-    for comment in flat_comments:
-        c = make_comment(comment)
-        indent = comment['indent_level']
-
-        if indent == 1:
-            nested_comments.append(c)
-            parent_stack = [c]
-        else:
-            parent_stack = parent_stack[:indent-1]
-            p = parent_stack[-1]
-            p['comments'].append(c)
-            parent_stack.append(c)
-    return nested_comments
-
-def story(ref):
-    r = api(API_ITEM, ref)
-    if not r: return False
-
-    s = {}
-    try:
-        s['author'] = r['submitter_user']['username']
-        s['author_link'] = SITE_AUTHOR_LINK(s['author'])
-    except KeyError:
-        s['author'] = ''
-        s['author_link'] = ''
-    s['score'] = r.get('score', 0)
-    try:
-        s['date'] = unix(r['created_at'])
-    except KeyError:
-        s['date'] = 0
-    s['title'] = r.get('title', '')
-    s['link'] = SITE_LINK(ref)
-    s['url'] = r.get('url', '')
-    s['comments'] = iter_comments(r['comments'])
-    s['num_comments'] = r['comment_count']
-
-    if 'description' in r and r['description']:
-        s['text'] = clean(r['description'] or '')
-
-    return s
-
-# scratchpad so I can quickly develop the parser
-if __name__ == '__main__':
-    #print(feed())
-    import json
-    print(json.dumps(story('fzvd1v')))
-    #print(story(20802050))

apiserver/server.py

@@ -72,9 +72,6 @@ def submit():
         elif 'tildes.net' in parse.hostname and '~' in url:
             source = 'tildes'
             ref = parse.path.split('/')[2]
-        elif 'lobste.rs' in parse.hostname and '/s/' in url:
-            source = 'lobsters'
-            ref = parse.path.split('/')[2]
         elif 'reddit.com' in parse.hostname and 'comments' in url:
             source = 'reddit'
             ref = parse.path.split('/')[4]
@@ -123,7 +120,7 @@ def index():
     return render_template('index.html',
             title='Feed',
             url=settings.HOSTNAME,
-            description='Hacker News, Reddit, Lobsters, and Tildes articles rendered in reader mode')
+            description='Reddit, Hacker News, and Tildes combined, then pre-rendered in reader mode')
 
 @flask_app.route('/<sid>', strict_slashes=False)
 @flask_app.route('/<sid>/c', strict_slashes=False)

apiserver/settings.py.example

@@ -13,7 +13,6 @@ HEADLESS_READER_PORT = 33843
 # Number of top items from each site to pull
 # set to 0 to disable that site
 NUM_HACKERNEWS = 15
-NUM_LOBSTERS = 10
 NUM_REDDIT = 10
 NUM_TILDES = 5
 NUM_SUBSTACK = 10
@@ -75,6 +74,8 @@ SUBREDDITS = [
     'HistoryofIdeas',
     'LaymanJournals',
     'PhilosophyofScience',
+    'PoliticsPDFs',
+    'Scholar',
     'StateOfTheUnion',
     'TheAgora',
     'TrueFilm',
@@ -88,7 +89,4 @@ SUBREDDITS = [
     'neurophilosophy',
     'resilientcommunities',
     'worldevents',
-    'StallmanWasRight',
-    'DarkFuturology',
-    'EverythingScience',
 ]

webclient/src/App.js

@@ -66,7 +66,7 @@ class App extends React.Component {
 							<Link to='/'>QotNews - Feed</Link>
 							<span className='theme'>Theme: <a href='#' onClick={() => this.light()}>Light</a> - <a href='#' onClick={() => this.dark()}>Dark</a></span>
 							<br />
-							<span className='slogan'>Hacker News, Reddit, Lobsters, and Tildes articles rendered in reader mode.</span>
+							<span className='slogan'>Reddit, Hacker News, and Tildes combined, then pre-rendered in reader mode.</span>
 						</p>
 						<Route path='/(|search)' component={Search} />
 						<Route path='/(|search)' component={Submit} />