Compare commits
28 Commits
e65047fead
...
master
Author | SHA1 | Date | |
---|---|---|---|
9ec61ea5bc | |||
bdc7a6c10d | |||
4858516b01 | |||
f10e6063fc | |||
249a616531 | |||
ab92bd5441 | |||
6b16a768a7 | |||
57de076fec | |||
074b898508 | |||
f049d194ab | |||
c2b9a1cb7a | |||
4435f49e17 | |||
494d89ac30 | |||
e79fca6ecc | |||
c65fb69092 | |||
632d028e4c | |||
ea8e9e5a23 | |||
2838ea9b41 | |||
f15d108971 | |||
f777348af8 | |||
486404a413 | |||
7c9c07a4cf | |||
08d02f6013 | |||
1b54342702 | |||
9e9571a3c0 | |||
dc83a70887 | |||
2e2c9ae837 | |||
61021d8f91 |
@@ -5,7 +5,7 @@ from sqlalchemy.ext.declarative import declarative_base
|
|||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
from sqlalchemy.exc import IntegrityError
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
|
||||||
engine = create_engine('sqlite:///data/qotnews.sqlite', connect_args={'timeout': 180})
|
engine = create_engine('sqlite:///data/qotnews.sqlite', connect_args={'timeout': 360})
|
||||||
Session = sessionmaker(bind=engine)
|
Session = sessionmaker(bind=engine)
|
||||||
|
|
||||||
Base = declarative_base()
|
Base = declarative_base()
|
||||||
@@ -101,7 +101,22 @@ def del_ref(ref):
|
|||||||
finally:
|
finally:
|
||||||
session.close()
|
session.close()
|
||||||
|
|
||||||
|
def count_stories():
|
||||||
|
try:
|
||||||
|
session = Session()
|
||||||
|
return session.query(Story).count()
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
def get_story_list():
|
||||||
|
try:
|
||||||
|
session = Session()
|
||||||
|
return session.query(Story.sid).all()
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
init()
|
init()
|
||||||
|
|
||||||
print(get_story_by_ref('hgi3sy'))
|
#print(get_story_by_ref('hgi3sy'))
|
||||||
|
print(len(get_reflist(99999)))
|
||||||
|
@@ -9,6 +9,7 @@ from bs4 import BeautifulSoup
|
|||||||
|
|
||||||
import settings
|
import settings
|
||||||
from feeds import hackernews, reddit, tildes, manual, lobsters
|
from feeds import hackernews, reddit, tildes, manual, lobsters
|
||||||
|
import utils
|
||||||
|
|
||||||
INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com', 'sec.gov']
|
INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com', 'sec.gov']
|
||||||
TWO_DAYS = 60*60*24*2
|
TWO_DAYS = 60*60*24*2
|
||||||
@@ -68,6 +69,7 @@ def get_content_type(url):
|
|||||||
def update_story(story, is_manual=False):
|
def update_story(story, is_manual=False):
|
||||||
res = {}
|
res = {}
|
||||||
|
|
||||||
|
try:
|
||||||
if story['source'] == 'hackernews':
|
if story['source'] == 'hackernews':
|
||||||
res = hackernews.story(story['ref'])
|
res = hackernews.story(story['ref'])
|
||||||
elif story['source'] == 'lobsters':
|
elif story['source'] == 'lobsters':
|
||||||
@@ -78,6 +80,10 @@ def update_story(story, is_manual=False):
|
|||||||
res = tildes.story(story['ref'])
|
res = tildes.story(story['ref'])
|
||||||
elif story['source'] == 'manual':
|
elif story['source'] == 'manual':
|
||||||
res = manual.story(story['ref'])
|
res = manual.story(story['ref'])
|
||||||
|
except BaseException as e:
|
||||||
|
utils.alert_tanner('Problem updating {} story, ref {}: {}'.format(story['source'], story['ref'], str(e)))
|
||||||
|
logging.exception(e)
|
||||||
|
return False
|
||||||
|
|
||||||
if res:
|
if res:
|
||||||
story.update(res) # join dicts
|
story.update(res) # join dicts
|
||||||
@@ -100,6 +106,12 @@ def update_story(story, is_manual=False):
|
|||||||
logging.info(story['url'])
|
logging.info(story['url'])
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
if 'trump' in story['title'].lower() or 'musk' in story['title'].lower():
|
||||||
|
logging.info('Trump / Musk story, skipping')
|
||||||
|
logging.info(story['url'])
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
logging.info('Getting article ' + story['url'])
|
logging.info('Getting article ' + story['url'])
|
||||||
story['text'] = get_article(story['url'])
|
story['text'] = get_article(story['url'])
|
||||||
if not story['text']: return False
|
if not story['text']: return False
|
||||||
|
@@ -12,7 +12,8 @@ import requests
|
|||||||
from utils import clean
|
from utils import clean
|
||||||
|
|
||||||
API_TOPSTORIES = lambda x: 'https://hacker-news.firebaseio.com/v0/topstories.json'
|
API_TOPSTORIES = lambda x: 'https://hacker-news.firebaseio.com/v0/topstories.json'
|
||||||
API_ITEM = lambda x : 'https://hn.algolia.com/api/v1/items/{}'.format(x)
|
ALG_API_ITEM = lambda x : 'https://hn.algolia.com/api/v1/items/{}'.format(x)
|
||||||
|
BHN_API_ITEM = lambda x : 'https://api.hnpwa.com/v0/item/{}.json'.format(x)
|
||||||
|
|
||||||
SITE_LINK = lambda x : 'https://news.ycombinator.com/item?id={}'.format(x)
|
SITE_LINK = lambda x : 'https://news.ycombinator.com/item?id={}'.format(x)
|
||||||
SITE_AUTHOR_LINK = lambda x : 'https://news.ycombinator.com/user?id={}'.format(x)
|
SITE_AUTHOR_LINK = lambda x : 'https://news.ycombinator.com/user?id={}'.format(x)
|
||||||
@@ -42,7 +43,7 @@ def api(route, ref=None):
|
|||||||
def feed():
|
def feed():
|
||||||
return [str(x) for x in api(API_TOPSTORIES) or []]
|
return [str(x) for x in api(API_TOPSTORIES) or []]
|
||||||
|
|
||||||
def comment(i):
|
def alg_comment(i):
|
||||||
if 'author' not in i:
|
if 'author' not in i:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -51,19 +52,19 @@ def comment(i):
|
|||||||
c['score'] = i.get('points', 0)
|
c['score'] = i.get('points', 0)
|
||||||
c['date'] = i.get('created_at_i', 0)
|
c['date'] = i.get('created_at_i', 0)
|
||||||
c['text'] = clean(i.get('text', '') or '')
|
c['text'] = clean(i.get('text', '') or '')
|
||||||
c['comments'] = [comment(j) for j in i['children']]
|
c['comments'] = [alg_comment(j) for j in i['children']]
|
||||||
c['comments'] = list(filter(bool, c['comments']))
|
c['comments'] = list(filter(bool, c['comments']))
|
||||||
return c
|
return c
|
||||||
|
|
||||||
def comment_count(i):
|
def alg_comment_count(i):
|
||||||
alive = 1 if i['author'] else 0
|
alive = 1 if i['author'] else 0
|
||||||
return sum([comment_count(c) for c in i['comments']]) + alive
|
return sum([alg_comment_count(c) for c in i['comments']]) + alive
|
||||||
|
|
||||||
def story(ref):
|
def alg_story(ref):
|
||||||
r = api(API_ITEM, ref)
|
r = api(ALG_API_ITEM, ref)
|
||||||
if not r:
|
if not r:
|
||||||
logging.info('Bad Hackernews API response.')
|
logging.info('Bad Algolia Hackernews API response.')
|
||||||
return False
|
return None
|
||||||
|
|
||||||
if 'deleted' in r:
|
if 'deleted' in r:
|
||||||
logging.info('Story was deleted.')
|
logging.info('Story was deleted.')
|
||||||
@@ -80,17 +81,85 @@ def story(ref):
|
|||||||
s['title'] = r.get('title', '')
|
s['title'] = r.get('title', '')
|
||||||
s['link'] = SITE_LINK(ref)
|
s['link'] = SITE_LINK(ref)
|
||||||
s['url'] = r.get('url', '')
|
s['url'] = r.get('url', '')
|
||||||
s['comments'] = [comment(i) for i in r['children']]
|
s['comments'] = [alg_comment(i) for i in r['children']]
|
||||||
s['comments'] = list(filter(bool, s['comments']))
|
s['comments'] = list(filter(bool, s['comments']))
|
||||||
s['num_comments'] = comment_count(s) - 1
|
s['num_comments'] = alg_comment_count(s) - 1
|
||||||
|
|
||||||
if 'text' in r and r['text']:
|
if 'text' in r and r['text']:
|
||||||
s['text'] = clean(r['text'] or '')
|
s['text'] = clean(r['text'] or '')
|
||||||
|
|
||||||
return s
|
return s
|
||||||
|
|
||||||
|
def bhn_comment(i):
|
||||||
|
if 'user' not in i:
|
||||||
|
return False
|
||||||
|
|
||||||
|
c = {}
|
||||||
|
c['author'] = i.get('user', '')
|
||||||
|
c['score'] = 0 # Not present?
|
||||||
|
c['date'] = i.get('time', 0)
|
||||||
|
c['text'] = clean(i.get('content', '') or '')
|
||||||
|
c['comments'] = [bhn_comment(j) for j in i['comments']]
|
||||||
|
c['comments'] = list(filter(bool, c['comments']))
|
||||||
|
return c
|
||||||
|
|
||||||
|
def bhn_story(ref):
|
||||||
|
r = api(BHN_API_ITEM, ref)
|
||||||
|
if not r:
|
||||||
|
logging.info('Bad BetterHN Hackernews API response.')
|
||||||
|
return None
|
||||||
|
|
||||||
|
if 'deleted' in r: # TODO: verify
|
||||||
|
logging.info('Story was deleted.')
|
||||||
|
return False
|
||||||
|
elif r.get('dead', False):
|
||||||
|
logging.info('Story was deleted.')
|
||||||
|
return False
|
||||||
|
elif r.get('type', '') != 'link':
|
||||||
|
logging.info('Type "{}" is not "link".'.format(r.get('type', '')))
|
||||||
|
return False
|
||||||
|
|
||||||
|
s = {}
|
||||||
|
s['author'] = r.get('user', '')
|
||||||
|
s['author_link'] = SITE_AUTHOR_LINK(r.get('user', ''))
|
||||||
|
s['score'] = r.get('points', 0)
|
||||||
|
s['date'] = r.get('time', 0)
|
||||||
|
s['title'] = r.get('title', '')
|
||||||
|
s['link'] = SITE_LINK(ref)
|
||||||
|
s['url'] = r.get('url', '')
|
||||||
|
if s['url'].startswith('item'):
|
||||||
|
s['url'] = SITE_LINK(ref)
|
||||||
|
s['comments'] = [bhn_comment(i) for i in r['comments']]
|
||||||
|
s['comments'] = list(filter(bool, s['comments']))
|
||||||
|
s['num_comments'] = r.get('comments_count', 0)
|
||||||
|
|
||||||
|
if 'content' in r and r['content']:
|
||||||
|
s['text'] = clean(r['content'] or '')
|
||||||
|
|
||||||
|
return s
|
||||||
|
|
||||||
|
def story(ref):
|
||||||
|
s = alg_story(ref)
|
||||||
|
if s is None:
|
||||||
|
s = bhn_story(ref)
|
||||||
|
if not s:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
if s['score'] < 25 and s['num_comments'] < 10:
|
||||||
|
logging.info('Score ({}) or num comments ({}) below threshold.'.format(s['score'], s['num_comments']))
|
||||||
|
return False
|
||||||
|
|
||||||
|
return s
|
||||||
|
|
||||||
# scratchpad so I can quickly develop the parser
|
# scratchpad so I can quickly develop the parser
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
print(feed())
|
print(feed())
|
||||||
#print(story(20763961))
|
#print(story(20763961))
|
||||||
#print(story(20802050))
|
#print(story(20802050))
|
||||||
|
|
||||||
|
#print(story(42899834)) # type "job"
|
||||||
|
#print(story(42900076)) # Ask HN
|
||||||
|
#print(story(42898201)) # Show HN
|
||||||
|
#print(story(42899703)) # normal
|
||||||
|
print(story(42902678)) # bad title?
|
||||||
|
@@ -50,7 +50,7 @@ def unix(date_str):
|
|||||||
def make_comment(i):
|
def make_comment(i):
|
||||||
c = {}
|
c = {}
|
||||||
try:
|
try:
|
||||||
c['author'] = i['commenting_user']['username']
|
c['author'] = i['commenting_user']
|
||||||
except KeyError:
|
except KeyError:
|
||||||
c['author'] = ''
|
c['author'] = ''
|
||||||
c['score'] = i.get('score', 0)
|
c['score'] = i.get('score', 0)
|
||||||
@@ -67,13 +67,13 @@ def iter_comments(flat_comments):
|
|||||||
parent_stack = []
|
parent_stack = []
|
||||||
for comment in flat_comments:
|
for comment in flat_comments:
|
||||||
c = make_comment(comment)
|
c = make_comment(comment)
|
||||||
indent = comment['indent_level']
|
indent = comment['depth']
|
||||||
|
|
||||||
if indent == 1:
|
if indent == 0:
|
||||||
nested_comments.append(c)
|
nested_comments.append(c)
|
||||||
parent_stack = [c]
|
parent_stack = [c]
|
||||||
else:
|
else:
|
||||||
parent_stack = parent_stack[:indent-1]
|
parent_stack = parent_stack[:indent]
|
||||||
p = parent_stack[-1]
|
p = parent_stack[-1]
|
||||||
p['comments'].append(c)
|
p['comments'].append(c)
|
||||||
parent_stack.append(c)
|
parent_stack.append(c)
|
||||||
@@ -87,7 +87,7 @@ def story(ref):
|
|||||||
|
|
||||||
s = {}
|
s = {}
|
||||||
try:
|
try:
|
||||||
s['author'] = r['submitter_user']['username']
|
s['author'] = r['submitter_user']
|
||||||
s['author_link'] = SITE_AUTHOR_LINK(s['author'])
|
s['author_link'] = SITE_AUTHOR_LINK(s['author'])
|
||||||
except KeyError:
|
except KeyError:
|
||||||
s['author'] = ''
|
s['author'] = ''
|
||||||
@@ -103,6 +103,10 @@ def story(ref):
|
|||||||
s['comments'] = iter_comments(r['comments'])
|
s['comments'] = iter_comments(r['comments'])
|
||||||
s['num_comments'] = r['comment_count']
|
s['num_comments'] = r['comment_count']
|
||||||
|
|
||||||
|
if s['score'] < 15 and s['num_comments'] < 10:
|
||||||
|
logging.info('Score ({}) or num comments ({}) below threshold.'.format(s['score'], s['num_comments']))
|
||||||
|
return False
|
||||||
|
|
||||||
if 'description' in r and r['description']:
|
if 'description' in r and r['description']:
|
||||||
s['text'] = clean(r['description'] or '')
|
s['text'] = clean(r['description'] or '')
|
||||||
|
|
||||||
@@ -112,5 +116,5 @@ def story(ref):
|
|||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
#print(feed())
|
#print(feed())
|
||||||
import json
|
import json
|
||||||
print(json.dumps(story('fzvd1v')))
|
print(json.dumps(story('fzvd1v'), indent=4))
|
||||||
#print(story(20802050))
|
#print(json.dumps(story('ixyv5u'), indent=4))
|
||||||
|
@@ -32,10 +32,7 @@ def feed():
|
|||||||
return [x.id for x in reddit.subreddit(subs).hot()]
|
return [x.id for x in reddit.subreddit(subs).hot()]
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
raise
|
raise
|
||||||
except PRAWException as e:
|
except BaseException as e:
|
||||||
logging.critical('Problem hitting reddit API: {}'.format(str(e)))
|
|
||||||
return []
|
|
||||||
except PrawcoreException as e:
|
|
||||||
logging.critical('Problem hitting reddit API: {}'.format(str(e)))
|
logging.critical('Problem hitting reddit API: {}'.format(str(e)))
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
@@ -107,7 +107,20 @@ def story(ref):
|
|||||||
ch = a.find('header', class_='topic-comments-header')
|
ch = a.find('header', class_='topic-comments-header')
|
||||||
s['num_comments'] = int(ch.h2.string.split(' ')[0]) if ch else 0
|
s['num_comments'] = int(ch.h2.string.split(' ')[0]) if ch else 0
|
||||||
|
|
||||||
if s['score'] < 8 and s['num_comments'] < 6:
|
if s['group'].split('.')[0] not in [
|
||||||
|
'~arts',
|
||||||
|
'~comp',
|
||||||
|
'~creative',
|
||||||
|
'~design',
|
||||||
|
'~engineering',
|
||||||
|
'~finance',
|
||||||
|
'~science',
|
||||||
|
'~tech',
|
||||||
|
]:
|
||||||
|
logging.info('Group ({}) not in whitelist.'.format(s['group']))
|
||||||
|
return False
|
||||||
|
|
||||||
|
if s['score'] < 15 and s['num_comments'] < 10:
|
||||||
logging.info('Score ({}) or num comments ({}) below threshold.'.format(s['score'], s['num_comments']))
|
logging.info('Score ({}) or num comments ({}) below threshold.'.format(s['score'], s['num_comments']))
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
@@ -1,6 +1,8 @@
|
|||||||
import database
|
import database
|
||||||
import search
|
import search
|
||||||
import sys
|
import sys
|
||||||
|
import settings
|
||||||
|
import logging
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import requests
|
import requests
|
||||||
@@ -21,7 +23,7 @@ def database_del_story(sid):
|
|||||||
|
|
||||||
def search_del_story(sid):
|
def search_del_story(sid):
|
||||||
try:
|
try:
|
||||||
r = requests.delete(search.MEILI_URL + 'indexes/qotnews/documents/'+sid, timeout=2)
|
r = requests.delete(settings.MEILI_URL + 'indexes/qotnews/documents/'+sid, timeout=2)
|
||||||
if r.status_code != 202:
|
if r.status_code != 202:
|
||||||
raise Exception('Bad response code ' + str(r.status_code))
|
raise Exception('Bad response code ' + str(r.status_code))
|
||||||
return r.json()
|
return r.json()
|
58
apiserver/scripts/fix-stories.py
Normal file
58
apiserver/scripts/fix-stories.py
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
import time
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import feed
|
||||||
|
import database
|
||||||
|
import search
|
||||||
|
|
||||||
|
database.init()
|
||||||
|
|
||||||
|
def fix_gzip_bug(story_list):
|
||||||
|
FIX_THRESHOLD = 150
|
||||||
|
|
||||||
|
count = 1
|
||||||
|
for sid in story_list:
|
||||||
|
try:
|
||||||
|
sid = sid[0]
|
||||||
|
story = database.get_story(sid)
|
||||||
|
full_json = json.loads(story.full_json)
|
||||||
|
meta_json = json.loads(story.meta_json)
|
||||||
|
text = full_json.get('text', '')
|
||||||
|
|
||||||
|
count = text.count('<EFBFBD>')
|
||||||
|
if not count: continue
|
||||||
|
|
||||||
|
ratio = count / len(text) * 1000
|
||||||
|
print('Bad story:', sid, 'Num ?:', count, 'Ratio:', ratio)
|
||||||
|
if ratio < FIX_THRESHOLD: continue
|
||||||
|
|
||||||
|
print('Attempting to fix...')
|
||||||
|
|
||||||
|
valid = feed.update_story(meta_json, is_manual=True)
|
||||||
|
if valid:
|
||||||
|
database.put_story(meta_json)
|
||||||
|
search.put_story(meta_json)
|
||||||
|
print('Success')
|
||||||
|
else:
|
||||||
|
print('Story was not valid')
|
||||||
|
|
||||||
|
time.sleep(3)
|
||||||
|
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
raise
|
||||||
|
except BaseException as e:
|
||||||
|
logging.exception(e)
|
||||||
|
breakpoint()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
num_stories = database.count_stories()
|
||||||
|
|
||||||
|
print('Fix {} stories?'.format(num_stories))
|
||||||
|
print('Press ENTER to continue, ctrl-c to cancel')
|
||||||
|
input()
|
||||||
|
|
||||||
|
story_list = database.get_story_list()
|
||||||
|
|
||||||
|
fix_gzip_bug(story_list)
|
||||||
|
|
@@ -23,28 +23,14 @@ def put_stories(stories):
|
|||||||
def get_update(update_id):
|
def get_update(update_id):
|
||||||
return search.meili_api(requests.get, 'tasks/{}'.format(update_id))
|
return search.meili_api(requests.get, 'tasks/{}'.format(update_id))
|
||||||
|
|
||||||
def count_stories():
|
|
||||||
try:
|
|
||||||
session = database.Session()
|
|
||||||
return session.query(database.Story).count()
|
|
||||||
finally:
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
def get_story_list():
|
|
||||||
try:
|
|
||||||
session = database.Session()
|
|
||||||
return session.query(database.Story.sid).all()
|
|
||||||
finally:
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
num_stories = count_stories()
|
num_stories = database.count_stories()
|
||||||
|
|
||||||
print('Reindex {} stories?'.format(num_stories))
|
print('Reindex {} stories?'.format(num_stories))
|
||||||
print('Press ENTER to continue, ctrl-c to cancel')
|
print('Press ENTER to continue, ctrl-c to cancel')
|
||||||
input()
|
input()
|
||||||
|
|
||||||
story_list = get_story_list()
|
story_list = database.get_story_list()
|
||||||
|
|
||||||
count = 1
|
count = 1
|
||||||
while len(story_list):
|
while len(story_list):
|
@@ -53,7 +53,7 @@ def put_story(story):
|
|||||||
|
|
||||||
def search(q):
|
def search(q):
|
||||||
if not SEARCH_ENABLED: return []
|
if not SEARCH_ENABLED: return []
|
||||||
params = dict(q=q, limit=100)
|
params = dict(q=q, limit=settings.FEED_LENGTH)
|
||||||
r = meili_api(requests.get, 'indexes/qotnews/search', params=params, parse_json=False)
|
r = meili_api(requests.get, 'indexes/qotnews/search', params=params, parse_json=False)
|
||||||
return r
|
return r
|
||||||
|
|
||||||
|
@@ -15,6 +15,7 @@ import traceback
|
|||||||
import time
|
import time
|
||||||
from urllib.parse import urlparse, parse_qs
|
from urllib.parse import urlparse, parse_qs
|
||||||
|
|
||||||
|
import settings
|
||||||
import database
|
import database
|
||||||
import search
|
import search
|
||||||
import feed
|
import feed
|
||||||
@@ -27,7 +28,6 @@ from flask_cors import CORS
|
|||||||
database.init()
|
database.init()
|
||||||
search.init()
|
search.init()
|
||||||
|
|
||||||
FEED_LENGTH = 75
|
|
||||||
news_index = 0
|
news_index = 0
|
||||||
|
|
||||||
def new_id():
|
def new_id():
|
||||||
@@ -43,7 +43,7 @@ cors = CORS(flask_app)
|
|||||||
@flask_app.route('/api')
|
@flask_app.route('/api')
|
||||||
def api():
|
def api():
|
||||||
skip = request.args.get('skip', 0)
|
skip = request.args.get('skip', 0)
|
||||||
limit = request.args.get('limit', FEED_LENGTH)
|
limit = request.args.get('limit', settings.FEED_LENGTH)
|
||||||
stories = database.get_stories(limit, skip)
|
stories = database.get_stories(limit, skip)
|
||||||
# hacky nested json
|
# hacky nested json
|
||||||
res = Response('{"stories":[' + ','.join(stories) + ']}')
|
res = Response('{"stories":[' + ','.join(stories) + ']}')
|
||||||
@@ -177,7 +177,7 @@ def feed_thread():
|
|||||||
logging.info('Already have ID / ref, skipping.')
|
logging.info('Already have ID / ref, skipping.')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
ref_list = database.get_reflist(FEED_LENGTH)
|
ref_list = database.get_reflist(settings.FEED_LENGTH)
|
||||||
|
|
||||||
# update current stories
|
# update current stories
|
||||||
if news_index < len(ref_list):
|
if news_index < len(ref_list):
|
||||||
@@ -204,7 +204,7 @@ def feed_thread():
|
|||||||
gevent.sleep(6)
|
gevent.sleep(6)
|
||||||
|
|
||||||
news_index += 1
|
news_index += 1
|
||||||
if news_index == FEED_LENGTH: news_index = 0
|
if news_index == settings.FEED_LENGTH: news_index = 0
|
||||||
|
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
logging.info('Ending feed thread...')
|
logging.info('Ending feed thread...')
|
||||||
|
@@ -4,6 +4,7 @@
|
|||||||
# Feed Lengths
|
# Feed Lengths
|
||||||
# Number of top items from each site to pull
|
# Number of top items from each site to pull
|
||||||
# set to 0 to disable that site
|
# set to 0 to disable that site
|
||||||
|
FEED_LENGTH = 75
|
||||||
NUM_HACKERNEWS = 15
|
NUM_HACKERNEWS = 15
|
||||||
NUM_LOBSTERS = 10
|
NUM_LOBSTERS = 10
|
||||||
NUM_REDDIT = 15
|
NUM_REDDIT = 15
|
||||||
|
@@ -8,6 +8,14 @@ import string
|
|||||||
|
|
||||||
from bleach.sanitizer import Cleaner
|
from bleach.sanitizer import Cleaner
|
||||||
|
|
||||||
|
def alert_tanner(message):
|
||||||
|
try:
|
||||||
|
logging.info('Alerting Tanner: ' + message)
|
||||||
|
params = dict(qotnews=message)
|
||||||
|
requests.get('https://tbot.tannercollin.com/message', params=params, timeout=4)
|
||||||
|
except BaseException as e:
|
||||||
|
logging.error('Problem alerting Tanner: ' + str(e))
|
||||||
|
|
||||||
def gen_rand_id():
|
def gen_rand_id():
|
||||||
return ''.join(random.choice(string.ascii_uppercase) for _ in range(4))
|
return ''.join(random.choice(string.ascii_uppercase) for _ in range(4))
|
||||||
|
|
||||||
|
@@ -708,8 +708,8 @@ raw-body@2.4.3:
|
|||||||
unpipe "1.0.0"
|
unpipe "1.0.0"
|
||||||
|
|
||||||
"readability@https://github.com/mozilla/readability":
|
"readability@https://github.com/mozilla/readability":
|
||||||
version "0.4.2"
|
version "0.5.0"
|
||||||
resolved "https://github.com/mozilla/readability#1d2cb030b32e753cc4b7c4ce8b64c3ce4dc1b2ff"
|
resolved "https://github.com/mozilla/readability#39a5c5409fb653858b1832141895b882b9092b47"
|
||||||
|
|
||||||
request-promise-core@1.1.4:
|
request-promise-core@1.1.4:
|
||||||
version "1.1.4"
|
version "1.1.4"
|
||||||
|
@@ -3,8 +3,10 @@ import { BrowserRouter as Router, Route, Link, Switch } from 'react-router-dom';
|
|||||||
import localForage from 'localforage';
|
import localForage from 'localforage';
|
||||||
import './Style-light.css';
|
import './Style-light.css';
|
||||||
import './Style-dark.css';
|
import './Style-dark.css';
|
||||||
|
import './Style-black.css';
|
||||||
|
import './Style-red.css';
|
||||||
import './fonts/Fonts.css';
|
import './fonts/Fonts.css';
|
||||||
import { ForwardDot } from './utils.js';
|
import { BackwardDot, ForwardDot } from './utils.js';
|
||||||
import Feed from './Feed.js';
|
import Feed from './Feed.js';
|
||||||
import Article from './Article.js';
|
import Article from './Article.js';
|
||||||
import Comments from './Comments.js';
|
import Comments from './Comments.js';
|
||||||
@@ -38,6 +40,16 @@ class App extends React.Component {
|
|||||||
localStorage.setItem('theme', 'dark');
|
localStorage.setItem('theme', 'dark');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
black() {
|
||||||
|
this.setState({ theme: 'black' });
|
||||||
|
localStorage.setItem('theme', 'black');
|
||||||
|
}
|
||||||
|
|
||||||
|
red() {
|
||||||
|
this.setState({ theme: 'red' });
|
||||||
|
localStorage.setItem('theme', 'red');
|
||||||
|
}
|
||||||
|
|
||||||
componentDidMount() {
|
componentDidMount() {
|
||||||
if (!this.cache.length) {
|
if (!this.cache.length) {
|
||||||
localForage.iterate((value, key) => {
|
localForage.iterate((value, key) => {
|
||||||
@@ -47,9 +59,40 @@ class App extends React.Component {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
goFullScreen() {
|
||||||
|
if ('wakeLock' in navigator) {
|
||||||
|
navigator.wakeLock.request('screen');
|
||||||
|
}
|
||||||
|
|
||||||
|
document.body.requestFullscreen({ navigationUI: 'hide' }).then(() => {
|
||||||
|
window.addEventListener('resize', () => this.forceUpdate());
|
||||||
|
this.forceUpdate();
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
exitFullScreen() {
|
||||||
|
document.exitFullscreen().then(() => {
|
||||||
|
this.forceUpdate();
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
render() {
|
render() {
|
||||||
const theme = this.state.theme;
|
const theme = this.state.theme;
|
||||||
document.body.style.backgroundColor = theme === 'dark' ? '#000' : '#eeeeee';
|
|
||||||
|
if (theme === 'dark') {
|
||||||
|
document.body.style.backgroundColor = '#1a1a1a';
|
||||||
|
} else if (theme === 'black') {
|
||||||
|
document.body.style.backgroundColor = '#000';
|
||||||
|
} else if (theme === 'red') {
|
||||||
|
document.body.style.backgroundColor = '#000';
|
||||||
|
} else {
|
||||||
|
document.body.style.backgroundColor = '#eeeeee';
|
||||||
|
}
|
||||||
|
|
||||||
|
const fullScreenAvailable = document.fullscreenEnabled ||
|
||||||
|
document.mozFullscreenEnabled ||
|
||||||
|
document.webkitFullscreenEnabled ||
|
||||||
|
document.msFullscreenEnabled;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className={theme}>
|
<div className={theme}>
|
||||||
@@ -58,12 +101,19 @@ class App extends React.Component {
|
|||||||
<p>
|
<p>
|
||||||
<Link to='/'>QotNews</Link>
|
<Link to='/'>QotNews</Link>
|
||||||
|
|
||||||
<span className='theme'>Theme: <a href='#' onClick={() => this.light()}>Light</a> - <a href='#' onClick={() => this.dark()}>Dark</a></span>
|
<span className='theme'><a href='#' onClick={() => this.light()}>Light</a> - <a href='#' onClick={() => this.dark()}>Dark</a> - <a href='#' onClick={() => this.black()}>Black</a> - <a href='#' onClick={() => this.red()}>Red</a></span>
|
||||||
<br />
|
<br />
|
||||||
<span className='slogan'>Hacker News, Reddit, Lobsters, and Tildes articles rendered in reader mode.</span>
|
<span className='slogan'>Hacker News, Reddit, Lobsters, and Tildes articles rendered in reader mode.</span>
|
||||||
</p>
|
</p>
|
||||||
<Route path='/(|search)' component={Search} />
|
<Route path='/(|search)' component={Search} />
|
||||||
<Route path='/(|search)' component={Submit} />
|
<Route path='/(|search)' component={Submit} />
|
||||||
|
{fullScreenAvailable &&
|
||||||
|
<Route path='/(|search)' render={() => !document.fullscreenElement ?
|
||||||
|
<button className='fullscreen' onClick={() => this.goFullScreen()}>Enter Fullscreen</button>
|
||||||
|
:
|
||||||
|
<button className='fullscreen' onClick={() => this.exitFullScreen()}>Exit Fullscreen</button>
|
||||||
|
} />
|
||||||
|
}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<Route path='/' exact render={(props) => <Feed {...props} updateCache={this.updateCache} />} />
|
<Route path='/' exact render={(props) => <Feed {...props} updateCache={this.updateCache} />} />
|
||||||
@@ -73,6 +123,7 @@ class App extends React.Component {
|
|||||||
</Switch>
|
</Switch>
|
||||||
<Route path='/:id/c' exact render={(props) => <Comments {...props} cache={this.cache} />} />
|
<Route path='/:id/c' exact render={(props) => <Comments {...props} cache={this.cache} />} />
|
||||||
|
|
||||||
|
<BackwardDot />
|
||||||
<ForwardDot />
|
<ForwardDot />
|
||||||
|
|
||||||
<ScrollToTop />
|
<ScrollToTop />
|
||||||
|
@@ -15,6 +15,7 @@ class ScrollToTop extends React.Component {
|
|||||||
}
|
}
|
||||||
|
|
||||||
window.scrollTo(0, 0);
|
window.scrollTo(0, 0);
|
||||||
|
document.body.scrollTop = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
render() {
|
render() {
|
||||||
|
68
webclient/src/Style-black.css
Normal file
68
webclient/src/Style-black.css
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
.black {
|
||||||
|
color: #ddd;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black a {
|
||||||
|
color: #ddd;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black input {
|
||||||
|
color: #ddd;
|
||||||
|
border: 1px solid #828282;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black button {
|
||||||
|
background-color: #444444;
|
||||||
|
border-color: #bbb;
|
||||||
|
color: #ddd;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .item {
|
||||||
|
color: #828282;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .item .source-logo {
|
||||||
|
filter: grayscale(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .item a {
|
||||||
|
color: #828282;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .item a.link {
|
||||||
|
color: #ddd;
|
||||||
|
}
|
||||||
|
.black .item a.link:visited {
|
||||||
|
color: #828282;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .item .info a.hot {
|
||||||
|
color: #cccccc;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .article a {
|
||||||
|
border-bottom: 1px solid #aaaaaa;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .article u {
|
||||||
|
border-bottom: 1px solid #aaaaaa;
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .story-text video,
|
||||||
|
.black .story-text img {
|
||||||
|
filter: brightness(50%);
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .article .info {
|
||||||
|
color: #828282;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .article .info a {
|
||||||
|
border-bottom: none;
|
||||||
|
color: #828282;
|
||||||
|
}
|
||||||
|
|
||||||
|
.black .comment.lined {
|
||||||
|
border-left: 1px solid #444444;
|
||||||
|
}
|
@@ -11,12 +11,14 @@
|
|||||||
border: 1px solid #828282;
|
border: 1px solid #828282;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dark .item {
|
.dark button {
|
||||||
color: #828282;
|
background-color: #444444;
|
||||||
|
border-color: #bbb;
|
||||||
|
color: #ddd;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dark .item .source-logo {
|
.dark .item {
|
||||||
filter: grayscale(1);
|
color: #828282;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dark .item a {
|
.dark .item a {
|
||||||
@@ -43,6 +45,7 @@
|
|||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.dark .story-text video,
|
||||||
.dark .story-text img {
|
.dark .story-text img {
|
||||||
filter: brightness(50%);
|
filter: brightness(50%);
|
||||||
}
|
}
|
||||||
|
@@ -2,9 +2,30 @@ body {
|
|||||||
text-rendering: optimizeLegibility;
|
text-rendering: optimizeLegibility;
|
||||||
font: 1rem/1.3 sans-serif;
|
font: 1rem/1.3 sans-serif;
|
||||||
color: #000000;
|
color: #000000;
|
||||||
margin-bottom: 100vh;
|
|
||||||
word-break: break-word;
|
word-break: break-word;
|
||||||
font-kerning: normal;
|
font-kerning: normal;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
::backdrop {
|
||||||
|
background-color: rgba(0,0,0,0);
|
||||||
|
}
|
||||||
|
|
||||||
|
body:fullscreen {
|
||||||
|
overflow-y: scroll !important;
|
||||||
|
}
|
||||||
|
body:-ms-fullscreen {
|
||||||
|
overflow-y: scroll !important;
|
||||||
|
}
|
||||||
|
body:-webkit-full-screen {
|
||||||
|
overflow-y: scroll !important;
|
||||||
|
}
|
||||||
|
body:-moz-full-screen {
|
||||||
|
overflow-y: scroll !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
#root {
|
||||||
|
margin: 8px 8px 100vh 8px !important;
|
||||||
}
|
}
|
||||||
|
|
||||||
a {
|
a {
|
||||||
@@ -22,6 +43,12 @@ input {
|
|||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.fullscreen {
|
||||||
|
margin: 0.25rem;
|
||||||
|
padding: 0.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
pre {
|
pre {
|
||||||
overflow: auto;
|
overflow: auto;
|
||||||
}
|
}
|
||||||
@@ -185,16 +212,20 @@ span.source {
|
|||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
}
|
}
|
||||||
|
|
||||||
.toggleDot {
|
.dot {
|
||||||
|
cursor: pointer;
|
||||||
position: fixed;
|
position: fixed;
|
||||||
bottom: 1rem;
|
|
||||||
left: 1rem;
|
|
||||||
height: 3rem;
|
height: 3rem;
|
||||||
width: 3rem;
|
width: 3rem;
|
||||||
background-color: #828282;
|
background-color: #828282;
|
||||||
border-radius: 50%;
|
border-radius: 50%;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.toggleDot {
|
||||||
|
bottom: 1rem;
|
||||||
|
left: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
.toggleDot .button {
|
.toggleDot .button {
|
||||||
font: 2rem/1 'icomoon';
|
font: 2rem/1 'icomoon';
|
||||||
position: relative;
|
position: relative;
|
||||||
@@ -203,21 +234,27 @@ span.source {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.forwardDot {
|
.forwardDot {
|
||||||
cursor: pointer;
|
|
||||||
position: fixed;
|
|
||||||
bottom: 1rem;
|
bottom: 1rem;
|
||||||
right: 1rem;
|
right: 1rem;
|
||||||
height: 3rem;
|
|
||||||
width: 3rem;
|
|
||||||
background-color: #828282;
|
|
||||||
border-radius: 50%;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.forwardDot .button {
|
.forwardDot .button {
|
||||||
font: 2.5rem/1 'icomoon';
|
font: 2rem/1 'icomoon';
|
||||||
position: relative;
|
position: relative;
|
||||||
top: 0.25rem;
|
top: 0.5rem;
|
||||||
left: 0.3rem;
|
left: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.backwardDot {
|
||||||
|
bottom: 1rem;
|
||||||
|
right: 5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.backwardDot .button {
|
||||||
|
font: 2rem/1 'icomoon';
|
||||||
|
position: relative;
|
||||||
|
top: 0.5rem;
|
||||||
|
left: 0.5rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.search form {
|
.search form {
|
||||||
|
82
webclient/src/Style-red.css
Normal file
82
webclient/src/Style-red.css
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
.red {
|
||||||
|
color: #b00;
|
||||||
|
scrollbar-color: #b00 #440000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red a {
|
||||||
|
color: #b00;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red input {
|
||||||
|
color: #b00;
|
||||||
|
border: 1px solid #690000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red input::placeholder {
|
||||||
|
color: #690000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red hr {
|
||||||
|
background-color: #690000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red button {
|
||||||
|
background-color: #440000;
|
||||||
|
border-color: #b00;
|
||||||
|
color: #b00;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .item,
|
||||||
|
.red .slogan {
|
||||||
|
color: #690000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .item .source-logo {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .item a {
|
||||||
|
color: #690000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .item a.link {
|
||||||
|
color: #b00;
|
||||||
|
}
|
||||||
|
.red .item a.link:visited {
|
||||||
|
color: #690000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .item .info a.hot {
|
||||||
|
color: #cc0000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .article a {
|
||||||
|
border-bottom: 1px solid #aa0000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .article u {
|
||||||
|
border-bottom: 1px solid #aa0000;
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .story-text video,
|
||||||
|
.red .story-text img {
|
||||||
|
filter: grayscale(100%) brightness(20%) sepia(100%) hue-rotate(-50deg) saturate(600%) contrast(0.8);
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .article .info {
|
||||||
|
color: #690000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .article .info a {
|
||||||
|
border-bottom: none;
|
||||||
|
color: #690000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .comment.lined {
|
||||||
|
border-left: 1px solid #440000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.red .dot {
|
||||||
|
background-color: #440000;
|
||||||
|
}
|
@@ -41,7 +41,7 @@ class Submit extends Component {
|
|||||||
<span className='search'>
|
<span className='search'>
|
||||||
<form onSubmit={this.submitArticle}>
|
<form onSubmit={this.submitArticle}>
|
||||||
<input
|
<input
|
||||||
placeholder='Submit Article'
|
placeholder='Submit URL'
|
||||||
ref={this.inputRef}
|
ref={this.inputRef}
|
||||||
/>
|
/>
|
||||||
</form>
|
</form>
|
||||||
|
Binary file not shown.
@@ -25,8 +25,9 @@ export class ToggleDot extends React.Component {
|
|||||||
render() {
|
render() {
|
||||||
const id = this.props.id;
|
const id = this.props.id;
|
||||||
const article = this.props.article;
|
const article = this.props.article;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className='toggleDot'>
|
<div className='dot toggleDot'>
|
||||||
<div className='button'>
|
<div className='button'>
|
||||||
<Link to={'/' + id + (article ? '' : '/c')}>
|
<Link to={'/' + id + (article ? '' : '/c')}>
|
||||||
{article ? '' : ''}
|
{article ? '' : ''}
|
||||||
@@ -37,6 +38,27 @@ export class ToggleDot extends React.Component {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class BackwardDot extends React.Component {
|
||||||
|
goBackward() {
|
||||||
|
localStorage.setItem('scrollLock', 'True');
|
||||||
|
window.history.back();
|
||||||
|
}
|
||||||
|
|
||||||
|
render() {
|
||||||
|
const isMobile = /iPhone|iPad|iPod|Android/i.test(navigator.userAgent);
|
||||||
|
if (!isMobile) return null;
|
||||||
|
if (!document.fullscreenElement) return null;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className='dot backwardDot' onClick={this.goBackward}>
|
||||||
|
<div className='button'>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export class ForwardDot extends React.Component {
|
export class ForwardDot extends React.Component {
|
||||||
goForward() {
|
goForward() {
|
||||||
localStorage.setItem('scrollLock', 'True');
|
localStorage.setItem('scrollLock', 'True');
|
||||||
@@ -48,9 +70,9 @@ export class ForwardDot extends React.Component {
|
|||||||
if (!isMobile) return null;
|
if (!isMobile) return null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className='forwardDot' onClick={this.goForward}>
|
<div className='dot forwardDot' onClick={this.goForward}>
|
||||||
<div className='button'>
|
<div className='button'>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
Reference in New Issue
Block a user