Compare commits

..

No commits in common. "9a449bf3ca1f8a2864f6075b2c96d933294643d4" and "b252c6a207c00c25784249f67b157dbf4ba426ea" have entirely different histories.

14 changed files with 144 additions and 364 deletions

View File

@ -109,4 +109,3 @@ praw.ini
data.db data.db
data.db.bak data.db.bak
data/archive/* data/archive/*
qotnews.sqlite

View File

@ -1,106 +0,0 @@
import json
from sqlalchemy import create_engine, Column, String, ForeignKey, Integer
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError
engine = create_engine('sqlite:///data/qotnews.sqlite')
Session = sessionmaker(bind=engine)
Base = declarative_base()
class Story(Base):
__tablename__ = 'stories'
sid = Column(String(16), primary_key=True)
ref = Column(String(16), unique=True)
meta_json = Column(String)
full_json = Column(String)
title = Column(String)
class Reflist(Base):
__tablename__ = 'reflist'
rid = Column(Integer, primary_key=True)
ref = Column(String(16), unique=True)
sid = Column(String, ForeignKey('stories.sid'), unique=True)
source = Column(String(16))
def init():
Base.metadata.create_all(engine)
def get_story(sid):
session = Session()
return session.query(Story).get(sid)
def put_story(story):
story = story.copy()
full_json = json.dumps(story)
story.pop('text', None)
story.pop('comments', None)
meta_json = json.dumps(story)
try:
session = Session()
s = Story(
sid=story['id'],
ref=story['ref'],
full_json=full_json,
meta_json=meta_json,
title=story.get('title', None),
)
session.merge(s)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def get_story_by_ref(ref):
session = Session()
return session.query(Story).filter(Story.ref==ref).first()
def get_reflist(amount):
session = Session()
q = session.query(Reflist).order_by(Reflist.rid.desc()).limit(amount)
return [dict(ref=x.ref, sid=x.sid, source=x.source) for x in q.all()]
def get_stories(amount):
session = Session()
q = session.query(Reflist, Story.meta_json).\
order_by(Reflist.rid.desc()).\
join(Story).\
filter(Story.title != None).\
limit(amount)
return [x[1] for x in q]
def put_ref(ref, sid, source):
try:
session = Session()
r = Reflist(ref=ref, sid=sid, source=source)
session.add(r)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def del_ref(ref):
try:
session = Session()
session.query(Reflist).filter(Reflist.ref==ref).delete()
session.commit()
except:
session.rollback()
raise
finally:
session.close()
if __name__ == '__main__':
init()
print(get_story_by_ref('hgi3sy'))

View File

@ -9,10 +9,11 @@ from bs4 import BeautifulSoup
from feeds import hackernews, reddit, tildes, manual from feeds import hackernews, reddit, tildes, manual
OUTLINE_API = 'https://api.outline.com/v3/parse_article' OUTLINE_API = 'https://outlineapi.com/article'
ARCHIVE_API = 'https://archive.fo/submit/' ARCHIVE_API = 'https://archive.fo/submit/'
READ_API = 'http://127.0.0.1:33843' READ_API = 'http://127.0.0.1:33843'
INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif']
INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com'] INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com']
TWO_DAYS = 60*60*24*2 TWO_DAYS = 60*60*24*2
@ -56,18 +57,18 @@ def get_article(url):
logging.error('Problem getting article: {}'.format(str(e))) logging.error('Problem getting article: {}'.format(str(e)))
return '' return ''
def get_content_type(url): def get_first_image(text):
try: soup = BeautifulSoup(text, features='html.parser')
headers = {'User-Agent': 'Twitterbot/1.0'}
return requests.get(url, headers=headers, timeout=2).headers['content-type']
except:
pass
try: try:
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0'} first_img = soup.find('img')
return requests.get(url, headers=headers, timeout=2).headers['content-type'] url = first_img['src']
headers = {'User-Agent': 'Twitterbot/1.0'}
length = requests.get(url, headers=headers).headers['Content-length']
if int(length) > 1000000: raise
return url
except: except:
return 'text/' return ''
def update_story(story, is_manual=False): def update_story(story, is_manual=False):
res = {} res = {}
@ -86,29 +87,28 @@ def update_story(story, is_manual=False):
if res: if res:
story.update(res) # join dicts story.update(res) # join dicts
else: else:
logging.info('Story not ready yet') logging.info('Article not ready yet')
return False return False
if story['date'] and not is_manual and story['date'] + TWO_DAYS < time.time(): if story['date'] and not is_manual and story['date'] + TWO_DAYS < time.time():
logging.info('Story too old, removing') logging.info('Article too old, removing')
return False return False
if story.get('url', '') and not story.get('text', ''): if story.get('url', '') and not story.get('text', ''):
logging.info('inside if') if any([story['url'].endswith(ext) for ext in INVALID_FILES]):
if not get_content_type(story['url']).startswith('text/'): logging.info('URL invalid file type')
logging.info('URL invalid file type / content type:')
logging.info(story['url'])
return False return False
if any([domain in story['url'] for domain in INVALID_DOMAINS]): if any([domain in story['url'] for domain in INVALID_DOMAINS]):
logging.info('URL invalid domain:') logging.info('URL invalid domain')
logging.info(story['url'])
return False return False
logging.info('Getting article ' + story['url']) logging.info('Getting article ' + story['url'])
story['text'] = get_article(story['url']) story['text'] = get_article(story['url'])
if not story['text']: return False if not story['text']: return False
story['img'] = get_first_image(story['text'])
return True return True
if __name__ == '__main__': if __name__ == '__main__':
@ -124,5 +124,7 @@ if __name__ == '__main__':
a = get_article('https://blog.joinmastodon.org/2019/10/mastodon-3.0/') a = get_article('https://blog.joinmastodon.org/2019/10/mastodon-3.0/')
print(a) print(a)
u = get_first_image(a)
print(u)
print('done') print('done')

View File

@ -30,7 +30,7 @@ def api(route, ref=None):
return False return False
def feed(): def feed():
return [str(x) for x in api(API_TOPSTORIES) or []] return api(API_TOPSTORIES) or []
def comment(i): def comment(i):
if 'author' not in i: if 'author' not in i:
@ -77,6 +77,6 @@ def story(ref):
# scratchpad so I can quickly develop the parser # scratchpad so I can quickly develop the parser
if __name__ == '__main__': if __name__ == '__main__':
print(feed()) #print(feed())
#print(story(20763961)) #print(story(20763961))
#print(story(20802050)) print(story(20802050))

View File

@ -33,7 +33,7 @@ def story(ref):
s['author_link'] = 'https://news.t0.vc' s['author_link'] = 'https://news.t0.vc'
s['score'] = 0 s['score'] = 0
s['date'] = int(time.time()) s['date'] = int(time.time())
s['title'] = str(soup.title.string) if soup.title else ref s['title'] = str(soup.title.string)
s['link'] = ref s['link'] = ref
s['url'] = ref s['url'] = ref
s['comments'] = [] s['comments'] = []

View File

@ -14,9 +14,9 @@ from prawcore.exceptions import PrawcoreException
from utils import render_md, clean from utils import render_md, clean
SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+privacy' SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+technology+privacy'
SITE_LINK = lambda x : 'https://old.reddit.com{}'.format(x) SITE_LINK = lambda x : 'https://old.reddit.com/{}'.format(x)
SITE_AUTHOR_LINK = lambda x : 'https://old.reddit.com/u/{}'.format(x) SITE_AUTHOR_LINK = lambda x : 'https://old.reddit.com/u/{}'.format(x)
reddit = praw.Reddit('bot') reddit = praw.Reddit('bot')

View File

@ -91,7 +91,7 @@ def story(ref):
s['score'] = int(h.find('span', class_='topic-voting-votes').string) s['score'] = int(h.find('span', class_='topic-voting-votes').string)
s['date'] = unix(h.find('time')['datetime']) s['date'] = unix(h.find('time')['datetime'])
s['title'] = str(h.h1.string) s['title'] = str(h.h1.string)
s['group'] = str(soup.find('div', class_='site-header-context').a.string) s['group'] = str(soup.find('a', class_='site-header-context').string)
group_lookup[ref] = s['group'] group_lookup[ref] = s['group']
s['link'] = SITE_LINK(s['group'], ref) s['link'] = SITE_LINK(s['group'], ref)
ud = a.find('div', class_='topic-full-link') ud = a.find('div', class_='topic-full-link')
@ -122,7 +122,7 @@ if __name__ == '__main__':
#print(self_post) #print(self_post)
#li_comment = story('gqx') #li_comment = story('gqx')
#print(li_comment) #print(li_comment)
broken = story('q4y') broken = story('l11')
print(broken) print(broken)
# make sure there's no self-reference # make sure there's no self-reference

View File

@ -1,74 +0,0 @@
import archive
import database
import search
import json
import requests
database.init()
archive.init()
search.init()
count = 0
def database_del_story_by_ref(ref):
try:
session = database.Session()
session.query(database.Story).filter(database.Story.ref==ref).delete()
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def search_del_story(sid):
try:
r = requests.delete(search.MEILI_URL + 'indexes/qotnews/documents/'+sid, timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem deleting MeiliSearch story: {}'.format(str(e)))
return False
with archive.ix.searcher() as searcher:
print('count all', searcher.doc_count_all())
print('count', searcher.doc_count())
for doc in searcher.documents():
try:
print('num', count, 'id', doc['id'])
count += 1
story = doc['story']
story.pop('img', None)
if 'reddit.com/r/technology' in story['link']:
print('skipping r/technology')
continue
try:
database.put_story(story)
except database.IntegrityError:
print('collision!')
old_story = database.get_story_by_ref(story['ref'])
old_story = json.loads(old_story.full_json)
if story['num_comments'] > old_story['num_comments']:
print('more comments, replacing')
database_del_story_by_ref(story['ref'])
database.put_story(story)
search_del_story(old_story['id'])
else:
print('fewer comments, skipping')
continue
search.put_story(story)
print()
except KeyboardInterrupt:
break
except BaseException as e:
print('skipping', doc['id'])
print('reason:', e)

View File

@ -1,85 +0,0 @@
import logging
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.DEBUG)
import requests
MEILI_URL = 'http://127.0.0.1:7700/'
def create_index():
try:
json = dict(name='qotnews', uid='qotnews')
r = requests.post(MEILI_URL + 'indexes', json=json, timeout=2)
if r.status_code != 201:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem creating MeiliSearch index: {}'.format(str(e)))
return False
def update_rankings():
try:
json = ['typo', 'words', 'proximity', 'attribute', 'desc(date)', 'wordsPosition', 'exactness']
r = requests.post(MEILI_URL + 'indexes/qotnews/settings/ranking-rules', json=json, timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem setting MeiliSearch ranking rules: {}'.format(str(e)))
return False
def update_attributes():
try:
json = ['title', 'url', 'author', 'link', 'id']
r = requests.post(MEILI_URL + 'indexes/qotnews/settings/searchable-attributes', json=json, timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem setting MeiliSearch searchable attributes: {}'.format(str(e)))
return False
def init():
create_index()
update_rankings()
update_attributes()
def put_story(story):
story = story.copy()
story.pop('text', None)
story.pop('comments', None)
try:
r = requests.post(MEILI_URL + 'indexes/qotnews/documents', json=[story], timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem putting MeiliSearch story: {}'.format(str(e)))
return False
def search(q):
try:
params = dict(q=q, limit=250)
r = requests.get(MEILI_URL + 'indexes/qotnews/search', params=params, timeout=2)
if r.status_code != 200:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()['hits']
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem searching MeiliSearch: {}'.format(str(e)))
return False
if __name__ == '__main__':
create_index()
print(search('the'))

View File

@ -4,14 +4,12 @@ logging.basicConfig(
level=logging.INFO) level=logging.INFO)
import copy import copy
import json
import threading import threading
import traceback
import time import time
import shelve
from urllib.parse import urlparse, parse_qs from urllib.parse import urlparse, parse_qs
import database import archive
import search
import feed import feed
from utils import gen_rand_id from utils import gen_rand_id
@ -25,38 +23,75 @@ from gevent.pywsgi import WSGIServer
monkey.patch_all() monkey.patch_all()
database.init() archive.init()
search.init()
CACHE_LENGTH = 150
DATA_FILE = 'data/data'
FEED_LENGTH = 75
news_index = 0 news_index = 0
with shelve.open(DATA_FILE) as db:
logging.info('Reading caches from disk...')
news_list = db.get('news_list', [])
news_ref_to_id = db.get('news_ref_to_id', {})
news_cache = db.get('news_cache', {})
# clean cache if broken
try:
for ref in news_list:
nid = news_ref_to_id[ref]
_ = news_cache[nid]
except KeyError as e:
logging.error('Unable to find key {}. Trying to remove...'.format(str(e)))
news_list.remove(str(e))
def get_story(sid):
if sid in news_cache:
return news_cache[sid]
else:
return archive.get_story(sid)
def new_id(): def new_id():
nid = gen_rand_id() nid = gen_rand_id()
while database.get_story(nid): while nid in news_cache or archive.get_story(nid):
nid = gen_rand_id() nid = gen_rand_id()
return nid return nid
def remove_ref(old_ref):
while old_ref in news_list:
news_list.remove(old_ref)
old_story = news_cache.pop(news_ref_to_id[old_ref])
old_id = news_ref_to_id.pop(old_ref)
logging.info('Removed ref {} id {}.'.format(old_ref, old_id))
build_folder = '../webclient/build' build_folder = '../webclient/build'
flask_app = Flask(__name__, template_folder=build_folder, static_folder=build_folder, static_url_path='') flask_app = Flask(__name__, template_folder=build_folder, static_folder=build_folder, static_url_path='')
cors = CORS(flask_app) cors = CORS(flask_app)
@flask_app.route('/api') @flask_app.route('/api')
def api(): def api():
stories = database.get_stories(FEED_LENGTH) try:
# hacky nested json front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list]
res = Response('{"stories":[' + ','.join(stories) + ']}') except KeyError as e:
res.headers['content-type'] = 'application/json' logging.error('Unable to find key {}. Trying to remove...'.format(str(e)))
return res news_list.remove(str(e))
front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']]
front_page = front_page[:60]
for story in front_page:
story.pop('text', None)
story.pop('comments', None)
return {'stories': front_page}
@flask_app.route('/api/search', strict_slashes=False) @flask_app.route('/api/search', strict_slashes=False)
def apisearch(): def search():
q = request.args.get('q', '') search = request.args.get('q', '')
if len(q) >= 3: if len(search) >= 3:
results = search.search(q) res = archive.search(search)
else: else:
results = [] res = []
return dict(results=results) return {'results': res}
@flask_app.route('/api/submit', methods=['POST'], strict_slashes=False) @flask_app.route('/api/submit', methods=['POST'], strict_slashes=False)
def submit(): def submit():
@ -74,41 +109,29 @@ def submit():
elif 'reddit.com' in parse.hostname and 'comments' in url: elif 'reddit.com' in parse.hostname and 'comments' in url:
source = 'reddit' source = 'reddit'
ref = parse.path.split('/')[4] ref = parse.path.split('/')[4]
elif 'news.t0.vc' in parse.hostname:
raise Exception('Invalid article')
else: else:
source = 'manual' source = 'manual'
ref = url ref = url
existing = database.get_story_by_ref(ref) news_story = dict(id=nid, ref=ref, source=source)
if existing: news_cache[nid] = news_story
return {'nid': existing.sid} valid = feed.update_story(news_story, is_manual=True)
else:
story = dict(id=nid, ref=ref, source=source)
valid = feed.update_story(story, is_manual=True)
if valid: if valid:
database.put_story(story) archive.update(news_story)
search.put_story(story)
return {'nid': nid} return {'nid': nid}
else: else:
news_cache.pop(nid, '')
raise Exception('Invalid article') raise Exception('Invalid article')
except BaseException as e: except BaseException as e:
logging.error('Problem with article submission: {} - {}'.format(e.__class__.__name__, str(e))) logging.error('Problem with article submission: {} - {}'.format(e.__class__.__name__, str(e)))
print(traceback.format_exc())
abort(400) abort(400)
@flask_app.route('/api/<sid>') @flask_app.route('/api/<sid>')
def story(sid): def story(sid):
story = database.get_story(sid) story = get_story(sid)
if story: return dict(story=story) if story else abort(404)
# hacky nested json
res = Response('{"story":' + story.full_json + '}')
res.headers['content-type'] = 'application/json'
return res
else:
return abort(404)
@flask_app.route('/') @flask_app.route('/')
@flask_app.route('/search') @flask_app.route('/search')
@ -126,9 +149,8 @@ def static_story(sid):
except NotFound: except NotFound:
pass pass
story = database.get_story(sid) story = get_story(sid)
if not story: return abort(404) if not story: return abort(404)
story = json.loads(story.full_json)
score = story['score'] score = story['score']
num_comments = story['num_comments'] num_comments = story['num_comments']
@ -145,7 +167,7 @@ def static_story(sid):
url=url, url=url,
description=description) description=description)
http_server = WSGIServer(('', 43842), flask_app) http_server = WSGIServer(('', 33842), flask_app)
def feed_thread(): def feed_thread():
global news_index global news_index
@ -154,42 +176,39 @@ def feed_thread():
while True: while True:
# onboard new stories # onboard new stories
if news_index == 0: if news_index == 0:
for ref, source in feed.list(): feed_list = feed.list()
if database.get_story_by_ref(ref): new_items = [(ref, source) for ref, source in feed_list if ref not in news_list]
continue for ref, source in new_items:
try: news_list.insert(0, ref)
nid = new_id() nid = new_id()
database.put_ref(ref, nid, source) news_ref_to_id[ref] = nid
logging.info('Added ref ' + ref) news_cache[nid] = dict(id=nid, ref=ref, source=source)
except database.IntegrityError:
continue
ref_list = database.get_reflist(FEED_LENGTH) if len(new_items):
logging.info('Added {} new refs.'.format(len(new_items)))
# drop old ones
while len(news_list) > CACHE_LENGTH:
old_ref = news_list[-1]
remove_ref(old_ref)
# update current stories # update current stories
if news_index < len(ref_list): if news_index < len(news_list):
item = ref_list[news_index] update_ref = news_list[news_index]
update_id = news_ref_to_id[update_ref]
try: news_story = news_cache[update_id]
story_json = database.get_story(item['sid']).full_json valid = feed.update_story(news_story)
story = json.loads(story_json)
except AttributeError:
story = dict(id=item['sid'], ref=item['ref'], source=item['source'])
valid = feed.update_story(story)
if valid: if valid:
database.put_story(story) archive.update(news_story)
search.put_story(story)
else: else:
database.del_ref(item['ref']) remove_ref(update_ref)
logging.info('Removed ref {}'.format(item['ref']))
else: else:
logging.info('Skipping index') logging.info('Skipping update - no story #' + str(news_index+1))
gevent.sleep(6) gevent.sleep(6)
news_index += 1 news_index += 1
if news_index == FEED_LENGTH: news_index = 0 if news_index == CACHE_LENGTH: news_index = 0
except KeyboardInterrupt: except KeyboardInterrupt:
logging.info('Ending feed thread...') logging.info('Ending feed thread...')
@ -205,3 +224,9 @@ try:
http_server.serve_forever() http_server.serve_forever()
except KeyboardInterrupt: except KeyboardInterrupt:
logging.info('Exiting...') logging.info('Exiting...')
finally:
with shelve.open(DATA_FILE) as db:
logging.info('Writing caches to disk...')
db['news_list'] = news_list
db['news_ref_to_id'] = news_ref_to_id
db['news_cache'] = news_cache

View File

@ -36,6 +36,12 @@ class Feed extends React.Component {
this.props.updateCache(x.id, result.story); this.props.updateCache(x.id, result.story);
}, error => {} }, error => {}
); );
if (i < 20 && x.img) {
const img = new Image();
img.src = x.img;
console.log('prefetched image', x.img);
}
}); });
} }
}, },
@ -59,6 +65,10 @@ class Feed extends React.Component {
<div> <div>
{stories.map((x, i) => {stories.map((x, i) =>
<div className='item' key={i}> <div className='item' key={i}>
<div className='num'>
{i+1}.
</div>
<div className='title'> <div className='title'>
<Link className='link' to={'/' + x.id}> <Link className='link' to={'/' + x.id}>
<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title} <img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}

View File

@ -64,6 +64,10 @@ class Results extends React.Component {
{stories.length ? {stories.length ?
stories.map((x, i) => stories.map((x, i) =>
<div className='item' key={i}> <div className='item' key={i}>
<div className='num'>
{i+1}.
</div>
<div className='title'> <div className='title'>
<Link className='link' to={'/' + x.id}> <Link className='link' to={'/' + x.id}>
<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title} <img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}

View File

@ -37,7 +37,7 @@ class Search extends Component {
<span className='search'> <span className='search'>
<form onSubmit={this.searchAgain}> <form onSubmit={this.searchAgain}>
<input <input
placeholder='Search... (fixed)' placeholder='Search...'
value={search} value={search}
onChange={this.searchArticles} onChange={this.searchArticles}
ref={this.inputRef} ref={this.inputRef}

View File

@ -46,7 +46,7 @@ pre {
.item { .item {
display: table; display: table;
color: #828282; color: #828282;
margin-bottom: 0.7rem; margin-bottom: 0.6rem;
} }
.item .source-logo { .item .source-logo {
@ -61,6 +61,11 @@ pre {
text-decoration: underline; text-decoration: underline;
} }
.item .num {
display: table-cell;
width: 2em;
}
.item a.link { .item a.link {
font-size: 1.1rem; font-size: 1.1rem;
color: #000000; color: #000000;