Compare commits

..

19 Commits

Author SHA1 Message Date
9a449bf3ca Remove extra logging 2020-07-08 02:36:40 +00:00
0bd9f05250 Fix crash when HN feed fails 2020-07-08 02:36:40 +00:00
9c116bde4a Remove document img and ignore r/technology 2020-07-08 02:36:40 +00:00
ebedaef00b Tune search rankings and attributes 2020-07-08 02:36:40 +00:00
d7f0643bd7 Add more logging 2020-07-08 02:36:40 +00:00
eb1137299d Remove article numbers 2020-07-08 02:36:40 +00:00
72d4a68929 Remove pre-fetching image 2020-07-08 02:36:40 +00:00
f1c846acd0 Remove get first image 2020-07-08 02:36:40 +00:00
850b30e353 Add requests timeouts and temporary logging 2020-07-08 02:36:40 +00:00
d614ad0743 Integrate with external MeiliSearch server 2020-07-08 02:36:40 +00:00
f46cafdc90 Integrate sqlite database with server 2020-07-08 02:36:40 +00:00
873dc44cb1 Update whoosh migration script 2020-07-08 02:36:40 +00:00
1fb9db3f4b Store ref list in database too 2020-07-08 02:36:40 +00:00
b923908a45 Begin initial sqlite conversion 2020-07-08 02:36:40 +00:00
dbdcfaa921 Check if cache is broken 2020-07-08 02:36:40 +00:00
8799b10525 Fall back to ref on manual submission title 2020-07-08 02:36:40 +00:00
6430fe5e9f Check content-type 2020-07-08 02:36:40 +00:00
a4cf719cb8 Remove technology subreddit 2020-07-08 02:36:40 +00:00
595f469b4a Update tildes parser group tag 2020-07-08 02:36:40 +00:00
14 changed files with 364 additions and 144 deletions

View File

@ -109,3 +109,4 @@ praw.ini
data.db
data.db.bak
data/archive/*
qotnews.sqlite

106
apiserver/database.py Normal file
View File

@ -0,0 +1,106 @@
import json
from sqlalchemy import create_engine, Column, String, ForeignKey, Integer
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError
engine = create_engine('sqlite:///data/qotnews.sqlite')
Session = sessionmaker(bind=engine)
Base = declarative_base()
class Story(Base):
__tablename__ = 'stories'
sid = Column(String(16), primary_key=True)
ref = Column(String(16), unique=True)
meta_json = Column(String)
full_json = Column(String)
title = Column(String)
class Reflist(Base):
__tablename__ = 'reflist'
rid = Column(Integer, primary_key=True)
ref = Column(String(16), unique=True)
sid = Column(String, ForeignKey('stories.sid'), unique=True)
source = Column(String(16))
def init():
Base.metadata.create_all(engine)
def get_story(sid):
session = Session()
return session.query(Story).get(sid)
def put_story(story):
story = story.copy()
full_json = json.dumps(story)
story.pop('text', None)
story.pop('comments', None)
meta_json = json.dumps(story)
try:
session = Session()
s = Story(
sid=story['id'],
ref=story['ref'],
full_json=full_json,
meta_json=meta_json,
title=story.get('title', None),
)
session.merge(s)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def get_story_by_ref(ref):
session = Session()
return session.query(Story).filter(Story.ref==ref).first()
def get_reflist(amount):
session = Session()
q = session.query(Reflist).order_by(Reflist.rid.desc()).limit(amount)
return [dict(ref=x.ref, sid=x.sid, source=x.source) for x in q.all()]
def get_stories(amount):
session = Session()
q = session.query(Reflist, Story.meta_json).\
order_by(Reflist.rid.desc()).\
join(Story).\
filter(Story.title != None).\
limit(amount)
return [x[1] for x in q]
def put_ref(ref, sid, source):
try:
session = Session()
r = Reflist(ref=ref, sid=sid, source=source)
session.add(r)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def del_ref(ref):
try:
session = Session()
session.query(Reflist).filter(Reflist.ref==ref).delete()
session.commit()
except:
session.rollback()
raise
finally:
session.close()
if __name__ == '__main__':
init()
print(get_story_by_ref('hgi3sy'))

View File

@ -9,11 +9,10 @@ from bs4 import BeautifulSoup
from feeds import hackernews, reddit, tildes, manual
OUTLINE_API = 'https://outlineapi.com/article'
OUTLINE_API = 'https://api.outline.com/v3/parse_article'
ARCHIVE_API = 'https://archive.fo/submit/'
READ_API = 'http://127.0.0.1:33843'
INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif']
INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com']
TWO_DAYS = 60*60*24*2
@ -57,18 +56,18 @@ def get_article(url):
logging.error('Problem getting article: {}'.format(str(e)))
return ''
def get_first_image(text):
soup = BeautifulSoup(text, features='html.parser')
def get_content_type(url):
try:
headers = {'User-Agent': 'Twitterbot/1.0'}
return requests.get(url, headers=headers, timeout=2).headers['content-type']
except:
pass
try:
first_img = soup.find('img')
url = first_img['src']
headers = {'User-Agent': 'Twitterbot/1.0'}
length = requests.get(url, headers=headers).headers['Content-length']
if int(length) > 1000000: raise
return url
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0'}
return requests.get(url, headers=headers, timeout=2).headers['content-type']
except:
return ''
return 'text/'
def update_story(story, is_manual=False):
res = {}
@ -87,28 +86,29 @@ def update_story(story, is_manual=False):
if res:
story.update(res) # join dicts
else:
logging.info('Article not ready yet')
logging.info('Story not ready yet')
return False
if story['date'] and not is_manual and story['date'] + TWO_DAYS < time.time():
logging.info('Article too old, removing')
logging.info('Story too old, removing')
return False
if story.get('url', '') and not story.get('text', ''):
if any([story['url'].endswith(ext) for ext in INVALID_FILES]):
logging.info('URL invalid file type')
logging.info('inside if')
if not get_content_type(story['url']).startswith('text/'):
logging.info('URL invalid file type / content type:')
logging.info(story['url'])
return False
if any([domain in story['url'] for domain in INVALID_DOMAINS]):
logging.info('URL invalid domain')
logging.info('URL invalid domain:')
logging.info(story['url'])
return False
logging.info('Getting article ' + story['url'])
story['text'] = get_article(story['url'])
if not story['text']: return False
story['img'] = get_first_image(story['text'])
return True
if __name__ == '__main__':
@ -124,7 +124,5 @@ if __name__ == '__main__':
a = get_article('https://blog.joinmastodon.org/2019/10/mastodon-3.0/')
print(a)
u = get_first_image(a)
print(u)
print('done')

View File

@ -30,7 +30,7 @@ def api(route, ref=None):
return False
def feed():
return api(API_TOPSTORIES) or []
return [str(x) for x in api(API_TOPSTORIES) or []]
def comment(i):
if 'author' not in i:
@ -77,6 +77,6 @@ def story(ref):
# scratchpad so I can quickly develop the parser
if __name__ == '__main__':
#print(feed())
print(feed())
#print(story(20763961))
print(story(20802050))
#print(story(20802050))

View File

@ -33,7 +33,7 @@ def story(ref):
s['author_link'] = 'https://news.t0.vc'
s['score'] = 0
s['date'] = int(time.time())
s['title'] = str(soup.title.string)
s['title'] = str(soup.title.string) if soup.title else ref
s['link'] = ref
s['url'] = ref
s['comments'] = []

View File

@ -14,9 +14,9 @@ from prawcore.exceptions import PrawcoreException
from utils import render_md, clean
SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+technology+privacy'
SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+privacy'
SITE_LINK = lambda x : 'https://old.reddit.com/{}'.format(x)
SITE_LINK = lambda x : 'https://old.reddit.com{}'.format(x)
SITE_AUTHOR_LINK = lambda x : 'https://old.reddit.com/u/{}'.format(x)
reddit = praw.Reddit('bot')

View File

@ -91,7 +91,7 @@ def story(ref):
s['score'] = int(h.find('span', class_='topic-voting-votes').string)
s['date'] = unix(h.find('time')['datetime'])
s['title'] = str(h.h1.string)
s['group'] = str(soup.find('a', class_='site-header-context').string)
s['group'] = str(soup.find('div', class_='site-header-context').a.string)
group_lookup[ref] = s['group']
s['link'] = SITE_LINK(s['group'], ref)
ud = a.find('div', class_='topic-full-link')
@ -122,7 +122,7 @@ if __name__ == '__main__':
#print(self_post)
#li_comment = story('gqx')
#print(li_comment)
broken = story('l11')
broken = story('q4y')
print(broken)
# make sure there's no self-reference

View File

@ -0,0 +1,74 @@
import archive
import database
import search
import json
import requests
database.init()
archive.init()
search.init()
count = 0
def database_del_story_by_ref(ref):
try:
session = database.Session()
session.query(database.Story).filter(database.Story.ref==ref).delete()
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def search_del_story(sid):
try:
r = requests.delete(search.MEILI_URL + 'indexes/qotnews/documents/'+sid, timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem deleting MeiliSearch story: {}'.format(str(e)))
return False
with archive.ix.searcher() as searcher:
print('count all', searcher.doc_count_all())
print('count', searcher.doc_count())
for doc in searcher.documents():
try:
print('num', count, 'id', doc['id'])
count += 1
story = doc['story']
story.pop('img', None)
if 'reddit.com/r/technology' in story['link']:
print('skipping r/technology')
continue
try:
database.put_story(story)
except database.IntegrityError:
print('collision!')
old_story = database.get_story_by_ref(story['ref'])
old_story = json.loads(old_story.full_json)
if story['num_comments'] > old_story['num_comments']:
print('more comments, replacing')
database_del_story_by_ref(story['ref'])
database.put_story(story)
search_del_story(old_story['id'])
else:
print('fewer comments, skipping')
continue
search.put_story(story)
print()
except KeyboardInterrupt:
break
except BaseException as e:
print('skipping', doc['id'])
print('reason:', e)

85
apiserver/search.py Normal file
View File

@ -0,0 +1,85 @@
import logging
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.DEBUG)
import requests
MEILI_URL = 'http://127.0.0.1:7700/'
def create_index():
try:
json = dict(name='qotnews', uid='qotnews')
r = requests.post(MEILI_URL + 'indexes', json=json, timeout=2)
if r.status_code != 201:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem creating MeiliSearch index: {}'.format(str(e)))
return False
def update_rankings():
try:
json = ['typo', 'words', 'proximity', 'attribute', 'desc(date)', 'wordsPosition', 'exactness']
r = requests.post(MEILI_URL + 'indexes/qotnews/settings/ranking-rules', json=json, timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem setting MeiliSearch ranking rules: {}'.format(str(e)))
return False
def update_attributes():
try:
json = ['title', 'url', 'author', 'link', 'id']
r = requests.post(MEILI_URL + 'indexes/qotnews/settings/searchable-attributes', json=json, timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem setting MeiliSearch searchable attributes: {}'.format(str(e)))
return False
def init():
create_index()
update_rankings()
update_attributes()
def put_story(story):
story = story.copy()
story.pop('text', None)
story.pop('comments', None)
try:
r = requests.post(MEILI_URL + 'indexes/qotnews/documents', json=[story], timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem putting MeiliSearch story: {}'.format(str(e)))
return False
def search(q):
try:
params = dict(q=q, limit=250)
r = requests.get(MEILI_URL + 'indexes/qotnews/search', params=params, timeout=2)
if r.status_code != 200:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()['hits']
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem searching MeiliSearch: {}'.format(str(e)))
return False
if __name__ == '__main__':
create_index()
print(search('the'))

View File

@ -4,12 +4,14 @@ logging.basicConfig(
level=logging.INFO)
import copy
import json
import threading
import traceback
import time
import shelve
from urllib.parse import urlparse, parse_qs
import archive
import database
import search
import feed
from utils import gen_rand_id
@ -23,75 +25,38 @@ from gevent.pywsgi import WSGIServer
monkey.patch_all()
archive.init()
CACHE_LENGTH = 150
DATA_FILE = 'data/data'
database.init()
search.init()
FEED_LENGTH = 75
news_index = 0
with shelve.open(DATA_FILE) as db:
logging.info('Reading caches from disk...')
news_list = db.get('news_list', [])
news_ref_to_id = db.get('news_ref_to_id', {})
news_cache = db.get('news_cache', {})
# clean cache if broken
try:
for ref in news_list:
nid = news_ref_to_id[ref]
_ = news_cache[nid]
except KeyError as e:
logging.error('Unable to find key {}. Trying to remove...'.format(str(e)))
news_list.remove(str(e))
def get_story(sid):
if sid in news_cache:
return news_cache[sid]
else:
return archive.get_story(sid)
def new_id():
nid = gen_rand_id()
while nid in news_cache or archive.get_story(nid):
while database.get_story(nid):
nid = gen_rand_id()
return nid
def remove_ref(old_ref):
while old_ref in news_list:
news_list.remove(old_ref)
old_story = news_cache.pop(news_ref_to_id[old_ref])
old_id = news_ref_to_id.pop(old_ref)
logging.info('Removed ref {} id {}.'.format(old_ref, old_id))
build_folder = '../webclient/build'
flask_app = Flask(__name__, template_folder=build_folder, static_folder=build_folder, static_url_path='')
cors = CORS(flask_app)
@flask_app.route('/api')
def api():
try:
front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list]
except KeyError as e:
logging.error('Unable to find key {}. Trying to remove...'.format(str(e)))
news_list.remove(str(e))
front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']]
front_page = front_page[:60]
for story in front_page:
story.pop('text', None)
story.pop('comments', None)
return {'stories': front_page}
stories = database.get_stories(FEED_LENGTH)
# hacky nested json
res = Response('{"stories":[' + ','.join(stories) + ']}')
res.headers['content-type'] = 'application/json'
return res
@flask_app.route('/api/search', strict_slashes=False)
def search():
search = request.args.get('q', '')
if len(search) >= 3:
res = archive.search(search)
def apisearch():
q = request.args.get('q', '')
if len(q) >= 3:
results = search.search(q)
else:
res = []
return {'results': res}
results = []
return dict(results=results)
@flask_app.route('/api/submit', methods=['POST'], strict_slashes=False)
def submit():
@ -109,29 +74,41 @@ def submit():
elif 'reddit.com' in parse.hostname and 'comments' in url:
source = 'reddit'
ref = parse.path.split('/')[4]
elif 'news.t0.vc' in parse.hostname:
raise Exception('Invalid article')
else:
source = 'manual'
ref = url
news_story = dict(id=nid, ref=ref, source=source)
news_cache[nid] = news_story
valid = feed.update_story(news_story, is_manual=True)
if valid:
archive.update(news_story)
return {'nid': nid}
existing = database.get_story_by_ref(ref)
if existing:
return {'nid': existing.sid}
else:
news_cache.pop(nid, '')
raise Exception('Invalid article')
story = dict(id=nid, ref=ref, source=source)
valid = feed.update_story(story, is_manual=True)
if valid:
database.put_story(story)
search.put_story(story)
return {'nid': nid}
else:
raise Exception('Invalid article')
except BaseException as e:
logging.error('Problem with article submission: {} - {}'.format(e.__class__.__name__, str(e)))
print(traceback.format_exc())
abort(400)
@flask_app.route('/api/<sid>')
def story(sid):
story = get_story(sid)
return dict(story=story) if story else abort(404)
story = database.get_story(sid)
if story:
# hacky nested json
res = Response('{"story":' + story.full_json + '}')
res.headers['content-type'] = 'application/json'
return res
else:
return abort(404)
@flask_app.route('/')
@flask_app.route('/search')
@ -149,8 +126,9 @@ def static_story(sid):
except NotFound:
pass
story = get_story(sid)
story = database.get_story(sid)
if not story: return abort(404)
story = json.loads(story.full_json)
score = story['score']
num_comments = story['num_comments']
@ -167,7 +145,7 @@ def static_story(sid):
url=url,
description=description)
http_server = WSGIServer(('', 33842), flask_app)
http_server = WSGIServer(('', 43842), flask_app)
def feed_thread():
global news_index
@ -176,39 +154,42 @@ def feed_thread():
while True:
# onboard new stories
if news_index == 0:
feed_list = feed.list()
new_items = [(ref, source) for ref, source in feed_list if ref not in news_list]
for ref, source in new_items:
news_list.insert(0, ref)
nid = new_id()
news_ref_to_id[ref] = nid
news_cache[nid] = dict(id=nid, ref=ref, source=source)
for ref, source in feed.list():
if database.get_story_by_ref(ref):
continue
try:
nid = new_id()
database.put_ref(ref, nid, source)
logging.info('Added ref ' + ref)
except database.IntegrityError:
continue
if len(new_items):
logging.info('Added {} new refs.'.format(len(new_items)))
# drop old ones
while len(news_list) > CACHE_LENGTH:
old_ref = news_list[-1]
remove_ref(old_ref)
ref_list = database.get_reflist(FEED_LENGTH)
# update current stories
if news_index < len(news_list):
update_ref = news_list[news_index]
update_id = news_ref_to_id[update_ref]
news_story = news_cache[update_id]
valid = feed.update_story(news_story)
if news_index < len(ref_list):
item = ref_list[news_index]
try:
story_json = database.get_story(item['sid']).full_json
story = json.loads(story_json)
except AttributeError:
story = dict(id=item['sid'], ref=item['ref'], source=item['source'])
valid = feed.update_story(story)
if valid:
archive.update(news_story)
database.put_story(story)
search.put_story(story)
else:
remove_ref(update_ref)
database.del_ref(item['ref'])
logging.info('Removed ref {}'.format(item['ref']))
else:
logging.info('Skipping update - no story #' + str(news_index+1))
logging.info('Skipping index')
gevent.sleep(6)
news_index += 1
if news_index == CACHE_LENGTH: news_index = 0
if news_index == FEED_LENGTH: news_index = 0
except KeyboardInterrupt:
logging.info('Ending feed thread...')
@ -224,9 +205,3 @@ try:
http_server.serve_forever()
except KeyboardInterrupt:
logging.info('Exiting...')
finally:
with shelve.open(DATA_FILE) as db:
logging.info('Writing caches to disk...')
db['news_list'] = news_list
db['news_ref_to_id'] = news_ref_to_id
db['news_cache'] = news_cache

View File

@ -36,12 +36,6 @@ class Feed extends React.Component {
this.props.updateCache(x.id, result.story);
}, error => {}
);
if (i < 20 && x.img) {
const img = new Image();
img.src = x.img;
console.log('prefetched image', x.img);
}
});
}
},
@ -65,10 +59,6 @@ class Feed extends React.Component {
<div>
{stories.map((x, i) =>
<div className='item' key={i}>
<div className='num'>
{i+1}.
</div>
<div className='title'>
<Link className='link' to={'/' + x.id}>
<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}

View File

@ -64,10 +64,6 @@ class Results extends React.Component {
{stories.length ?
stories.map((x, i) =>
<div className='item' key={i}>
<div className='num'>
{i+1}.
</div>
<div className='title'>
<Link className='link' to={'/' + x.id}>
<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}

View File

@ -37,7 +37,7 @@ class Search extends Component {
<span className='search'>
<form onSubmit={this.searchAgain}>
<input
placeholder='Search...'
placeholder='Search... (fixed)'
value={search}
onChange={this.searchArticles}
ref={this.inputRef}

View File

@ -46,7 +46,7 @@ pre {
.item {
display: table;
color: #828282;
margin-bottom: 0.6rem;
margin-bottom: 0.7rem;
}
.item .source-logo {
@ -61,11 +61,6 @@ pre {
text-decoration: underline;
}
.item .num {
display: table-cell;
width: 2em;
}
.item a.link {
font-size: 1.1rem;
color: #000000;