tanner/qotnews
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: tanner:9a449bf3ca1f8a2864f6075b2c96d933294643d4
Branches Tags
tanner:master
..
compare: tanner:b252c6a207c00c25784249f67b157dbf4ba426ea
Branches Tags
tanner:master

No commits in common. "9a449bf3ca1f8a2864f6075b2c96d933294643d4" and "b252c6a207c00c25784249f67b157dbf4ba426ea" have entirely different histories.
9a449bf3ca ... b252c6a207

14 changed files with 144 additions and 364 deletions
Show Stats Expand all files Collapse all files

1
apiserver/.gitignore vendored
View File

@ -109,4 +109,3 @@ praw.ini
data.db
data.db.bak
data/archive/*
qotnews.sqlite

106
apiserver/database.py
View File

@ -1,106 +0,0 @@
import json
from sqlalchemy import create_engine, Column, String, ForeignKey, Integer
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError
engine = create_engine('sqlite:///data/qotnews.sqlite')
Session = sessionmaker(bind=engine)
Base = declarative_base()
class Story(Base):
__tablename__ = 'stories'
sid = Column(String(16), primary_key=True)
ref = Column(String(16), unique=True)
meta_json = Column(String)
full_json = Column(String)
title = Column(String)
class Reflist(Base):
__tablename__ = 'reflist'
rid = Column(Integer, primary_key=True)
ref = Column(String(16), unique=True)
sid = Column(String, ForeignKey('stories.sid'), unique=True)
source = Column(String(16))
def init():
Base.metadata.create_all(engine)
def get_story(sid):
session = Session()
return session.query(Story).get(sid)
def put_story(story):
story = story.copy()
full_json = json.dumps(story)
story.pop('text', None)
story.pop('comments', None)
meta_json = json.dumps(story)
try:
session = Session()
s = Story(
sid=story['id'],
ref=story['ref'],
full_json=full_json,
meta_json=meta_json,
title=story.get('title', None),
)
session.merge(s)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def get_story_by_ref(ref):
session = Session()
return session.query(Story).filter(Story.ref==ref).first()
def get_reflist(amount):
session = Session()
q = session.query(Reflist).order_by(Reflist.rid.desc()).limit(amount)
return [dict(ref=x.ref, sid=x.sid, source=x.source) for x in q.all()]
def get_stories(amount):
session = Session()
q = session.query(Reflist, Story.meta_json).\
order_by(Reflist.rid.desc()).\
join(Story).\
filter(Story.title != None).\
limit(amount)
return [x[1] for x in q]
def put_ref(ref, sid, source):
try:
session = Session()
r = Reflist(ref=ref, sid=sid, source=source)
session.add(r)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def del_ref(ref):
try:
session = Session()
session.query(Reflist).filter(Reflist.ref==ref).delete()
session.commit()
except:
session.rollback()
raise
finally:
session.close()
if __name__ == '__main__':
init()
print(get_story_by_ref('hgi3sy'))

38
apiserver/feed.py
View File

@ -9,10 +9,11 @@ from bs4 import BeautifulSoup
from feeds import hackernews, reddit, tildes, manual
OUTLINE_API = 'https://api.outline.com/v3/parse_article'
OUTLINE_API = 'https://outlineapi.com/article'
ARCHIVE_API = 'https://archive.fo/submit/'
READ_API = 'http://127.0.0.1:33843'
INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif']
INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com']
TWO_DAYS = 60*60*24*2
@ -56,18 +57,18 @@ def get_article(url):
logging.error('Problem getting article: {}'.format(str(e)))
return ''
def get_content_type(url):
try:
headers = {'User-Agent': 'Twitterbot/1.0'}
return requests.get(url, headers=headers, timeout=2).headers['content-type']
except:
pass
def get_first_image(text):
soup = BeautifulSoup(text, features='html.parser')
try:
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0'}
return requests.get(url, headers=headers, timeout=2).headers['content-type']
first_img = soup.find('img')
url = first_img['src']
headers = {'User-Agent': 'Twitterbot/1.0'}
length = requests.get(url, headers=headers).headers['Content-length']
if int(length) > 1000000: raise
return url
except:
return 'text/'
return ''
def update_story(story, is_manual=False):
res = {}
@ -86,29 +87,28 @@ def update_story(story, is_manual=False):
if res:
story.update(res) # join dicts
else:
logging.info('Story not ready yet')
logging.info('Article not ready yet')
return False
if story['date'] and not is_manual and story['date'] + TWO_DAYS < time.time():
logging.info('Story too old, removing')
logging.info('Article too old, removing')
return False
if story.get('url', '') and not story.get('text', ''):
logging.info('inside if')
if not get_content_type(story['url']).startswith('text/'):
logging.info('URL invalid file type / content type:')
logging.info(story['url'])
if any([story['url'].endswith(ext) for ext in INVALID_FILES]):
logging.info('URL invalid file type')
return False
if any([domain in story['url'] for domain in INVALID_DOMAINS]):
logging.info('URL invalid domain:')
logging.info(story['url'])
logging.info('URL invalid domain')
return False
logging.info('Getting article ' + story['url'])
story['text'] = get_article(story['url'])
if not story['text']: return False
story['img'] = get_first_image(story['text'])
return True
if __name__ == '__main__':
@ -124,5 +124,7 @@ if __name__ == '__main__':
a = get_article('https://blog.joinmastodon.org/2019/10/mastodon-3.0/')
print(a)
u = get_first_image(a)
print(u)
print('done')

6
apiserver/feeds/hackernews.py
View File

@ -30,7 +30,7 @@ def api(route, ref=None):
return False
def feed():
return [str(x) for x in api(API_TOPSTORIES) or []]
return api(API_TOPSTORIES) or []
def comment(i):
if 'author' not in i:
@ -77,6 +77,6 @@ def story(ref):
# scratchpad so I can quickly develop the parser
if __name__ == '__main__':
print(feed())
#print(feed())
#print(story(20763961))
#print(story(20802050))
print(story(20802050))

2
apiserver/feeds/manual.py
View File

@ -33,7 +33,7 @@ def story(ref):
s['author_link'] = 'https://news.t0.vc'
s['score'] = 0
s['date'] = int(time.time())
s['title'] = str(soup.title.string) if soup.title else ref
s['title'] = str(soup.title.string)
s['link'] = ref
s['url'] = ref
s['comments'] = []

4
apiserver/feeds/reddit.py
View File

@ -14,9 +14,9 @@ from prawcore.exceptions import PrawcoreException
from utils import render_md, clean
SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+privacy'
SUBREDDITS = 'Economics+Foodforthought+TrueReddit+business+technology+privacy'
SITE_LINK = lambda x : 'https://old.reddit.com{}'.format(x)
SITE_LINK = lambda x : 'https://old.reddit.com/{}'.format(x)
SITE_AUTHOR_LINK = lambda x : 'https://old.reddit.com/u/{}'.format(x)
reddit = praw.Reddit('bot')

4
apiserver/feeds/tildes.py
View File

@ -91,7 +91,7 @@ def story(ref):
s['score'] = int(h.find('span', class_='topic-voting-votes').string)
s['date'] = unix(h.find('time')['datetime'])
s['title'] = str(h.h1.string)
s['group'] = str(soup.find('div', class_='site-header-context').a.string)
s['group'] = str(soup.find('a', class_='site-header-context').string)
group_lookup[ref] = s['group']
s['link'] = SITE_LINK(s['group'], ref)
ud = a.find('div', class_='topic-full-link')
@ -122,7 +122,7 @@ if __name__ == '__main__':
#print(self_post)
#li_comment = story('gqx')
#print(li_comment)
broken = story('q4y')
broken = story('l11')
print(broken)
# make sure there's no self-reference

74
apiserver/migrate-whoosh-to-sqlite.py
View File

@ -1,74 +0,0 @@
import archive
import database
import search
import json
import requests
database.init()
archive.init()
search.init()
count = 0
def database_del_story_by_ref(ref):
try:
session = database.Session()
session.query(database.Story).filter(database.Story.ref==ref).delete()
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def search_del_story(sid):
try:
r = requests.delete(search.MEILI_URL + 'indexes/qotnews/documents/'+sid, timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem deleting MeiliSearch story: {}'.format(str(e)))
return False
with archive.ix.searcher() as searcher:
print('count all', searcher.doc_count_all())
print('count', searcher.doc_count())
for doc in searcher.documents():
try:
print('num', count, 'id', doc['id'])
count += 1
story = doc['story']
story.pop('img', None)
if 'reddit.com/r/technology' in story['link']:
print('skipping r/technology')
continue
try:
database.put_story(story)
except database.IntegrityError:
print('collision!')
old_story = database.get_story_by_ref(story['ref'])
old_story = json.loads(old_story.full_json)
if story['num_comments'] > old_story['num_comments']:
print('more comments, replacing')
database_del_story_by_ref(story['ref'])
database.put_story(story)
search_del_story(old_story['id'])
else:
print('fewer comments, skipping')
continue
search.put_story(story)
print()
except KeyboardInterrupt:
break
except BaseException as e:
print('skipping', doc['id'])
print('reason:', e)

85
apiserver/search.py
View File

@ -1,85 +0,0 @@
import logging
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.DEBUG)
import requests
MEILI_URL = 'http://127.0.0.1:7700/'
def create_index():
try:
json = dict(name='qotnews', uid='qotnews')
r = requests.post(MEILI_URL + 'indexes', json=json, timeout=2)
if r.status_code != 201:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem creating MeiliSearch index: {}'.format(str(e)))
return False
def update_rankings():
try:
json = ['typo', 'words', 'proximity', 'attribute', 'desc(date)', 'wordsPosition', 'exactness']
r = requests.post(MEILI_URL + 'indexes/qotnews/settings/ranking-rules', json=json, timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem setting MeiliSearch ranking rules: {}'.format(str(e)))
return False
def update_attributes():
try:
json = ['title', 'url', 'author', 'link', 'id']
r = requests.post(MEILI_URL + 'indexes/qotnews/settings/searchable-attributes', json=json, timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem setting MeiliSearch searchable attributes: {}'.format(str(e)))
return False
def init():
create_index()
update_rankings()
update_attributes()
def put_story(story):
story = story.copy()
story.pop('text', None)
story.pop('comments', None)
try:
r = requests.post(MEILI_URL + 'indexes/qotnews/documents', json=[story], timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem putting MeiliSearch story: {}'.format(str(e)))
return False
def search(q):
try:
params = dict(q=q, limit=250)
r = requests.get(MEILI_URL + 'indexes/qotnews/search', params=params, timeout=2)
if r.status_code != 200:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()['hits']
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem searching MeiliSearch: {}'.format(str(e)))
return False
if __name__ == '__main__':
create_index()
print(search('the'))

165
apiserver/server.py
View File

@ -4,14 +4,12 @@ logging.basicConfig(
level=logging.INFO)
import copy
import json
import threading
import traceback
import time
import shelve
from urllib.parse import urlparse, parse_qs
import database
import search
import archive
import feed
from utils import gen_rand_id
@ -25,38 +23,75 @@ from gevent.pywsgi import WSGIServer
monkey.patch_all()
database.init()
search.init()
archive.init()
CACHE_LENGTH = 150
DATA_FILE = 'data/data'
FEED_LENGTH = 75
news_index = 0
with shelve.open(DATA_FILE) as db:
logging.info('Reading caches from disk...')
news_list = db.get('news_list', [])
news_ref_to_id = db.get('news_ref_to_id', {})
news_cache = db.get('news_cache', {})
# clean cache if broken
try:
for ref in news_list:
nid = news_ref_to_id[ref]
_ = news_cache[nid]
except KeyError as e:
logging.error('Unable to find key {}. Trying to remove...'.format(str(e)))
news_list.remove(str(e))
def get_story(sid):
if sid in news_cache:
return news_cache[sid]
else:
return archive.get_story(sid)
def new_id():
nid = gen_rand_id()
while database.get_story(nid):
while nid in news_cache or archive.get_story(nid):
nid = gen_rand_id()
return nid
def remove_ref(old_ref):
while old_ref in news_list:
news_list.remove(old_ref)
old_story = news_cache.pop(news_ref_to_id[old_ref])
old_id = news_ref_to_id.pop(old_ref)
logging.info('Removed ref {} id {}.'.format(old_ref, old_id))
build_folder = '../webclient/build'
flask_app = Flask(__name__, template_folder=build_folder, static_folder=build_folder, static_url_path='')
cors = CORS(flask_app)
@flask_app.route('/api')
def api():
stories = database.get_stories(FEED_LENGTH)
# hacky nested json
res = Response('{"stories":[' + ','.join(stories) + ']}')
res.headers['content-type'] = 'application/json'
return res
try:
front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list]
except KeyError as e:
logging.error('Unable to find key {}. Trying to remove...'.format(str(e)))
news_list.remove(str(e))
front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']]
front_page = front_page[:60]
for story in front_page:
story.pop('text', None)
story.pop('comments', None)
return {'stories': front_page}
@flask_app.route('/api/search', strict_slashes=False)
def apisearch():
q = request.args.get('q', '')
if len(q) >= 3:
results = search.search(q)
def search():
search = request.args.get('q', '')
if len(search) >= 3:
res = archive.search(search)
else:
results = []
return dict(results=results)
res = []
return {'results': res}
@flask_app.route('/api/submit', methods=['POST'], strict_slashes=False)
def submit():
@ -74,41 +109,29 @@ def submit():
elif 'reddit.com' in parse.hostname and 'comments' in url:
source = 'reddit'
ref = parse.path.split('/')[4]
elif 'news.t0.vc' in parse.hostname:
raise Exception('Invalid article')
else:
source = 'manual'
ref = url
existing = database.get_story_by_ref(ref)
if existing:
return {'nid': existing.sid}
news_story = dict(id=nid, ref=ref, source=source)
news_cache[nid] = news_story
valid = feed.update_story(news_story, is_manual=True)
if valid:
archive.update(news_story)
return {'nid': nid}
else:
story = dict(id=nid, ref=ref, source=source)
valid = feed.update_story(story, is_manual=True)
if valid:
database.put_story(story)
search.put_story(story)
return {'nid': nid}
else:
raise Exception('Invalid article')
news_cache.pop(nid, '')
raise Exception('Invalid article')
except BaseException as e:
logging.error('Problem with article submission: {} - {}'.format(e.__class__.__name__, str(e)))
print(traceback.format_exc())
abort(400)
@flask_app.route('/api/<sid>')
def story(sid):
story = database.get_story(sid)
if story:
# hacky nested json
res = Response('{"story":' + story.full_json + '}')
res.headers['content-type'] = 'application/json'
return res
else:
return abort(404)
story = get_story(sid)
return dict(story=story) if story else abort(404)
@flask_app.route('/')
@flask_app.route('/search')
@ -126,9 +149,8 @@ def static_story(sid):
except NotFound:
pass
story = database.get_story(sid)
story = get_story(sid)
if not story: return abort(404)
story = json.loads(story.full_json)
score = story['score']
num_comments = story['num_comments']
@ -145,7 +167,7 @@ def static_story(sid):
url=url,
description=description)
http_server = WSGIServer(('', 43842), flask_app)
http_server = WSGIServer(('', 33842), flask_app)
def feed_thread():
global news_index
@ -154,42 +176,39 @@ def feed_thread():
while True:
# onboard new stories
if news_index == 0:
for ref, source in feed.list():
if database.get_story_by_ref(ref):
continue
try:
nid = new_id()
database.put_ref(ref, nid, source)
logging.info('Added ref ' + ref)
except database.IntegrityError:
continue
feed_list = feed.list()
new_items = [(ref, source) for ref, source in feed_list if ref not in news_list]
for ref, source in new_items:
news_list.insert(0, ref)
nid = new_id()
news_ref_to_id[ref] = nid
news_cache[nid] = dict(id=nid, ref=ref, source=source)
ref_list = database.get_reflist(FEED_LENGTH)
if len(new_items):
logging.info('Added {} new refs.'.format(len(new_items)))
# drop old ones
while len(news_list) > CACHE_LENGTH:
old_ref = news_list[-1]
remove_ref(old_ref)
# update current stories
if news_index < len(ref_list):
item = ref_list[news_index]
try:
story_json = database.get_story(item['sid']).full_json
story = json.loads(story_json)
except AttributeError:
story = dict(id=item['sid'], ref=item['ref'], source=item['source'])
valid = feed.update_story(story)
if news_index < len(news_list):
update_ref = news_list[news_index]
update_id = news_ref_to_id[update_ref]
news_story = news_cache[update_id]
valid = feed.update_story(news_story)
if valid:
database.put_story(story)
search.put_story(story)
archive.update(news_story)
else:
database.del_ref(item['ref'])
logging.info('Removed ref {}'.format(item['ref']))
remove_ref(update_ref)
else:
logging.info('Skipping index')
logging.info('Skipping update - no story #' + str(news_index+1))
gevent.sleep(6)
news_index += 1
if news_index == FEED_LENGTH: news_index = 0
if news_index == CACHE_LENGTH: news_index = 0
except KeyboardInterrupt:
logging.info('Ending feed thread...')
@ -205,3 +224,9 @@ try:
http_server.serve_forever()
except KeyboardInterrupt:
logging.info('Exiting...')
finally:
with shelve.open(DATA_FILE) as db:
logging.info('Writing caches to disk...')
db['news_list'] = news_list
db['news_ref_to_id'] = news_ref_to_id
db['news_cache'] = news_cache

10
webclient/src/Feed.js
View File

@ -36,6 +36,12 @@ class Feed extends React.Component {
this.props.updateCache(x.id, result.story);
}, error => {}
);
if (i < 20 && x.img) {
const img = new Image();
img.src = x.img;
console.log('prefetched image', x.img);
}
});
}
},
@ -59,6 +65,10 @@ class Feed extends React.Component {
<div>
{stories.map((x, i) =>
<div className='item' key={i}>
<div className='num'>
{i+1}.
</div>
<div className='title'>
<Link className='link' to={'/' + x.id}>
<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}

4
webclient/src/Results.js
View File

@ -64,6 +64,10 @@ class Results extends React.Component {
{stories.length ?
stories.map((x, i) =>
<div className='item' key={i}>
<div className='num'>
{i+1}.
</div>
<div className='title'>
<Link className='link' to={'/' + x.id}>
<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}

2
webclient/src/Search.js
View File

@ -37,7 +37,7 @@ class Search extends Component {
<span className='search'>
<form onSubmit={this.searchAgain}>
<input
placeholder='Search... (fixed)'
placeholder='Search...'
value={search}
onChange={this.searchArticles}
ref={this.inputRef}

7
webclient/src/Style-light.css
View File

@ -46,7 +46,7 @@ pre {
.item {
display: table;
color: #828282;
margin-bottom: 0.7rem;
margin-bottom: 0.6rem;
}
.item .source-logo {
@ -61,6 +61,11 @@ pre {
text-decoration: underline;
}
.item .num {
display: table-cell;
width: 2em;
}
.item a.link {
font-size: 1.1rem;
color: #000000;