tanner
/
qotnews
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Integrate sqlite database with server

Browse Source
master
Tanner Collin 4 years ago
parent 873dc44cb1
commit f46cafdc90
5 changed files with 100 additions and 128 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      apiserver/.gitignore
  2. 49
      apiserver/database.py
  3. 6
      apiserver/feeds/hackernews.py
  4. 5
      apiserver/migrate-whoosh-to-sqlite.py
  5. 167
      apiserver/server.py

1
apiserver/.gitignore vendored
Unescape View File

@ -109,3 +109,4 @@ praw.ini
data.db data.db
data.db.bak data.db.bak
data/archive/* data/archive/*
qotnews.sqlite

49
apiserver/database.py
Unescape View File

@ -3,6 +3,7 @@ import json
from sqlalchemy import create_engine, Column, String, ForeignKey, Integer from sqlalchemy import create_engine, Column, String, ForeignKey, Integer
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError
engine = create_engine('sqlite:///data/qotnews.sqlite') engine = create_engine('sqlite:///data/qotnews.sqlite')
Session = sessionmaker(bind=engine) Session = sessionmaker(bind=engine)
@ -15,6 +16,8 @@ class Story(Base):
sid = Column(String(16), primary_key=True) sid = Column(String(16), primary_key=True)
meta_json = Column(String) meta_json = Column(String)
full_json = Column(String) full_json = Column(String)
title = Column(String)
date = Column(Integer)
class Reflist(Base): class Reflist(Base):
__tablename__ = 'reflist' __tablename__ = 'reflist'
@ -26,24 +29,26 @@ class Reflist(Base):
def init(): def init():
Base.metadata.create_all(engine) Base.metadata.create_all(engine)
def get_meta(sid): def get_story(sid):
session = Session() session = Session()
return session.query(Story).get(sid).meta_json return session.query(Story).get(sid)
def get_full(sid):
session = Session()
return session.query(Story).get(sid).full_json
def put_story(story): def put_story(story):
full_json = json.dumps(story) full_json = json.dumps(story)
story.pop('text') story.pop('text', None)
story.pop('comments') story.pop('comments', None)
meta_json = json.dumps(story) meta_json = json.dumps(story)
try: try:
session = Session() session = Session()
s = Story(sid=story['id'], full_json=full_json, meta_json=meta_json) s = Story(
sid=story['id'],
full_json=full_json,
meta_json=meta_json,
title=story.get('title', None),
date=story.get('date', None),
)
session.merge(s) session.merge(s)
session.commit() session.commit()
except: except:
@ -52,11 +57,24 @@ def put_story(story):
finally: finally:
session.close() session.close()
def search(q):
session = Session()
return session.query(Story).filter(Story.title.contains(q))
def get_reflist(amount): def get_reflist(amount):
session = Session() session = Session()
q = session.query(Reflist).order_by(Reflist.rid.desc()).limit(amount) q = session.query(Reflist).order_by(Reflist.rid.desc()).limit(amount)
return [dict(ref=x.ref, sid=x.sid) for x in q.all()] return [dict(ref=x.ref, sid=x.sid) for x in q.all()]
def get_stories(amount):
session = Session()
q = session.query(Reflist, Story.meta_json).\
order_by(Reflist.rid.desc()).\
join(Story).\
filter(Story.title != None).\
limit(amount)
return [x[1] for x in q]
def put_ref(ref, sid): def put_ref(ref, sid):
try: try:
session = Session() session = Session()
@ -69,5 +87,18 @@ def put_ref(ref, sid):
finally: finally:
session.close() session.close()
def del_ref(ref):
try:
session = Session()
session.query(Reflist).filter(Reflist.ref==ref).delete()
session.commit()
except:
session.rollback()
raise
finally:
session.close()
if __name__ == '__main__': if __name__ == '__main__':
init() init()
print(get_stories(5))

6
apiserver/feeds/hackernews.py
Unescape View File

@ -30,7 +30,7 @@ def api(route, ref=None):
return False return False
def feed(): def feed():
return api(API_TOPSTORIES) or [] return [str(x) for x in api(API_TOPSTORIES)] or []
def comment(i): def comment(i):
if 'author' not in i: if 'author' not in i:
@ -77,6 +77,6 @@ def story(ref):
# scratchpad so I can quickly develop the parser # scratchpad so I can quickly develop the parser
if __name__ == '__main__': if __name__ == '__main__':
#print(feed()) print(feed())
#print(story(20763961)) #print(story(20763961))
print(story(20802050)) #print(story(20802050))

5
apiserver/migrate-whoosh-to-sqlite.py
Unescape View File

@ -8,14 +8,13 @@ archive.init()
with archive.ix.searcher() as searcher: with archive.ix.searcher() as searcher:
for docnum in searcher.document_numbers(): for docnum in searcher.document_numbers():
try: try:
if docnum > 500: #if docnum > 500:
break # break
print('docnum', docnum) print('docnum', docnum)
res = searcher.stored_fields(docnum) res = searcher.stored_fields(docnum)
print('id', res['id']) print('id', res['id'])
database.put_story(res['story']) database.put_story(res['story'])
database.put_ref(res['story']['ref'], res['id'])
print() print()
except BaseException as e: except BaseException as e:
print('skipping', docnum) print('skipping', docnum)

167
apiserver/server.py
Unescape View File

@ -4,13 +4,13 @@ logging.basicConfig(
level=logging.INFO) level=logging.INFO)
import copy import copy
import json
import threading import threading
import traceback import traceback
import time import time
import shelve
from urllib.parse import urlparse, parse_qs from urllib.parse import urlparse, parse_qs
import archive import database
import feed import feed
from utils import gen_rand_id from utils import gen_rand_id
@ -24,85 +24,40 @@ from gevent.pywsgi import WSGIServer
monkey.patch_all() monkey.patch_all()
archive.init() database.init()
CACHE_LENGTH = 150
DATA_FILE = 'data/data'
FEED_LENGTH = 75
news_index = 0 news_index = 0
with shelve.open(DATA_FILE) as db:
logging.info('Reading caches from disk...')
news_list = db.get('news_list', [])
news_ref_to_id = db.get('news_ref_to_id', {})
news_cache = db.get('news_cache', {})
# clean cache if broken
try:
for ref in news_list:
nid = news_ref_to_id[ref]
_ = news_cache[nid]
except KeyError as e:
ekey = str(e)
logging.error('Unable to find key {}. Trying to remove...'.format(ekey))
if ekey in news_cache:
news_cache.remove(ekey)
if ekey in news_list:
news_list.remove(ekey)
if ekey in news_ref_to_id:
news_ref_to_id.remove(ekey)
def get_story(sid):
if sid in news_cache:
return news_cache[sid]
else:
return archive.get_story(sid)
def new_id(): def new_id():
nid = gen_rand_id() nid = gen_rand_id()
while nid in news_cache or archive.get_story(nid): while database.get_story(nid):
nid = gen_rand_id() nid = gen_rand_id()
return nid return nid
def remove_ref(old_ref):
while old_ref in news_list:
news_list.remove(old_ref)
old_story = news_cache.pop(news_ref_to_id[old_ref])
old_id = news_ref_to_id.pop(old_ref)
logging.info('Removed ref {} id {}.'.format(old_ref, old_id))
build_folder = '../webclient/build' build_folder = '../webclient/build'
flask_app = Flask(__name__, template_folder=build_folder, static_folder=build_folder, static_url_path='') flask_app = Flask(__name__, template_folder=build_folder, static_folder=build_folder, static_url_path='')
cors = CORS(flask_app) cors = CORS(flask_app)
@flask_app.route('/api') @flask_app.route('/api')
def api(): def api():
try: stories = database.get_stories(FEED_LENGTH)
front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list] # hacky nested json
except KeyError as e: res = Response('{"stories":[' + ','.join(stories) + ']}')
ekey = str(e) res.headers['content-type'] = 'application/json'
logging.error('Unable to find key {}. Trying to remove...'.format(ekey)) return res
if ekey in news_cache:
news_cache.remove(ekey)
if ekey in news_list:
news_list.remove(ekey)
front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']]
front_page = front_page[:60]
for story in front_page:
story.pop('text', None)
story.pop('comments', None)
return {'stories': front_page}
@flask_app.route('/api/search', strict_slashes=False) @flask_app.route('/api/search', strict_slashes=False)
def search(): def search():
search = request.args.get('q', '') q = request.args.get('q', '')
if len(search) >= 3: if len(q) >= 3:
res = archive.search(search) results = [x.meta_json for x in database.search(q)]
else: else:
res = [] results = []
return {'results': res} # hacky nested json
res = Response('{"results":[' + ','.join(results) + ']}')
res.headers['content-type'] = 'application/json'
return res
@flask_app.route('/api/submit', methods=['POST'], strict_slashes=False) @flask_app.route('/api/submit', methods=['POST'], strict_slashes=False)
def submit(): def submit():
@ -124,14 +79,14 @@ def submit():
source = 'manual' source = 'manual'
ref = url ref = url
news_story = dict(id=nid, ref=ref, source=source) # TODO: return existing refs
news_cache[nid] = news_story
valid = feed.update_story(news_story, is_manual=True) story = dict(id=nid, ref=ref, source=source)
valid = feed.update_story(story, is_manual=True)
if valid: if valid:
archive.update(news_story) database.put_story(story)
return {'nid': nid} return {'nid': nid}
else: else:
news_cache.pop(nid, '')
raise Exception('Invalid article') raise Exception('Invalid article')
except BaseException as e: except BaseException as e:
@ -142,8 +97,14 @@ def submit():
@flask_app.route('/api/<sid>') @flask_app.route('/api/<sid>')
def story(sid): def story(sid):
story = get_story(sid) story = database.get_story(sid)
return dict(story=story) if story else abort(404) if story:
# hacky nested json
res = Response('{"story":' + story.full_json + '}')
res.headers['content-type'] = 'application/json'
return res
else:
return abort(404)
@flask_app.route('/') @flask_app.route('/')
@flask_app.route('/search') @flask_app.route('/search')
@ -161,8 +122,9 @@ def static_story(sid):
except NotFound: except NotFound:
pass pass
story = get_story(sid) story = database.get_story(sid)
if not story: return abort(404) if not story: return abort(404)
story = json.loads(story.full_json)
score = story['score'] score = story['score']
num_comments = story['num_comments'] num_comments = story['num_comments']
@ -179,58 +141,43 @@ def static_story(sid):
url=url, url=url,
description=description) description=description)
http_server = WSGIServer(('', 33842), flask_app) http_server = WSGIServer(('', 43842), flask_app)
def feed_thread(): def feed_thread():
global news_index global news_index
try: try:
while True: while True:
ref_list = database.get_reflist(FEED_LENGTH)
# onboard new stories # onboard new stories
if news_index == 0: if news_index == 0:
feed_list = feed.list() for ref, source in feed.list():
new_items = [(ref, source) for ref, source in feed_list if ref not in news_list] try:
for ref, source in new_items: nid = new_id()
news_list.insert(0, ref) database.put_ref(ref, nid)
nid = new_id() database.put_story(dict(id=nid, ref=ref, source=source))
news_ref_to_id[ref] = nid logging.info('Added ref ' + ref)
news_cache[nid] = dict(id=nid, ref=ref, source=source) except database.IntegrityError:
continue
if len(new_items):
logging.info('Added {} new refs.'.format(len(new_items)))
# drop old ones
while len(news_list) > CACHE_LENGTH:
old_ref = news_list[-1]
remove_ref(old_ref)
# update current stories # update current stories
if news_index < len(news_list): if news_index < len(ref_list):
try: update_ref = ref_list[news_index]['ref']
update_ref = news_list[news_index] update_sid = ref_list[news_index]['sid']
update_id = news_ref_to_id[update_ref] story_json = database.get_story(update_sid).full_json
news_story = news_cache[update_id] story = json.loads(story_json)
except KeyError as e: valid = feed.update_story(story)
ekey = str(e)
logging.error('Unable to find key {}. Trying to remove...'.format(ekey))
if ekey in news_cache:
news_cache.remove(ekey)
if ekey in news_list:
news_list.remove(ekey)
if ekey in news_ref_to_id:
news_ref_to_id.remove(ekey)
valid = feed.update_story(news_story)
if valid: if valid:
archive.update(news_story) database.put_story(story)
else: else:
remove_ref(update_ref) database.del_ref(update_ref)
else: logging.info('Removed ref {}'.format(update_ref))
logging.info('Skipping update - no story #' + str(news_index+1))
gevent.sleep(6) gevent.sleep(6)
news_index += 1 news_index += 1
if news_index == CACHE_LENGTH: news_index = 0 if news_index == FEED_LENGTH: news_index = 0
except KeyboardInterrupt: except KeyboardInterrupt:
logging.info('Ending feed thread...') logging.info('Ending feed thread...')
@ -246,9 +193,3 @@ try:
http_server.serve_forever() http_server.serve_forever()
except KeyboardInterrupt: except KeyboardInterrupt:
logging.info('Exiting...') logging.info('Exiting...')
finally:
with shelve.open(DATA_FILE) as db:
logging.info('Writing caches to disk...')
db['news_list'] = news_list
db['news_ref_to_id'] = news_ref_to_id
db['news_cache'] = news_cache

Write Preview
Loading…
Cancel
Save