From b923908a45b6cce78718c108ab41b6bf6cd558d4 Mon Sep 17 00:00:00 2001 From: Tanner Collin Date: Thu, 25 Jun 2020 23:41:15 +0000 Subject: [PATCH] Begin initial sqlite conversion --- apiserver/database.py | 46 +++++++++++++++++++++++++++ apiserver/migrate-whoosh-to-sqlite.py | 18 +++++++++++ 2 files changed, 64 insertions(+) create mode 100644 apiserver/database.py create mode 100644 apiserver/migrate-whoosh-to-sqlite.py diff --git a/apiserver/database.py b/apiserver/database.py new file mode 100644 index 0000000..dc7fd37 --- /dev/null +++ b/apiserver/database.py @@ -0,0 +1,46 @@ +import json + +from sqlalchemy import create_engine, Column, String +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +engine = create_engine('sqlite:///data/qotnews.sqlite') +Session = sessionmaker(bind=engine) + +Base = declarative_base() + +class Story(Base): + __tablename__ = 'stories' + + sid = Column(String, primary_key=True) + meta_json = Column(String) + full_json = Column(String) + +def init(): + Base.metadata.create_all(engine) + +def get_meta(sid): + session = Session() + return session.query(Story).get(sid).meta_json + +def get_full(sid): + session = Session() + return session.query(Story).get(sid).full_json + +def put(story): + full_json = json.dumps(story) + + story.pop('text') + story.pop('comments') + meta_json = json.dumps(story) + + try: + session = Session() + s = Story(sid=story['id'], full_json=full_json, meta_json=meta_json) + session.merge(s) + session.commit() + except: + session.rollback() + raise + finally: + session.close() diff --git a/apiserver/migrate-whoosh-to-sqlite.py b/apiserver/migrate-whoosh-to-sqlite.py new file mode 100644 index 0000000..bd0c65c --- /dev/null +++ b/apiserver/migrate-whoosh-to-sqlite.py @@ -0,0 +1,18 @@ +import archive +import database +import json + +database.init() +archive.init() + +with archive.ix.searcher() as searcher: + for docnum in searcher.document_numbers(): + try: + print('docnum', docnum) + res = searcher.stored_fields(docnum) + print('id', res['id']) + database.put(res['story']) + print() + except: + print('skipping', docnum) + pass