From b923908a45b6cce78718c108ab41b6bf6cd558d4 Mon Sep 17 00:00:00 2001
From: Tanner Collin <git@tannercollin.com>
Date: Thu, 25 Jun 2020 23:41:15 +0000
Subject: [PATCH] Begin initial sqlite conversion

---
 apiserver/database.py                 | 46 +++++++++++++++++++++++++++
 apiserver/migrate-whoosh-to-sqlite.py | 18 +++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 apiserver/database.py
 create mode 100644 apiserver/migrate-whoosh-to-sqlite.py

diff --git a/apiserver/database.py b/apiserver/database.py
new file mode 100644
index 0000000..dc7fd37
--- /dev/null
+++ b/apiserver/database.py
@@ -0,0 +1,46 @@
+import json
+
+from sqlalchemy import create_engine, Column, String
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+
+engine = create_engine('sqlite:///data/qotnews.sqlite')
+Session = sessionmaker(bind=engine)
+
+Base = declarative_base()
+
+class Story(Base):
+    __tablename__ = 'stories'
+
+    sid = Column(String, primary_key=True)
+    meta_json = Column(String)
+    full_json = Column(String)
+
+def init():
+    Base.metadata.create_all(engine)
+
+def get_meta(sid):
+    session = Session()
+    return session.query(Story).get(sid).meta_json
+
+def get_full(sid):
+    session = Session()
+    return session.query(Story).get(sid).full_json
+
+def put(story):
+    full_json = json.dumps(story)
+
+    story.pop('text')
+    story.pop('comments')
+    meta_json = json.dumps(story)
+
+    try:
+        session = Session()
+        s = Story(sid=story['id'], full_json=full_json, meta_json=meta_json)
+        session.merge(s)
+        session.commit()
+    except:
+        session.rollback()
+        raise
+    finally:
+        session.close() 
diff --git a/apiserver/migrate-whoosh-to-sqlite.py b/apiserver/migrate-whoosh-to-sqlite.py
new file mode 100644
index 0000000..bd0c65c
--- /dev/null
+++ b/apiserver/migrate-whoosh-to-sqlite.py
@@ -0,0 +1,18 @@
+import archive
+import database
+import json
+
+database.init()
+archive.init()
+
+with archive.ix.searcher() as searcher:
+    for docnum in searcher.document_numbers():
+        try:
+            print('docnum', docnum)
+            res = searcher.stored_fields(docnum)
+            print('id', res['id'])
+            database.put(res['story'])
+            print()
+        except:
+            print('skipping', docnum)
+            pass