parent
dbdcfaa921
commit
b923908a45
2 changed files with 64 additions and 0 deletions
@ -0,0 +1,46 @@ |
|||||||
|
import json |
||||||
|
|
||||||
|
from sqlalchemy import create_engine, Column, String |
||||||
|
from sqlalchemy.ext.declarative import declarative_base |
||||||
|
from sqlalchemy.orm import sessionmaker |
||||||
|
|
||||||
|
engine = create_engine('sqlite:///data/qotnews.sqlite') |
||||||
|
Session = sessionmaker(bind=engine) |
||||||
|
|
||||||
|
Base = declarative_base() |
||||||
|
|
||||||
|
class Story(Base): |
||||||
|
__tablename__ = 'stories' |
||||||
|
|
||||||
|
sid = Column(String, primary_key=True) |
||||||
|
meta_json = Column(String) |
||||||
|
full_json = Column(String) |
||||||
|
|
||||||
|
def init(): |
||||||
|
Base.metadata.create_all(engine) |
||||||
|
|
||||||
|
def get_meta(sid): |
||||||
|
session = Session() |
||||||
|
return session.query(Story).get(sid).meta_json |
||||||
|
|
||||||
|
def get_full(sid): |
||||||
|
session = Session() |
||||||
|
return session.query(Story).get(sid).full_json |
||||||
|
|
||||||
|
def put(story): |
||||||
|
full_json = json.dumps(story) |
||||||
|
|
||||||
|
story.pop('text') |
||||||
|
story.pop('comments') |
||||||
|
meta_json = json.dumps(story) |
||||||
|
|
||||||
|
try: |
||||||
|
session = Session() |
||||||
|
s = Story(sid=story['id'], full_json=full_json, meta_json=meta_json) |
||||||
|
session.merge(s) |
||||||
|
session.commit() |
||||||
|
except: |
||||||
|
session.rollback() |
||||||
|
raise |
||||||
|
finally: |
||||||
|
session.close() |
@ -0,0 +1,18 @@ |
|||||||
|
import archive |
||||||
|
import database |
||||||
|
import json |
||||||
|
|
||||||
|
database.init() |
||||||
|
archive.init() |
||||||
|
|
||||||
|
with archive.ix.searcher() as searcher: |
||||||
|
for docnum in searcher.document_numbers(): |
||||||
|
try: |
||||||
|
print('docnum', docnum) |
||||||
|
res = searcher.stored_fields(docnum) |
||||||
|
print('id', res['id']) |
||||||
|
database.put(res['story']) |
||||||
|
print() |
||||||
|
except: |
||||||
|
print('skipping', docnum) |
||||||
|
pass |
Loading…
Reference in new issue