tanner/qotnews
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Remove document img and ignore r/technology

Browse Source
This commit is contained in:
Tanner Collin 2020-07-06 21:44:16 +00:00
parent ebedaef00b
commit 9c116bde4a
1 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
apiserver/migrate-whoosh-to-sqlite.py
View File

@ -43,22 +43,29 @@ with archive.ix.searcher() as searcher:
print('num', count, 'id', doc['id']) print('num', count, 'id', doc['id'])
count += 1 count += 1
story = doc['story']
story.pop('img', None)
if 'reddit.com/r/technology' in story['link']:
print('skipping r/technology')
continue
try: try:
database.put_story(doc['story']) database.put_story(story)
except database.IntegrityError: except database.IntegrityError:
print('collision!') print('collision!')
old_story = database.get_story_by_ref(doc['story']['ref']) old_story = database.get_story_by_ref(story['ref'])
story = json.loads(old_story.full_json) old_story = json.loads(old_story.full_json)
if doc['story']['num_comments'] > story['num_comments']: if story['num_comments'] > old_story['num_comments']:
print('more comments, replacing') print('more comments, replacing')
database_del_story_by_ref(doc['story']['ref']) database_del_story_by_ref(story['ref'])
database.put_story(doc['story']) database.put_story(story)
search_del_story(story['id']) search_del_story(old_story['id'])
else: else:
print('fewer comments, skipping') print('fewer comments, skipping')
continue continue
search.put_story(doc['story']) search.put_story(story)
print() print()
except KeyboardInterrupt: except KeyboardInterrupt:
break break