jason/qotnews
1
0
Fork 0
forked from tanner/qotnews
Code Issues Pull Requests Releases Wiki Activity

Write fixed stories to database

Browse Source
This commit is contained in:
Tanner Collin 2022-07-05 00:57:56 +00:00
parent dc83a70887
commit 9e9571a3c0
1 changed files with 20 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
apiserver/fix-stories.py
View File

@ -1,7 +1,9 @@
import logging
import database
import json import json
import logging
import feed
import database
import search
database.init() database.init()
@ -14,6 +16,7 @@ def fix_gzip_bug(story_list):
sid = sid[0] sid = sid[0]
story = database.get_story(sid) story = database.get_story(sid)
full_json = json.loads(story.full_json) full_json = json.loads(story.full_json)
meta_json = json.loads(story.meta_json)
text = full_json.get('text', '') text = full_json.get('text', '')
count = text.count('<EFBFBD>') count = text.count('<EFBFBD>')
@ -21,6 +24,20 @@ def fix_gzip_bug(story_list):
ratio = count / len(text) * 1000 ratio = count / len(text) * 1000
print('Bad story:', sid, 'Num ?:', count, 'Ratio:', ratio) print('Bad story:', sid, 'Num ?:', count, 'Ratio:', ratio)
if ratio < FIX_THRESHOLD: continue
print('Attempting to fix...')
valid = feed.update_story(meta_json, is_manual=True)
if valid:
database.put_story(meta_json)
search.put_story(meta_json)
print('Success')
else:
print('Story was not valid')
time.sleep(3)
except KeyboardInterrupt: except KeyboardInterrupt:
raise raise
except BaseException as e: except BaseException as e: