Move scripts into own folder

2024-02-27 18:32:29 +00:00
parent c2b9a1cb7a
commit f049d194ab
4 changed files with 0 additions and 0 deletions
--- a/apiserver/scripts/fix-stories.py
+++ b/apiserver/scripts/fix-stories.py
@@ -0,0 +1,58 @@
+import time
+import json
+import logging
+
+import feed
+import database
+import search
+
+database.init()
+
+def fix_gzip_bug(story_list):
+    FIX_THRESHOLD = 150
+
+    count = 1
+    for sid in story_list:
+        try:
+            sid = sid[0]
+            story = database.get_story(sid)
+            full_json = json.loads(story.full_json)
+            meta_json = json.loads(story.meta_json)
+            text = full_json.get('text', '')
+
+            count = text.count('<EFBFBD>')
+            if not count: continue
+
+            ratio = count / len(text) * 1000
+            print('Bad story:', sid, 'Num ?:', count, 'Ratio:', ratio)
+            if ratio < FIX_THRESHOLD: continue
+
+            print('Attempting to fix...')
+
+            valid = feed.update_story(meta_json, is_manual=True)
+            if valid:
+                database.put_story(meta_json)
+                search.put_story(meta_json)
+                print('Success')
+            else:
+                print('Story was not valid')
+
+            time.sleep(3)
+
+        except KeyboardInterrupt:
+            raise
+        except BaseException as e:
+            logging.exception(e)
+            breakpoint()
+
+if __name__ == '__main__':
+    num_stories = database.count_stories()
+
+    print('Fix {} stories?'.format(num_stories))
+    print('Press ENTER to continue, ctrl-c to cancel')
+    input()
+
+    story_list = database.get_story_list()
+
+    fix_gzip_bug(story_list)
+