79 lines
2.2 KiB
Python
79 lines
2.2 KiB
Python
import logging
|
|
logging.basicConfig(
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
level=logging.INFO)
|
|
|
|
import database
|
|
from sqlalchemy import select
|
|
import search
|
|
import sys
|
|
|
|
import time
|
|
import json
|
|
import requests
|
|
|
|
database.init()
|
|
search.init()
|
|
|
|
BATCH_SIZE = 5000
|
|
|
|
def put_stories(stories):
|
|
return search.meili_api(requests.post, 'indexes/qotnews/documents', stories)
|
|
|
|
def get_update(update_id):
|
|
return search.meili_api(requests.get, 'tasks/{}'.format(update_id))
|
|
|
|
if __name__ == '__main__':
|
|
num_stories = database.count_stories()
|
|
|
|
print('Reindex {} stories?'.format(num_stories))
|
|
print('Press ENTER to continue, ctrl-c to cancel')
|
|
input()
|
|
|
|
story_list = database.get_story_list()
|
|
|
|
count = 1
|
|
while len(story_list):
|
|
stories = []
|
|
|
|
for _ in range(BATCH_SIZE):
|
|
try:
|
|
sid = story_list.pop()
|
|
except IndexError:
|
|
break
|
|
|
|
story = database.get_story(sid)
|
|
print('Indexing {}/{} id: {} title: {}'.format(count, num_stories, sid[0], story.title))
|
|
story_obj = json.loads(story.meta_json)
|
|
stories.append(story_obj)
|
|
count += 1
|
|
|
|
res = put_stories(stories)
|
|
if not isinstance(res, dict):
|
|
raise RuntimeError('Meilisearch failed to enqueue documents: {}'.format(res))
|
|
|
|
# Meilisearch task responses expose the asynchronous task ID as taskUid.
|
|
update_id = res.get('taskUid') or res.get('uid')
|
|
if update_id is None:
|
|
raise RuntimeError('Meilisearch response did not include a task ID: {}'.format(res))
|
|
|
|
print('Waiting for processing', end='')
|
|
while True:
|
|
task = get_update(update_id)
|
|
if not isinstance(task, dict):
|
|
raise RuntimeError('Could not retrieve Meilisearch task {}: {}'.format(update_id, task))
|
|
|
|
status = task.get('status')
|
|
if status == 'succeeded':
|
|
break
|
|
if status in ('failed', 'canceled'):
|
|
raise RuntimeError('Meilisearch task {} {}: {}'.format(update_id, status, task))
|
|
|
|
time.sleep(0.5)
|
|
print('.', end='', flush=True)
|
|
|
|
print()
|
|
|
|
print('Done.')
|
|
|