Fix search to work with low-RAM server
This commit is contained in:
parent
3f774a9e38
commit
55c282ee69
|
@ -8,12 +8,21 @@ from sqlalchemy import select
|
|||
import search
|
||||
import sys
|
||||
|
||||
import time
|
||||
import json
|
||||
import requests
|
||||
|
||||
database.init()
|
||||
search.init()
|
||||
|
||||
BATCH_SIZE = 5000
|
||||
|
||||
def put_stories(stories):
|
||||
return search.meili_api(requests.post, 'indexes/qotnews/documents', stories)
|
||||
|
||||
def get_update(update_id):
|
||||
return search.meili_api(requests.get, 'indexes/qotnews/updates/{}'.format(update_id))
|
||||
|
||||
def count_stories():
|
||||
try:
|
||||
session = database.Session()
|
||||
|
@ -35,11 +44,34 @@ if __name__ == '__main__':
|
|||
print('Press ENTER to continue, ctrl-c to cancel')
|
||||
input()
|
||||
|
||||
count = 1
|
||||
for sid in get_story_list():
|
||||
story = database.get_story(sid)
|
||||
print('Indexing {}/{} id: {} title: {}'.format(count, num_stories, sid[0], story.title))
|
||||
story_obj = json.loads(story.meta_json)
|
||||
search.put_story(story_obj)
|
||||
count += 1
|
||||
story_list = get_story_list()
|
||||
|
||||
count = 1
|
||||
while len(story_list):
|
||||
stories = []
|
||||
|
||||
for _ in range(BATCH_SIZE):
|
||||
try:
|
||||
sid = story_list.pop()
|
||||
except IndexError:
|
||||
break
|
||||
|
||||
story = database.get_story(sid)
|
||||
print('Indexing {}/{} id: {} title: {}'.format(count, num_stories, sid[0], story.title))
|
||||
story_obj = json.loads(story.meta_json)
|
||||
to_add = dict(title=story_obj['title'], id=story_obj['id'], date=story_obj['date'])
|
||||
stories.append(to_add)
|
||||
count += 1
|
||||
|
||||
res = put_stories(stories)
|
||||
update_id = res['updateId']
|
||||
|
||||
print('Waiting for processing', end='')
|
||||
while get_update(update_id)['status'] != 'processed':
|
||||
time.sleep(0.5)
|
||||
print('.', end='', flush=True)
|
||||
|
||||
print()
|
||||
|
||||
print('Done.')
|
||||
|
||||
|
|
|
@ -24,13 +24,14 @@ def create_index():
|
|||
return meili_api(requests.post, 'indexes', json=json)
|
||||
|
||||
def update_rankings():
|
||||
json = ['typo', 'words', 'proximity', 'attribute', 'desc(date)', 'wordsPosition', 'exactness']
|
||||
json = ['typo', 'words', 'proximity', 'date:desc', 'exactness']
|
||||
return meili_api(requests.post, 'indexes/qotnews/settings/ranking-rules', json=json)
|
||||
|
||||
def update_attributes():
|
||||
json = ['title', 'url', 'author', 'link', 'id']
|
||||
json = ['title']
|
||||
r = meili_api(requests.post, 'indexes/qotnews/settings/searchable-attributes', json=json)
|
||||
meili_api(requests.delete, 'indexes/qotnews/settings/displayed-attributes', json=json)
|
||||
json = ['id']
|
||||
r = meili_api(requests.post, 'indexes/qotnews/settings/displayed-attributes', json=json)
|
||||
return r
|
||||
|
||||
def init():
|
||||
|
@ -39,10 +40,8 @@ def init():
|
|||
update_attributes()
|
||||
|
||||
def put_story(story):
|
||||
story = story.copy()
|
||||
story.pop('text', None)
|
||||
story.pop('comments', None)
|
||||
return meili_api(requests.post, 'indexes/qotnews/documents', [story])
|
||||
to_add = dict(title=story['title'], id=story['id'], date=story['date'])
|
||||
return meili_api(requests.post, 'indexes/qotnews/documents', [to_add])
|
||||
|
||||
def search(q):
|
||||
params = dict(q=q, limit=250)
|
||||
|
@ -52,4 +51,6 @@ def search(q):
|
|||
if __name__ == '__main__':
|
||||
init()
|
||||
|
||||
print(update_rankings())
|
||||
|
||||
print(search('qot'))
|
||||
|
|
|
@ -57,7 +57,11 @@ def apisearch():
|
|||
results = search.search(q)
|
||||
else:
|
||||
results = []
|
||||
return dict(results=results)
|
||||
story_metas = [database.get_story(x['id']).meta_json for x in results]
|
||||
# hacky nested json
|
||||
res = Response('{"results":[' + ','.join(story_metas) + ']}')
|
||||
res.headers['content-type'] = 'application/json'
|
||||
return res
|
||||
|
||||
@flask_app.route('/api/submit', methods=['POST'], strict_slashes=False)
|
||||
def submit():
|
||||
|
|
Loading…
Reference in New Issue
Block a user