Add script to reindex search, abstract search API

This commit is contained in:
Tanner Collin 2021-09-06 00:20:21 +00:00
parent 7a131ebd03
commit dcedd4caa1
3 changed files with 67 additions and 56 deletions

45
apiserver/reindex.py Normal file
View File

@ -0,0 +1,45 @@
import logging
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.INFO)
import database
from sqlalchemy import select
import search
import sys
import json
import requests
database.init()
search.init()
def count_stories():
try:
session = database.Session()
return session.query(database.Story).count()
finally:
session.close()
def get_story_list():
try:
session = database.Session()
return session.query(database.Story.sid).all()
finally:
session.close()
if __name__ == '__main__':
num_stories = count_stories()
print('Reindex {} stories?'.format(num_stories))
print('Press ENTER to continue, ctrl-c to cancel')
input()
count = 1
for sid in get_story_list():
story = database.get_story(sid)
print('Indexing {}/{} id: {} title: {}'.format(count, num_stories, sid[0], story.title))
story_obj = json.loads(story.meta_json)
search.put_story(story_obj)
count += 1

View File

@ -7,51 +7,34 @@ import requests
MEILI_URL = 'http://127.0.0.1:7700/' MEILI_URL = 'http://127.0.0.1:7700/'
def create_index(): def meili_api(method, route, json=None, params=None):
try: try:
json = dict(name='qotnews', uid='qotnews') r = method(MEILI_URL + route, json=json, params=params, timeout=4)
r = requests.post(MEILI_URL + 'indexes', json=json, timeout=2) if r.status_code > 299:
if r.status_code != 201:
raise Exception('Bad response code ' + str(r.status_code)) raise Exception('Bad response code ' + str(r.status_code))
return r.json() return r.json()
except KeyboardInterrupt: except KeyboardInterrupt:
raise raise
except BaseException as e: except BaseException as e:
logging.error('Problem creating MeiliSearch index: {}'.format(str(e))) logging.error('Problem with MeiliSearch api route: %s: %s', route, str(e))
return False return False
def create_index():
json = dict(uid='qotnews', primaryKey='id')
return meili_api(requests.post, 'indexes', json=json)
def update_rankings(): def update_rankings():
try:
json = ['typo', 'words', 'proximity', 'attribute', 'desc(date)', 'wordsPosition', 'exactness'] json = ['typo', 'words', 'proximity', 'attribute', 'desc(date)', 'wordsPosition', 'exactness']
r = requests.post(MEILI_URL + 'indexes/qotnews/settings/ranking-rules', json=json, timeout=2) return meili_api(requests.post, 'indexes/qotnews/settings/ranking-rules', json=json)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem setting MeiliSearch ranking rules: {}'.format(str(e)))
return False
def update_attributes(): def update_attributes():
try:
json = ['title', 'url', 'author', 'link', 'id'] json = ['title', 'url', 'author', 'link', 'id']
r = requests.post(MEILI_URL + 'indexes/qotnews/settings/searchable-attributes', json=json, timeout=2) r = meili_api(requests.post, 'indexes/qotnews/settings/searchable-attributes', json=json)
if r.status_code != 202: meili_api(requests.delete, 'indexes/qotnews/settings/displayed-attributes', json=json)
raise Exception('Bad response code ' + str(r.status_code)) return r
return r.json()
r = requests.delete(MEILI_URL + 'indexes/qotnews/settings/displayed-attributes', timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem setting MeiliSearch searchable attributes: {}'.format(str(e)))
return False
def init(): def init():
create_index() print(create_index())
update_rankings() update_rankings()
update_attributes() update_attributes()
@ -59,31 +42,14 @@ def put_story(story):
story = story.copy() story = story.copy()
story.pop('text', None) story.pop('text', None)
story.pop('comments', None) story.pop('comments', None)
try: return meili_api(requests.post, 'indexes/qotnews/documents', [story])
r = requests.post(MEILI_URL + 'indexes/qotnews/documents', json=[story], timeout=2)
if r.status_code != 202:
raise Exception('Bad response code ' + str(r.status_code))
return r.json()
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem putting MeiliSearch story: {}'.format(str(e)))
return False
def search(q): def search(q):
try:
params = dict(q=q, limit=250) params = dict(q=q, limit=250)
r = requests.get(MEILI_URL + 'indexes/qotnews/search', params=params, timeout=2) r = meili_api(requests.get, 'indexes/qotnews/search', params=params)
if r.status_code != 200: return r['hits']
raise Exception('Bad response code ' + str(r.status_code))
return r.json()['hits']
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem searching MeiliSearch: {}'.format(str(e)))
return False
if __name__ == '__main__': if __name__ == '__main__':
create_index() init()
print(search('the')) print(search('qot'))

View File

@ -3,7 +3,7 @@
Download MeiliSearch with: Download MeiliSearch with:
``` ```
wget https://github.com/meilisearch/MeiliSearch/releases/download/v0.11.1/meilisearch-linux-amd64 wget https://github.com/meilisearch/MeiliSearch/releases/download/v0.21.1/meilisearch-linux-amd64
chmod +x meilisearch-linux-amd64 chmod +x meilisearch-linux-amd64
``` ```