Add script to reindex search, abstract search API
This commit is contained in:
parent
7a131ebd03
commit
dcedd4caa1
45
apiserver/reindex.py
Normal file
45
apiserver/reindex.py
Normal file
|
@ -0,0 +1,45 @@
|
|||
import logging
|
||||
logging.basicConfig(
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
level=logging.INFO)
|
||||
|
||||
import database
|
||||
from sqlalchemy import select
|
||||
import search
|
||||
import sys
|
||||
|
||||
import json
|
||||
import requests
|
||||
|
||||
database.init()
|
||||
search.init()
|
||||
|
||||
def count_stories():
|
||||
try:
|
||||
session = database.Session()
|
||||
return session.query(database.Story).count()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
def get_story_list():
|
||||
try:
|
||||
session = database.Session()
|
||||
return session.query(database.Story.sid).all()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
num_stories = count_stories()
|
||||
|
||||
print('Reindex {} stories?'.format(num_stories))
|
||||
print('Press ENTER to continue, ctrl-c to cancel')
|
||||
input()
|
||||
|
||||
count = 1
|
||||
for sid in get_story_list():
|
||||
story = database.get_story(sid)
|
||||
print('Indexing {}/{} id: {} title: {}'.format(count, num_stories, sid[0], story.title))
|
||||
story_obj = json.loads(story.meta_json)
|
||||
search.put_story(story_obj)
|
||||
count += 1
|
||||
|
|
@ -7,51 +7,34 @@ import requests
|
|||
|
||||
MEILI_URL = 'http://127.0.0.1:7700/'
|
||||
|
||||
def create_index():
|
||||
def meili_api(method, route, json=None, params=None):
|
||||
try:
|
||||
json = dict(name='qotnews', uid='qotnews')
|
||||
r = requests.post(MEILI_URL + 'indexes', json=json, timeout=2)
|
||||
if r.status_code != 201:
|
||||
r = method(MEILI_URL + route, json=json, params=params, timeout=4)
|
||||
if r.status_code > 299:
|
||||
raise Exception('Bad response code ' + str(r.status_code))
|
||||
return r.json()
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except BaseException as e:
|
||||
logging.error('Problem creating MeiliSearch index: {}'.format(str(e)))
|
||||
logging.error('Problem with MeiliSearch api route: %s: %s', route, str(e))
|
||||
return False
|
||||
|
||||
def create_index():
|
||||
json = dict(uid='qotnews', primaryKey='id')
|
||||
return meili_api(requests.post, 'indexes', json=json)
|
||||
|
||||
def update_rankings():
|
||||
try:
|
||||
json = ['typo', 'words', 'proximity', 'attribute', 'desc(date)', 'wordsPosition', 'exactness']
|
||||
r = requests.post(MEILI_URL + 'indexes/qotnews/settings/ranking-rules', json=json, timeout=2)
|
||||
if r.status_code != 202:
|
||||
raise Exception('Bad response code ' + str(r.status_code))
|
||||
return r.json()
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except BaseException as e:
|
||||
logging.error('Problem setting MeiliSearch ranking rules: {}'.format(str(e)))
|
||||
return False
|
||||
json = ['typo', 'words', 'proximity', 'attribute', 'desc(date)', 'wordsPosition', 'exactness']
|
||||
return meili_api(requests.post, 'indexes/qotnews/settings/ranking-rules', json=json)
|
||||
|
||||
def update_attributes():
|
||||
try:
|
||||
json = ['title', 'url', 'author', 'link', 'id']
|
||||
r = requests.post(MEILI_URL + 'indexes/qotnews/settings/searchable-attributes', json=json, timeout=2)
|
||||
if r.status_code != 202:
|
||||
raise Exception('Bad response code ' + str(r.status_code))
|
||||
return r.json()
|
||||
r = requests.delete(MEILI_URL + 'indexes/qotnews/settings/displayed-attributes', timeout=2)
|
||||
if r.status_code != 202:
|
||||
raise Exception('Bad response code ' + str(r.status_code))
|
||||
return r.json()
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except BaseException as e:
|
||||
logging.error('Problem setting MeiliSearch searchable attributes: {}'.format(str(e)))
|
||||
return False
|
||||
json = ['title', 'url', 'author', 'link', 'id']
|
||||
r = meili_api(requests.post, 'indexes/qotnews/settings/searchable-attributes', json=json)
|
||||
meili_api(requests.delete, 'indexes/qotnews/settings/displayed-attributes', json=json)
|
||||
return r
|
||||
|
||||
def init():
|
||||
create_index()
|
||||
print(create_index())
|
||||
update_rankings()
|
||||
update_attributes()
|
||||
|
||||
|
@ -59,31 +42,14 @@ def put_story(story):
|
|||
story = story.copy()
|
||||
story.pop('text', None)
|
||||
story.pop('comments', None)
|
||||
try:
|
||||
r = requests.post(MEILI_URL + 'indexes/qotnews/documents', json=[story], timeout=2)
|
||||
if r.status_code != 202:
|
||||
raise Exception('Bad response code ' + str(r.status_code))
|
||||
return r.json()
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except BaseException as e:
|
||||
logging.error('Problem putting MeiliSearch story: {}'.format(str(e)))
|
||||
return False
|
||||
return meili_api(requests.post, 'indexes/qotnews/documents', [story])
|
||||
|
||||
def search(q):
|
||||
try:
|
||||
params = dict(q=q, limit=250)
|
||||
r = requests.get(MEILI_URL + 'indexes/qotnews/search', params=params, timeout=2)
|
||||
if r.status_code != 200:
|
||||
raise Exception('Bad response code ' + str(r.status_code))
|
||||
return r.json()['hits']
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except BaseException as e:
|
||||
logging.error('Problem searching MeiliSearch: {}'.format(str(e)))
|
||||
return False
|
||||
params = dict(q=q, limit=250)
|
||||
r = meili_api(requests.get, 'indexes/qotnews/search', params=params)
|
||||
return r['hits']
|
||||
|
||||
if __name__ == '__main__':
|
||||
create_index()
|
||||
init()
|
||||
|
||||
print(search('the'))
|
||||
print(search('qot'))
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
Download MeiliSearch with:
|
||||
|
||||
```
|
||||
wget https://github.com/meilisearch/MeiliSearch/releases/download/v0.11.1/meilisearch-linux-amd64
|
||||
wget https://github.com/meilisearch/MeiliSearch/releases/download/v0.21.1/meilisearch-linux-amd64
|
||||
chmod +x meilisearch-linux-amd64
|
||||
```
|
||||
|
||||
|
|
Loading…
Reference in New Issue
Block a user