parent
45b75b420b
commit
7cb87b59fe
14 changed files with 328 additions and 63 deletions
@ -0,0 +1,52 @@ |
||||
from whoosh.analysis import StemmingAnalyzer, CharsetFilter, NgramFilter |
||||
from whoosh.index import create_in, open_dir, exists_in |
||||
from whoosh.fields import * |
||||
from whoosh.qparser import QueryParser |
||||
from whoosh.support.charset import accent_map |
||||
|
||||
analyzer = StemmingAnalyzer() | CharsetFilter(accent_map) | NgramFilter(minsize=3) |
||||
|
||||
title_field = TEXT(analyzer=analyzer, stored=True) |
||||
id_field = ID(unique=True, stored=True) |
||||
|
||||
schema = Schema( |
||||
id=id_field, |
||||
title=title_field, |
||||
story=STORED, |
||||
) |
||||
|
||||
ARCHIVE_LOCATION = 'data/archive' |
||||
|
||||
ix = None |
||||
|
||||
def init(): |
||||
global ix |
||||
|
||||
if exists_in(ARCHIVE_LOCATION): |
||||
ix = open_dir(ARCHIVE_LOCATION) |
||||
else: |
||||
ix = create_in(ARCHIVE_LOCATION, schema) |
||||
|
||||
def update(story): |
||||
writer = ix.writer() |
||||
writer.update_document( |
||||
id=story['id'], |
||||
title=story['title'], |
||||
story=story, |
||||
) |
||||
writer.commit() |
||||
|
||||
def get_story(id): |
||||
with ix.searcher() as searcher: |
||||
result = searcher.document(id=id) |
||||
return result['story'] if result else None |
||||
|
||||
def search(search): |
||||
with ix.searcher() as searcher: |
||||
query = QueryParser('title', ix.schema).parse(search) |
||||
results = searcher.search(query) |
||||
stories = [r['story'] for r in results] |
||||
for s in stories: |
||||
s.pop('text', '') |
||||
s.pop('comments', '') |
||||
return stories |
||||
@ -0,0 +1,26 @@ |
||||
import shelve |
||||
|
||||
import archive |
||||
|
||||
archive.init() |
||||
|
||||
#with shelve.open('data/data') as db: |
||||
# to_delete = [] |
||||
# |
||||
# for s in db.values(): |
||||
# if 'title' in s: |
||||
# archive.update(s) |
||||
# if 'id' in s: |
||||
# to_delete.append(s['id']) |
||||
# |
||||
# for id in to_delete: |
||||
# del db[id] |
||||
# |
||||
# for s in db['news_cache'].values(): |
||||
# if 'title' in s: |
||||
# archive.update(s) |
||||
|
||||
#with shelve.open('data/whoosh') as db: |
||||
# for s in db['news_cache'].values(): |
||||
# if 'title' in s and not archive.get_story(s['id']): |
||||
# archive.update(s) |
||||
@ -0,0 +1,83 @@ |
||||
import React from 'react'; |
||||
import { Link } from 'react-router-dom'; |
||||
import { Helmet } from 'react-helmet'; |
||||
import { siteLogo, sourceLink, infoLine } from './utils.js'; |
||||
|
||||
class Results extends React.Component { |
||||
constructor(props) { |
||||
super(props); |
||||
|
||||
this.state = { |
||||
stories: false, |
||||
error: false, |
||||
}; |
||||
|
||||
} |
||||
|
||||
performSearch = () => { |
||||
const search = this.props.location.search; |
||||
fetch('/api/search' + search) |
||||
.then(res => res.json()) |
||||
.then( |
||||
(result) => { |
||||
this.setState({ stories: result.results }); |
||||
}, |
||||
(error) => { |
||||
this.setState({ error: true }); |
||||
} |
||||
); |
||||
} |
||||
|
||||
componentDidMount() { |
||||
this.performSearch(); |
||||
} |
||||
|
||||
componentDidUpdate(prevProps) { |
||||
if (this.props.location.search !== prevProps.location.search) { |
||||
this.performSearch(); |
||||
} |
||||
} |
||||
|
||||
render() { |
||||
const stories = this.state.stories; |
||||
const error = this.state.error; |
||||
|
||||
return ( |
||||
<div className='container'> |
||||
<Helmet> |
||||
<title>Feed - QotNews</title> |
||||
</Helmet> |
||||
{error && <p>Connection error?</p>} |
||||
{stories ? |
||||
<div> |
||||
{stories.length ? |
||||
stories.map((x, i) => |
||||
<div className='item' key={i}> |
||||
<div className='num'> |
||||
{i+1}. |
||||
</div> |
||||
|
||||
<div className='title'> |
||||
<Link className='link' to={'/' + x.id}>{siteLogo[x.source]} {x.title}</Link> |
||||
|
||||
<span className='source'> |
||||
​({sourceLink(x)}) |
||||
</span> |
||||
</div> |
||||
|
||||
{infoLine(x)} |
||||
</div> |
||||
) |
||||
: |
||||
<p>no results</p> |
||||
} |
||||
</div> |
||||
: |
||||
<p>loading...</p> |
||||
} |
||||
</div> |
||||
); |
||||
} |
||||
} |
||||
|
||||
export default Results; |
||||
@ -0,0 +1,53 @@ |
||||
import React, { Component } from 'react'; |
||||
import { withRouter } from 'react-router-dom'; |
||||
import queryString from 'query-string'; |
||||
|
||||
const getSearch = props => queryString.parse(props.location.search).q; |
||||
|
||||
class Search extends Component { |
||||
constructor(props) { |
||||
super(props); |
||||
|
||||
this.state = {search: getSearch(this.props)}; |
||||
this.inputRef = React.createRef(); |
||||
} |
||||
|
||||
searchArticles = (event) => { |
||||
const search = event.target.value; |
||||
this.setState({search: search}); |
||||
if (search.length >= 3) { |
||||
const searchQuery = queryString.stringify({ 'q': search }); |
||||
this.props.history.replace('/search?' + searchQuery); |
||||
} else { |
||||
this.props.history.replace('/'); |
||||
} |
||||
} |
||||
|
||||
searchAgain = (event) => { |
||||
event.preventDefault(); |
||||
const searchString = queryString.stringify({ 'q': event.target[0].value }); |
||||
this.props.history.push('/search?' + searchString); |
||||
this.inputRef.current.blur(); |
||||
} |
||||
|
||||
render() { |
||||
const search = this.state.search; |
||||
|
||||
return ( |
||||
<div className='search'> |
||||
<div className='search-inside'> |
||||
<form onSubmit={this.searchAgain}> |
||||
<input |
||||
placeholder='Search...' |
||||
value={search} |
||||
onChange={this.searchArticles} |
||||
ref={this.inputRef} |
||||
/> |
||||
</form> |
||||
</div> |
||||
</div> |
||||
); |
||||
} |
||||
} |
||||
|
||||
export default withRouter(Search); |
||||
Loading…
Reference in new issue