jason/qotnews
1
0
Fork 0
forked from tanner/qotnews
Code Issues Pull Requests Releases Wiki Activity

allow re-scraping if simple scraper was used.

Browse Source
This commit is contained in:
Jason Schwarzenberger 2020-12-04 15:34:04 +13:00
parent 8727be6d86
commit 33a25fa34e
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apiserver/feed.py
View File

@ -132,7 +132,11 @@ def update_story(story, is_manual=False, urlref=None):
logging.info('Story too old, removing')
return False
if story.get('url', '') and not story.get('text', ''):
has_url = story.get('url') or False
has_text = story.get('text') or False
is_simple = story.get('scaper', '') == 'simple'
if has_url and (not has_text or is_simple):
if not get_content_type(story['url']).startswith('text/'):
logging.info('URL invalid file type / content type:')
logging.info(story['url'])
@ -149,6 +153,7 @@ def update_story(story, is_manual=False, urlref=None):
story['scraper'] = scraper
story['text'] = details.get('content', '')
if not story['text']: return False
story['last_update'] = time.time()
story['excerpt'] = details.get('excerpt', '')
story['scraper_link'] = details.get('scraper_link', '')
meta = details.get('meta')