jason/qotnews
1
0
Fork 0
forked from tanner/qotnews
Code Issues Pull Requests Releases Wiki Activity

Drop articles more than two days old

Browse Source
This commit is contained in:
Tanner Collin 2019-11-08 21:50:33 +00:00
parent 2edb3ceba7
commit db5097ac57
2 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apiserver/feed.py
View File

@ -16,6 +16,7 @@ READ_API = 'http://127.0.0.1:33843'
ARCHIVE_FIRST = ['bloomberg.com', 'wsj.com'] ARCHIVE_FIRST = ['bloomberg.com', 'wsj.com']
INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif'] INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif']
INVALID_DOMAINS = ['youtube.com'] INVALID_DOMAINS = ['youtube.com']
TWO_DAYS = 60*60*24*2
def list(): def list():
feed = [] feed = []
@ -108,6 +109,10 @@ def update_story(story):
logging.info('Article not ready yet') logging.info('Article not ready yet')
return False return False
if story['date'] and story['date'] + TWO_DAYS < time.time():
logging.info('Article too old, removing')
return False
if story.get('url', '') and not story.get('text', ''): if story.get('url', '') and not story.get('text', ''):
if any([story['url'].endswith(ext) for ext in INVALID_FILES]): if any([story['url'].endswith(ext) for ext in INVALID_FILES]):
logging.info('URL invalid file type') logging.info('URL invalid file type')

4
apiserver/server.py
View File

@ -75,7 +75,7 @@ cors = CORS(flask_app)
def api(): def api():
front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list] front_page = [news_cache[news_ref_to_id[ref]] for ref in news_list]
front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']] front_page = [copy.copy(x) for x in front_page if 'title' in x and x['title']]
front_page = front_page[:100] front_page = front_page[:60]
for story in front_page: for story in front_page:
story.pop('text', None) story.pop('text', None)
story.pop('comments', None) story.pop('comments', None)
@ -179,6 +179,8 @@ def feed_thread():
archive.update(news_story) archive.update(news_story)
else: else:
remove_ref(update_ref) remove_ref(update_ref)
else:
logging.info('Skipping update - no story #' + str(news_index+1))
gevent.sleep(3) gevent.sleep(3)