Drop articles more than two days old

This commit is contained in:
2019-11-08 21:50:33 +00:00
parent 2edb3ceba7
commit db5097ac57
2 changed files with 8 additions and 1 deletions

View File

@@ -16,6 +16,7 @@ READ_API = 'http://127.0.0.1:33843'
ARCHIVE_FIRST = ['bloomberg.com', 'wsj.com']
INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif']
INVALID_DOMAINS = ['youtube.com']
TWO_DAYS = 60*60*24*2
def list():
feed = []
@@ -108,6 +109,10 @@ def update_story(story):
logging.info('Article not ready yet')
return False
if story['date'] and story['date'] + TWO_DAYS < time.time():
logging.info('Article too old, removing')
return False
if story.get('url', '') and not story.get('text', ''):
if any([story['url'].endswith(ext) for ext in INVALID_FILES]):
logging.info('URL invalid file type')