Ignore certain files and domains, remove refs

apiserver/feed.py

@@ -11,11 +11,14 @@ from feeds import hackernews, reddit, tildes
 OUTLINE_API = 'https://outlineapi.com/article'
 READ_API = 'http://127.0.0.1:33843'
 
+INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif']
+INVALID_DOMAINS = ['youtube.com']
+
 def list():
     feed = []
     feed += [(x, 'hackernews') for x in hackernews.feed()[:10]]
-    feed += [(x, 'reddit') for x in reddit.feed()[:5]]
-    feed += [(x, 'tildes') for x in tildes.feed()[:5]]
+    feed += [(x, 'reddit') for x in reddit.feed()[:10]]
+    feed += [(x, 'tildes') for x in tildes.feed()[:10]]
     return feed
 
 def get_article(url):
@@ -58,17 +61,29 @@ def update_story(story):
         res = reddit.story(story['ref'])
     elif story['source'] == 'tildes':
         res = tildes.story(story['ref'])
-    else:
-        return
 
     if res:
-        story.update(res)
+        story.update(res) # join dicts
+    else:
+        logging.info('Article not ready yet')
+        return False
+
     if story.get('url', '') and not story.get('text', ''):
-        if not story['url'].endswith('.pdf'):
-            logging.info('Getting article ' + story['url'])
-            story['text'] = get_article(story['url'])
-        else:
-            story['text'] = '<p>Unsupported article type.</p>'
+        for ext in INVALID_FILES:
+            if story['url'].endswith(ext):
+                logging.info('URL invalid file type ({})'.format(ext))
+                return False
+
+        for domain in INVALID_DOMAINS:
+            if domain in story['url']:
+                logging.info('URL invalid domain ({})'.format(domain))
+                return False
+
+        logging.info('Getting article ' + story['url'])
+        story['text'] = get_article(story['url'])
+        if not story['text']: return False
+
+    return True
 
 if __name__ == '__main__':
     test_news_cache = {}