diff --git a/apiserver/feed.py b/apiserver/feed.py index dab64a5..5e5605f 100644 --- a/apiserver/feed.py +++ b/apiserver/feed.py @@ -10,7 +10,7 @@ import itertools import settings from feeds import hackernews, reddit, tildes, substack, manual, news -from scrapers import outline, declutter, local +from scrapers import outline, declutter, browser, local INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com'] @@ -26,7 +26,7 @@ for key, value in settings.SITEMAP.items(): def get_list(): feeds = {} - + if settings.NUM_HACKERNEWS: feeds['hackernews'] = [(x, 'hackernews') for x in hackernews.feed()[:settings.NUM_HACKERNEWS]] @@ -63,6 +63,7 @@ def get_article(url): scrapers = { 'declutter': declutter, 'outline': outline, + 'browser': browser, 'local': local, } available = settings.SCRAPERS or ['local'] diff --git a/apiserver/scrapers/browser.py b/apiserver/scrapers/browser.py new file mode 100644 index 0000000..3de7dd0 --- /dev/null +++ b/apiserver/scrapers/browser.py @@ -0,0 +1,41 @@ +import logging +logging.basicConfig( + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + level=logging.DEBUG) +import requests + +READ_API = 'http://127.0.0.1:33843/browser/details' +READ_COMMENT__API = 'http://127.0.0.1:33843/browser/commentd' +TIMEOUT = 60 + + +def get_html(url): + logging.info(f"Reader Scraper: {url}") + details = get_details(url) + if not details: + return '' + return details['content'] + +def get_details(url): + try: + r = requests.post(READ_API, data=dict(url=url), timeout=TIMEOUT) + if r.status_code != 200: + raise Exception('Bad response code ' + str(r.status_code)) + return r.json() + except KeyboardInterrupt: + raise + except BaseException as e: + logging.error('Problem Scraping article: {}'.format(str(e))) + return None + +def get_comments(url): + try: + r = requests.post(READ_COMMENT_API, data=dict(url=url), timeout=TIMEOUT) + if r.status_code != 200: + raise Exception('Bad response code ' + str(r.status_code)) + return r.json() + except KeyboardInterrupt: + raise + except BaseException as e: + logging.error('Problem getting comments for article: {}'.format(str(e))) + return None diff --git a/apiserver/settings.py.example b/apiserver/settings.py.example index 089a559..62cec23 100644 --- a/apiserver/settings.py.example +++ b/apiserver/settings.py.example @@ -23,7 +23,7 @@ SUBSTACK = {} CATEGORY = {} # CATEGORY['rnz national'] = { 'url': "https://www.rnz.co.nz/news/national", 'count': 10}, -SCRAPERS = ['declutter', 'outline', 'local'] +SCRAPERS = ['browser', 'declutter', 'outline', 'local'] # Reddit account info # leave blank if not using Reddit