From 539350a83dbe200af5ff9ab6c9f3764b49c9e175 Mon Sep 17 00:00:00 2001 From: Jason Schwarzenberger Date: Wed, 18 Nov 2020 17:21:37 +1300 Subject: [PATCH] port separation. --- apiserver/scrapers/declutter.py | 6 +++--- apiserver/scrapers/headless.py | 11 +++++------ apiserver/scrapers/simple.py | 4 ++-- apiserver/settings.py.example | 3 ++- readerserver | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apiserver/scrapers/declutter.py b/apiserver/scrapers/declutter.py index 841f99d..84500ec 100644 --- a/apiserver/scrapers/declutter.py +++ b/apiserver/scrapers/declutter.py @@ -4,9 +4,9 @@ logging.basicConfig( level=logging.DEBUG) import requests -DECLUTTER_API = 'https://declutter.1j.nz/details' -DECLUTTER_COMMENT_API = 'https://declutter.1j.nz/comments' -TIMEOUT = 30 +DECLUTTER_API = 'https://declutter.1j.nz/headless/details' +DECLUTTER_COMMENT_API = 'https://declutter.1j.nz/headless/comments' +TIMEOUT = 90 def get_html(url): diff --git a/apiserver/scrapers/headless.py b/apiserver/scrapers/headless.py index b93dabe..7d32f82 100644 --- a/apiserver/scrapers/headless.py +++ b/apiserver/scrapers/headless.py @@ -3,15 +3,14 @@ logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG) import requests -from settings import READER_PORT - -READ_API = 'http://127.0.0.1:{}/headless/details'.format(READER_PORT or 3000) -READ_COMMENT__API = 'http://127.0.0.1:{}/headless/comments'.format(READER_PORT or 3000) -TIMEOUT = 60 +from settings import HEADLESS_READER_PORT +READ_API = 'http://127.0.0.1:{}/headless/details'.format(HEADLESS_READER_PORT or 33843) +READ_COMMENT__API = 'http://127.0.0.1:{}/headless/comments'.format(HEADLESS_READER_PORT or 33843) +TIMEOUT = 90 def get_html(url): - logging.info(f"Headless Browser Scraper: {url}") + logging.info(f"Headless Scraper: {url}") details = get_details(url) if not details: return '' diff --git a/apiserver/scrapers/simple.py b/apiserver/scrapers/simple.py index 22107e0..433ba75 100644 --- a/apiserver/scrapers/simple.py +++ b/apiserver/scrapers/simple.py @@ -3,9 +3,9 @@ logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG) import requests -from settings import READER_PORT +from settings import SIMPLE_READER_PORT -READ_API = 'http://127.0.0.1:{}/simple/details'.format(READER_PORT or 3000) +READ_API = 'http://127.0.0.1:{}/simple/details'.format(SIMPLE_READER_PORT or 33843) TIMEOUT = 20 def get_html(url): diff --git a/apiserver/settings.py.example b/apiserver/settings.py.example index 39a444f..e00f10a 100644 --- a/apiserver/settings.py.example +++ b/apiserver/settings.py.example @@ -6,7 +6,8 @@ MAX_STORY_AGE = 3*24*60*60 SCRAPERS = ['headless', 'outline', 'declutter', 'simple'] API_PORT = 33842 -READER_PORT = 3000 +SIMPLE_READER_PORT = 33843 +HEADLESS_READER_PORT = 33843 # Feed Lengths # Number of top items from each site to pull diff --git a/readerserver b/readerserver index 50a94df..b8f2d40 160000 --- a/readerserver +++ b/readerserver @@ -1 +1 @@ -Subproject commit 50a94df7283e31680c5d94dd666bab58aea2e475 +Subproject commit b8f2d401a79380fecc063c4184f49433b5900ee4