port separation.

apiserver/scrapers/declutter.py

@@ -4,9 +4,9 @@ logging.basicConfig(
         level=logging.DEBUG)
 import requests
 
-DECLUTTER_API = 'https://declutter.1j.nz/details'
-DECLUTTER_COMMENT_API = 'https://declutter.1j.nz/comments'
-TIMEOUT = 30
+DECLUTTER_API = 'https://declutter.1j.nz/headless/details'
+DECLUTTER_COMMENT_API = 'https://declutter.1j.nz/headless/comments'
+TIMEOUT = 90
 
 
 def get_html(url):

apiserver/scrapers/headless.py

@@ -3,15 +3,14 @@ logging.basicConfig(
         format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
         level=logging.DEBUG)
 import requests
-from settings import READER_PORT
-
-READ_API = 'http://127.0.0.1:{}/headless/details'.format(READER_PORT or 3000)
-READ_COMMENT__API = 'http://127.0.0.1:{}/headless/comments'.format(READER_PORT or 3000)
-TIMEOUT = 60
+from settings import HEADLESS_READER_PORT
 
+READ_API = 'http://127.0.0.1:{}/headless/details'.format(HEADLESS_READER_PORT or 33843)
+READ_COMMENT__API = 'http://127.0.0.1:{}/headless/comments'.format(HEADLESS_READER_PORT or 33843)
+TIMEOUT = 90
 
 def get_html(url):
-    logging.info(f"Headless Browser Scraper: {url}")
+    logging.info(f"Headless Scraper: {url}")
     details = get_details(url)
     if not details:
         return ''

apiserver/scrapers/simple.py

@@ -3,9 +3,9 @@ logging.basicConfig(
         format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
         level=logging.DEBUG)
 import requests
-from settings import READER_PORT
+from settings import SIMPLE_READER_PORT
 
-READ_API = 'http://127.0.0.1:{}/simple/details'.format(READER_PORT or 3000)
+READ_API = 'http://127.0.0.1:{}/simple/details'.format(SIMPLE_READER_PORT or 33843)
 TIMEOUT = 20
 
 def get_html(url):

apiserver/settings.py.example

@@ -6,7 +6,8 @@ MAX_STORY_AGE = 3*24*60*60
 
 SCRAPERS = ['headless', 'outline', 'declutter', 'simple']
 API_PORT = 33842
-READER_PORT = 3000
+SIMPLE_READER_PORT = 33843
+HEADLESS_READER_PORT = 33843
 
 # Feed Lengths
 # Number of top items from each site to pull