jason/qotnews
1
0
Fork 0
forked from tanner/qotnews
Code Issues Pull Requests Releases Wiki Activity

port separation.

Browse Source
This commit is contained in:
Jason Schwarzenberger 2020-11-18 17:21:37 +13:00
parent 2f730c1f52
commit 539350a83d
5 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apiserver/scrapers/declutter.py
View File

@ -4,9 +4,9 @@ logging.basicConfig(
level=logging.DEBUG) level=logging.DEBUG)
import requests import requests
DECLUTTER_API = 'https://declutter.1j.nz/details' DECLUTTER_API = 'https://declutter.1j.nz/headless/details'
DECLUTTER_COMMENT_API = 'https://declutter.1j.nz/comments' DECLUTTER_COMMENT_API = 'https://declutter.1j.nz/headless/comments'
TIMEOUT = 30 TIMEOUT = 90
def get_html(url): def get_html(url):

11
apiserver/scrapers/headless.py
View File

@ -3,15 +3,14 @@ logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.DEBUG) level=logging.DEBUG)
import requests import requests
from settings import READER_PORT from settings import HEADLESS_READER_PORT
READ_API = 'http://127.0.0.1:{}/headless/details'.format(READER_PORT or 3000)
READ_COMMENT__API = 'http://127.0.0.1:{}/headless/comments'.format(READER_PORT or 3000)
TIMEOUT = 60
READ_API = 'http://127.0.0.1:{}/headless/details'.format(HEADLESS_READER_PORT or 33843)
READ_COMMENT__API = 'http://127.0.0.1:{}/headless/comments'.format(HEADLESS_READER_PORT or 33843)
TIMEOUT = 90
def get_html(url): def get_html(url):
logging.info(f"Headless Browser Scraper: {url}") logging.info(f"Headless Scraper: {url}")
details = get_details(url) details = get_details(url)
if not details: if not details:
return '' return ''

4
apiserver/scrapers/simple.py
View File

@ -3,9 +3,9 @@ logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.DEBUG) level=logging.DEBUG)
import requests import requests
from settings import READER_PORT from settings import SIMPLE_READER_PORT
READ_API = 'http://127.0.0.1:{}/simple/details'.format(READER_PORT or 3000) READ_API = 'http://127.0.0.1:{}/simple/details'.format(SIMPLE_READER_PORT or 33843)
TIMEOUT = 20 TIMEOUT = 20
def get_html(url): def get_html(url):

3
apiserver/settings.py.example
View File

@ -6,7 +6,8 @@ MAX_STORY_AGE = 3*24*60*60
SCRAPERS = ['headless', 'outline', 'declutter', 'simple'] SCRAPERS = ['headless', 'outline', 'declutter', 'simple']
API_PORT = 33842 API_PORT = 33842
READER_PORT = 3000 SIMPLE_READER_PORT = 33843
HEADLESS_READER_PORT = 33843
# Feed Lengths # Feed Lengths
# Number of top items from each site to pull # Number of top items from each site to pull

2
readerserver

@ -1 +1 @@
Subproject commit 50a94df7283e31680c5d94dd666bab58aea2e475 Subproject commit b8f2d401a79380fecc063c4184f49433b5900ee4