jason/qotnews
1
0
Fork 0
forked from tanner/qotnews
Code Issues Pull Requests Releases Wiki Activity

renaming things.

Browse Source
This commit is contained in:
Jason Schwarzenberger
2020-11-17 15:50:31 +13:00
parent 55d50a86d8
commit 3b885e4327
13 changed files with 34 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
apiserver/scrapers/browser.py → apiserver/scrapers/headless.py
View File

@@ -4,13 +4,13 @@ logging.basicConfig(
level=logging.DEBUG)
import requests
READ_API = 'http://127.0.0.1:33843/browser/details'
READ_COMMENT__API = 'http://127.0.0.1:33843/browser/commentd'
READ_API = 'http://127.0.0.1:33843/headless/details'
READ_COMMENT__API = 'http://127.0.0.1:33843/headless/comments'
TIMEOUT = 60
def get_html(url):
logging.info(f"Reader Scraper: {url}")
logging.info(f"Headless Browser Scraper: {url}")
details = get_details(url)
if not details:
return ''
@@ -25,7 +25,7 @@ def get_details(url):
except KeyboardInterrupt:
raise
except BaseException as e:
logging.error('Problem Scraping article: {}'.format(str(e)))
logging.error('Problem scraping article: {}'.format(str(e)))
return None
def get_comments(url):

4
apiserver/scrapers/local.py → apiserver/scrapers/simple.py
View File

@@ -4,11 +4,11 @@ logging.basicConfig(
level=logging.DEBUG)
import requests
READ_API = 'http://127.0.0.1:33843/details'
READ_API = 'http://127.0.0.1:33843/simple/details'
TIMEOUT = 20
def get_html(url):
logging.info(f"Local Scraper: {url}")
logging.info(f"Simple Scraper: {url}")
details = get_details(url)
if not details:
return ''