forked from tanner/qotnews
parent
33e21e7f30
commit
9edc8b7cca
6 changed files with 159 additions and 71 deletions
@ -0,0 +1,28 @@ |
||||
import logging |
||||
logging.basicConfig( |
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', |
||||
level=logging.DEBUG) |
||||
import requests |
||||
|
||||
DECLUTTER_API = 'https://declutter.1j.nz/details' |
||||
|
||||
|
||||
def get_html(url): |
||||
try: |
||||
logging.info(f'Declutter Scraper: {url}') |
||||
details = get_details(url) |
||||
return details['content'] |
||||
except: |
||||
raise |
||||
|
||||
def get_details(url): |
||||
try: |
||||
r = requests.post(DECLUTTER_API, data=dict(url=url), timeout=20) |
||||
if r.status_code != 200: |
||||
raise Exception('Bad response code ' + str(r.status_code)) |
||||
return r.json() |
||||
except KeyboardInterrupt: |
||||
raise |
||||
except BaseException as e: |
||||
logging.error('Problem decluttering article: {}'.format(str(e))) |
||||
return {} |
@ -0,0 +1,28 @@ |
||||
import logging |
||||
logging.basicConfig( |
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', |
||||
level=logging.DEBUG) |
||||
import requests |
||||
|
||||
READ_API = 'http://127.0.0.1:33843/details' |
||||
|
||||
|
||||
def get_html(url): |
||||
try: |
||||
logging.info(f'Local Scraper: {url}') |
||||
details = get_details(url) |
||||
return details['content'] |
||||
except: |
||||
raise |
||||
|
||||
def get_details(url): |
||||
try: |
||||
r = requests.post(READ_API, data=dict(url=url), timeout=20) |
||||
if r.status_code != 200: |
||||
raise Exception('Bad response code ' + str(r.status_code)) |
||||
return r.json() |
||||
except KeyboardInterrupt: |
||||
raise |
||||
except BaseException as e: |
||||
logging.error('Problem getting article: {}'.format(str(e))) |
||||
return {} |
@ -0,0 +1,38 @@ |
||||
import logging |
||||
logging.basicConfig( |
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', |
||||
level=logging.DEBUG) |
||||
import requests |
||||
|
||||
OUTLINE_REFERER = 'https://outline.com/' |
||||
OUTLINE_API = 'https://api.outline.com/v3/parse_article' |
||||
|
||||
|
||||
def get_html(url): |
||||
try: |
||||
details = get_details(url) |
||||
return details['html'] |
||||
except: |
||||
raise |
||||
|
||||
def get_details(url): |
||||
try: |
||||
logging.info(f'Outline Scraper: {url}') |
||||
params = {'source_url': url} |
||||
headers = {'Referer': OUTLINE_REFERER} |
||||
r = requests.get(OUTLINE_API, params=params, headers=headers, timeout=20) |
||||
if r.status_code == 429: |
||||
logging.info('Rate limited by outline, sleeping 30s and skipping...') |
||||
time.sleep(30) |
||||
return '' |
||||
if r.status_code != 200: |
||||
raise Exception('Bad response code ' + str(r.status_code)) |
||||
data = r.json()['data'] |
||||
if 'URL is not supported by Outline' in data['html']: |
||||
raise Exception('URL not supported by Outline') |
||||
return data |
||||
except KeyboardInterrupt: |
||||
raise |
||||
except BaseException as e: |
||||
logging.error('Problem outlining article: {}'.format(str(e))) |
||||
return {} |
@ -0,0 +1,43 @@ |
||||
const request = require('request'); |
||||
const JSDOM = require('jsdom').JSDOM; |
||||
const { Readability } = require('readability'); |
||||
|
||||
const options = url => ({ |
||||
url: url, |
||||
headers: { |
||||
'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)', |
||||
'X-Forwarded-For': '66.249.66.1', |
||||
}, |
||||
}); |
||||
|
||||
const extract = (url, body) => { |
||||
const doc = new JSDOM(body, { url: url }); |
||||
const reader = new Readability(doc.window.document); |
||||
return reader.parse(); |
||||
}; |
||||
|
||||
|
||||
module.exports.FORM = '<form method="POST" action="/" accept-charset="UTF-8"><input name="url"><button type="submit">SUBMIT</button></form>'; |
||||
module.exports.scrape = (req, res) => request(options(req.body.url), (error, response, body) => { |
||||
if (error || response.statusCode != 200) { |
||||
console.log('Response error:', error ? error.toString() : response.statusCode); |
||||
return res.sendStatus(response ? response.statusCode : 404); |
||||
} |
||||
const article = extract(url, body); |
||||
if (article && article.content) { |
||||
return res.send(article.content); |
||||
} |
||||
return res.sendStatus(404); |
||||
}); |
||||
|
||||
module.exports.details = (req, res) => request(options(req.body.url), (error, response, body) => { |
||||
if (error || response.statusCode != 200) { |
||||
console.log('Response error:', error ? error.toString() : response.statusCode); |
||||
return res.sendStatus(response ? response.statusCode : 404); |
||||
} |
||||
const article = extract(url, body); |
||||
if (article) { |
||||
return res.send(article); |
||||
} |
||||
return res.sendStatus(404); |
||||
}); |
Loading…
Reference in new issue