|
|
@ -2,12 +2,11 @@ const request = require('request'); |
|
|
|
const JSDOM = require('jsdom').JSDOM; |
|
|
|
const JSDOM = require('jsdom').JSDOM; |
|
|
|
const { Readability } = require('readability'); |
|
|
|
const { Readability } = require('readability'); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const { headers } = require('../constants'); |
|
|
|
|
|
|
|
|
|
|
|
const options = url => ({ |
|
|
|
const options = url => ({ |
|
|
|
url: url, |
|
|
|
url, |
|
|
|
headers: { |
|
|
|
headers, |
|
|
|
'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)', |
|
|
|
|
|
|
|
'X-Forwarded-For': '66.249.66.1', |
|
|
|
|
|
|
|
}, |
|
|
|
|
|
|
|
}); |
|
|
|
}); |
|
|
|
|
|
|
|
|
|
|
|
const extract = (url, body) => { |
|
|
|
const extract = (url, body) => { |
|
|
@ -17,13 +16,12 @@ const extract = (url, body) => { |
|
|
|
}; |
|
|
|
}; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
module.exports.FORM = '<form method="POST" action="/" accept-charset="UTF-8"><input name="url"><button type="submit">SUBMIT</button></form>'; |
|
|
|
|
|
|
|
module.exports.scrape = (req, res) => request(options(req.body.url), (error, response, body) => { |
|
|
|
module.exports.scrape = (req, res) => request(options(req.body.url), (error, response, body) => { |
|
|
|
if (error || response.statusCode != 200) { |
|
|
|
if (error || response.statusCode != 200) { |
|
|
|
console.log('Response error:', error ? error.toString() : response.statusCode); |
|
|
|
console.log('Response error:', error ? error.toString() : response.statusCode); |
|
|
|
return res.sendStatus(response ? response.statusCode : 404); |
|
|
|
return res.sendStatus(response ? response.statusCode : 404); |
|
|
|
} |
|
|
|
} |
|
|
|
const article = extract(url, body); |
|
|
|
const article = extract(req.body.url, body); |
|
|
|
if (article && article.content) { |
|
|
|
if (article && article.content) { |
|
|
|
return res.send(article.content); |
|
|
|
return res.send(article.content); |
|
|
|
} |
|
|
|
} |
|
|
@ -35,7 +33,7 @@ module.exports.details = (req, res) => request(options(req.body.url), (error, re |
|
|
|
console.log('Response error:', error ? error.toString() : response.statusCode); |
|
|
|
console.log('Response error:', error ? error.toString() : response.statusCode); |
|
|
|
return res.sendStatus(response ? response.statusCode : 404); |
|
|
|
return res.sendStatus(response ? response.statusCode : 404); |
|
|
|
} |
|
|
|
} |
|
|
|
const article = extract(url, body); |
|
|
|
const article = extract(req.body.url, body); |
|
|
|
if (article) { |
|
|
|
if (article) { |
|
|
|
return res.send(article); |
|
|
|
return res.send(article); |
|
|
|
} |
|
|
|
} |