reader server

This commit is contained in:
Jason Schwarzenberger 2020-11-10 14:56:21 +13:00
parent 9cee370a25
commit 943a1cfa4f
3 changed files with 31 additions and 14 deletions

View File

@ -0,0 +1,4 @@
module.exports.headers = {
'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)',
'X-Forwarded-For': '66.249.66.1',
};

View File

@ -1,14 +1,29 @@
const port = 33843; const port = 33843;
const express = require('express'); const express = require('express');
const app = express(); const app = express();
const simple = require('./simple'); const simple = require('./scraper/simple');
app.use(express.urlencoded({ extended: true })); app.use(express.urlencoded({ extended: true }));
app.get('/', (req, res) => res.send(simple.FORM));
app.post('/', (req, res) => simple.scrape(req, res)); app.get('/', (req, res) => {
app.post('/details', (req, res) => simple.details(req, res)); // const routes = ['/', '/details', '/browser', '/browser/details', '/browser/comments'];
// app.post('/browser', (req, res) => browser.scrape(req, res)); const routes = ['/', '/details'];
// app.post('/browser/details', (req, res) => browser.details(req, res));
const html = routes.map(route => `
<form method="POST" action="${route}" accept-charset="UTF-8">
<fieldset>
<legend>route: POST ${route}</legend>
<input name="url">
<button type="submit">SUBMIT</button>
</fieldset>
</form>`).join('<hr />');
res.send(html);
});
app.post('/', simple.scrape);
app.post('/details', simple.details);
// app.post('/browser', browser.scrape);
// app.post('/browser/details', browser.details);
// app.post('/browser/comments', browser.comments);
app.listen(port, () => { app.listen(port, () => {
console.log(`Example app listening on port ${port}!`); console.log(`Example app listening on port ${port}!`);

View File

@ -2,12 +2,11 @@ const request = require('request');
const JSDOM = require('jsdom').JSDOM; const JSDOM = require('jsdom').JSDOM;
const { Readability } = require('readability'); const { Readability } = require('readability');
const { headers } = require('../constants');
const options = url => ({ const options = url => ({
url: url, url,
headers: { headers,
'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)',
'X-Forwarded-For': '66.249.66.1',
},
}); });
const extract = (url, body) => { const extract = (url, body) => {
@ -17,13 +16,12 @@ const extract = (url, body) => {
}; };
module.exports.FORM = '<form method="POST" action="/" accept-charset="UTF-8"><input name="url"><button type="submit">SUBMIT</button></form>';
module.exports.scrape = (req, res) => request(options(req.body.url), (error, response, body) => { module.exports.scrape = (req, res) => request(options(req.body.url), (error, response, body) => {
if (error || response.statusCode != 200) { if (error || response.statusCode != 200) {
console.log('Response error:', error ? error.toString() : response.statusCode); console.log('Response error:', error ? error.toString() : response.statusCode);
return res.sendStatus(response ? response.statusCode : 404); return res.sendStatus(response ? response.statusCode : 404);
} }
const article = extract(url, body); const article = extract(req.body.url, body);
if (article && article.content) { if (article && article.content) {
return res.send(article.content); return res.send(article.content);
} }
@ -35,7 +33,7 @@ module.exports.details = (req, res) => request(options(req.body.url), (error, re
console.log('Response error:', error ? error.toString() : response.statusCode); console.log('Response error:', error ? error.toString() : response.statusCode);
return res.sendStatus(response ? response.statusCode : 404); return res.sendStatus(response ? response.statusCode : 404);
} }
const article = extract(url, body); const article = extract(req.body.url, body);
if (article) { if (article) {
return res.send(article); return res.send(article);
} }