tanner/qotnews
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Change reader server useragent and port

Browse Source
This commit is contained in:
Tanner Collin 2019-08-23 08:21:25 +00:00
parent c0607b3fb6
commit 771c3987ec
1 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
readerserver/main.js
View File

@ -1,16 +1,16 @@
const express = require('express'); const express = require('express');
const app = express(); const app = express();
const port = 3000; const port = 33843;
const request = require('request'); const request = require('request');
const JSDOM = require('jsdom').JSDOM; const JSDOM = require('jsdom').JSDOM;
const createDOMPurify = require('dompurify'); const createDOMPurify = require('dompurify');
const Readability = require('readability'); const Readability = require('readability');
app.use(express.urlencoded()); app.use(express.urlencoded({ extended: true }));
app.get('/', (req, res) => { app.get('/', (req, res) => {
res.send('POST a URL to urlencoded \'url\' to parse.'); res.send('<form method="POST" accept-charset="UTF-8"><input name="url"><button type="submit">SUBMIT</button></form>');
}); });
const requestCallback = (url, res) => (error, response, body) => { const requestCallback = (url, res) => (error, response, body) => {
@ -24,7 +24,11 @@ const requestCallback = (url, res) => (error, response, body) => {
const reader = new Readability(cleanDoc.window.document); const reader = new Readability(cleanDoc.window.document);
const article = reader.parse(); const article = reader.parse();
res.send(article); if (article && article.content) {
res.send(article.content);
} else {
res.sendStatus(404);
}
} else { } else {
console.log('Response error:', error ? error.toString() : response.statusCode); console.log('Response error:', error ? error.toString() : response.statusCode);
res.sendStatus(response ? response.statusCode : 404); res.sendStatus(response ? response.statusCode : 404);
@ -35,7 +39,8 @@ app.post('/', (req, res) => {
const url = req.body.url; const url = req.body.url;
const requestOptions = { const requestOptions = {
url: url, url: url,
headers: {'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)'}, //headers: {'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)'},
headers: {'User-Agent': 'Twitterbot/1.0'},
}; };
console.log('Parse request for:', url); console.log('Parse request for:', url);