forked from tanner/qotnews
Prototype readability server
This commit is contained in:
parent
a814411c12
commit
c0607b3fb6
48
readerserver/main.js
Normal file
48
readerserver/main.js
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
const express = require('express');
|
||||||
|
const app = express();
|
||||||
|
const port = 3000;
|
||||||
|
|
||||||
|
const request = require('request');
|
||||||
|
const JSDOM = require('jsdom').JSDOM;
|
||||||
|
const createDOMPurify = require('dompurify');
|
||||||
|
const Readability = require('readability');
|
||||||
|
|
||||||
|
app.use(express.urlencoded());
|
||||||
|
|
||||||
|
app.get('/', (req, res) => {
|
||||||
|
res.send('POST a URL to urlencoded \'url\' to parse.');
|
||||||
|
});
|
||||||
|
|
||||||
|
const requestCallback = (url, res) => (error, response, body) => {
|
||||||
|
if (!error && response.statusCode == 200) {
|
||||||
|
console.log('Response OK.');
|
||||||
|
|
||||||
|
const doc = new JSDOM('', {url: url});
|
||||||
|
const DOMPurify = createDOMPurify(doc.window);
|
||||||
|
const clean = DOMPurify.sanitize(body);
|
||||||
|
const cleanDoc = new JSDOM(clean, {url: url});
|
||||||
|
const reader = new Readability(cleanDoc.window.document);
|
||||||
|
const article = reader.parse();
|
||||||
|
|
||||||
|
res.send(article);
|
||||||
|
} else {
|
||||||
|
console.log('Response error:', error ? error.toString() : response.statusCode);
|
||||||
|
res.sendStatus(response ? response.statusCode : 404);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
app.post('/', (req, res) => {
|
||||||
|
const url = req.body.url;
|
||||||
|
const requestOptions = {
|
||||||
|
url: url,
|
||||||
|
headers: {'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)'},
|
||||||
|
};
|
||||||
|
|
||||||
|
console.log('Parse request for:', url);
|
||||||
|
|
||||||
|
request(requestOptions, requestCallback(url, res));
|
||||||
|
});
|
||||||
|
|
||||||
|
app.listen(port, () => {
|
||||||
|
console.log(`Example app listening on port ${port}!`);
|
||||||
|
});
|
13
readerserver/package.json
Normal file
13
readerserver/package.json
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
{
|
||||||
|
"name": "readerserver",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"main": "main.js",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"dompurify": "^1.0.11",
|
||||||
|
"express": "^4.17.1",
|
||||||
|
"jsdom": "^15.1.1",
|
||||||
|
"readability": "https://github.com/mozilla/readability",
|
||||||
|
"request": "^2.88.0"
|
||||||
|
}
|
||||||
|
}
|
1013
readerserver/yarn.lock
Normal file
1013
readerserver/yarn.lock
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user