apiserver/database.py

@@ -83,7 +83,7 @@ def get_stories(maxage=60*60*24*2):
     q = session.query(Reflist, Story.meta).\
             join(Story).\
             filter(Story.title != None).\
-            filter(Story.meta['date'].as_integer() > time).\
+            filter(Story.meta['date'] > time).\
             order_by(Story.meta['date'].desc())
     return [x[1] for x in q]
 

apiserver/feeds/news.py

@@ -139,19 +139,6 @@ class _Base:
         s['url'] = ref
         s['date'] = 0
 
-        soup = BeautifulSoup(markup, features='html.parser')
-        icon32 = soup.find_all('link', rel="icon", href=True, sizes="32x32")
-        icon16 = soup.find_all('link', rel="icon", href=True, sizes="16x16")
-        favicon = soup.find_all('link', rel="shortcut icon", href=True)
-        others = soup.find_all('link', rel="icon", href=True)
-        icons = icon32 + icon16 + favicon + others
-        base_url = '/'.join(ref.split('/')[:3])
-        icons = list(set([i.get('href') for i in icons]))
-        icons = [i if i.startswith('http') else base_url + i for i in icons]
-
-        if icons:
-            s['icon'] = icons[0]
-
         data = extruct.extract(markup)
         s = parse_extruct(s, data)
         if s['date']:

readerserver/constants.js

@@ -1,4 +0,0 @@
-module.exports.headers = {
-	'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)',
-	'X-Forwarded-For': '66.249.66.1',
-};

readerserver/main.js

@@ -1,29 +1,14 @@
 const port = 33843;
 const express = require('express');
 const app = express();
-const simple = require('./scraper/simple');
+const simple = require('./simple');
 
 app.use(express.urlencoded({ extended: true }));
-
-app.get('/', (req, res) => {
-	// const routes = ['/', '/details', '/browser', '/browser/details', '/browser/comments'];
-	const routes = ['/', '/details'];
-
-	const html = routes.map(route => `
-	<form method="POST" action="${route}" accept-charset="UTF-8">
-		<fieldset>
-			<legend>route: POST ${route}</legend>
-			<input name="url">
-			<button type="submit">SUBMIT</button>
-		</fieldset>
-	</form>`).join('<hr />');
-	res.send(html);
-});
-app.post('/', simple.scrape);
-app.post('/details', simple.details);
-// app.post('/browser', browser.scrape);
-// app.post('/browser/details', browser.details);
-// app.post('/browser/comments', browser.comments);
+app.get('/', (req, res) => res.send(simple.FORM));
+app.post('/', (req, res) => simple.scrape(req, res));
+app.post('/details', (req, res) => simple.details(req, res));
+// app.post('/browser', (req, res) => browser.scrape(req, res));
+// app.post('/browser/details', (req, res) => browser.details(req, res));
 
 app.listen(port, () => {
 	console.log(`Example app listening on port ${port}!`);

readerserver/simple.js

@@ -2,11 +2,12 @@ const request = require('request');
 const JSDOM = require('jsdom').JSDOM;
 const { Readability } = require('readability');
 
-const { headers } = require('../constants');
-
 const options = url => ({
-	url,
-	headers,
+	url: url,
+	headers: {
+		'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)',
+		'X-Forwarded-For': '66.249.66.1',
+	},
 });
 
 const extract = (url, body) => {
@@ -16,12 +17,13 @@ const extract = (url, body) => {
 };
 
 
+module.exports.FORM = '<form method="POST" action="/" accept-charset="UTF-8"><input name="url"><button type="submit">SUBMIT</button></form>';
 module.exports.scrape = (req, res) => request(options(req.body.url), (error, response, body) => {
 	if (error || response.statusCode != 200) {
 		console.log('Response error:', error ? error.toString() : response.statusCode);
 		return res.sendStatus(response ? response.statusCode : 404);
 	}
-	const article = extract(req.body.url, body);
+	const article = extract(url, body);
 	if (article && article.content) {
 		return res.send(article.content);
 	}
@@ -33,7 +35,7 @@ module.exports.details = (req, res) => request(options(req.body.url), (error, re
 		console.log('Response error:', error ? error.toString() : response.statusCode);
 		return res.sendStatus(response ? response.statusCode : 404);
 	}
-	const article = extract(req.body.url, body);
+	const article = extract(url, body);
 	if (article) {
 		return res.send(article);
 	}

webclient/src/Feed.js

@@ -2,7 +2,7 @@ import React from 'react';
 import { Link } from 'react-router-dom';
 import { Helmet } from 'react-helmet';
 import localForage from 'localforage';
-import { sourceLink, infoLine, getLogoUrl } from './utils.js';
+import { sourceLink, infoLine, logos } from './utils.js';
 
 class Feed extends React.Component {
 	constructor(props) {
@@ -62,7 +62,7 @@ class Feed extends React.Component {
 							<div className='item' key={x.id}>
 								<div className='title'>
 									<Link className='link' to={'/' + x.id}>
-										<img className='source-logo' src={getLogoUrl(x)} alt='source logo' /> {x.title}
+										<img className='source-logo' src={logos[x.source] || logos[x.source.split(' ')[0]]} alt='source logo' /> {x.title}
 									</Link>
 
 									<span className='source'>

webclient/src/Results.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import { Link } from 'react-router-dom';
 import { Helmet } from 'react-helmet';
-import { sourceLink, infoLine, getLogoUrl } from './utils.js';
+import { sourceLink, infoLine, logos } from './utils.js';
 import AbortController from 'abort-controller';
 
 class Results extends React.Component {
@@ -68,7 +68,7 @@ class Results extends React.Component {
 									<div className='item' key={x.id}>
 										<div className='title'>
 											<Link className='link' to={'/' + x.id}>
-												<img className='source-logo' src={getLogoUrl(x)} alt='source logo' /> {x.title}
+												<img className='source-logo' src={logos[x.source]} alt='source logo' /> {x.title}
 											</Link>
 
 											<span className='source'>