forked from tanner/qotnews
Compare commits
No commits in common. "b80c1a5cb58e7746eca4cc1aeded8d06ee8e5817" and "3169af30026a2390ca4882e27f4d0b358ff2675b" have entirely different histories.
b80c1a5cb5
...
3169af3002
3
.gitmodules
vendored
3
.gitmodules
vendored
|
|
@ -1,3 +0,0 @@
|
|||
[submodule "readerserver/scraper/browser/scripts/bypass-paywalls-chrome"]
|
||||
path = readerserver/scraper/browser/scripts/bypass-paywalls-chrome
|
||||
url = https://github.com/iamadamdev/bypass-paywalls-chrome.git
|
||||
|
|
@ -10,7 +10,7 @@ import itertools
|
|||
|
||||
import settings
|
||||
from feeds import hackernews, reddit, tildes, substack, manual, news
|
||||
from scrapers import outline, declutter, browser, local
|
||||
from scrapers import outline, declutter, local
|
||||
|
||||
INVALID_DOMAINS = ['youtube.com', 'bloomberg.com', 'wsj.com']
|
||||
|
||||
|
|
@ -26,7 +26,7 @@ for key, value in settings.SITEMAP.items():
|
|||
|
||||
def get_list():
|
||||
feeds = {}
|
||||
|
||||
|
||||
if settings.NUM_HACKERNEWS:
|
||||
feeds['hackernews'] = [(x, 'hackernews') for x in hackernews.feed()[:settings.NUM_HACKERNEWS]]
|
||||
|
||||
|
|
@ -63,7 +63,6 @@ def get_article(url):
|
|||
scrapers = {
|
||||
'declutter': declutter,
|
||||
'outline': outline,
|
||||
'browser': browser,
|
||||
'local': local,
|
||||
}
|
||||
available = settings.SCRAPERS or ['local']
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ import extruct
|
|||
import pytz
|
||||
|
||||
from utils import clean
|
||||
import settings
|
||||
|
||||
tzinfos = {
|
||||
'NZDT': pytz.timezone('Pacific/Auckland'),
|
||||
|
|
@ -92,10 +91,7 @@ def parse_extruct(s, data):
|
|||
if props['datePublished']:
|
||||
s['date'] = props['datePublished']
|
||||
if 'author' in props and props['author']:
|
||||
if 'properties' in props['author']:
|
||||
s['author'] = props['author']['properties']['name']
|
||||
elif isinstance(props['author'], list):
|
||||
s['author'] = props['author'][0]['properties']['name']
|
||||
s['author'] = props['author']['properties']['name']
|
||||
|
||||
for ld in data['json-ld']:
|
||||
if '@type' in ld and ld['@type'] in ['Article', 'NewsArticle']:
|
||||
|
|
@ -105,10 +101,7 @@ def parse_extruct(s, data):
|
|||
if ld['datePublished']:
|
||||
s['date'] = ld['datePublished']
|
||||
if 'author' in ld and ld['author']:
|
||||
if 'name' in ld['author']:
|
||||
s['author'] = ld['author']['name']
|
||||
elif isinstance(ld['author'], list):
|
||||
s['author'] = ld['author'][0]['name']
|
||||
s['author'] = ld['author']['name']
|
||||
if '@graph' in ld:
|
||||
for gld in ld['@graph']:
|
||||
if '@type' in gld and gld['@type'] in ['Article', 'NewsArticle']:
|
||||
|
|
@ -206,102 +199,59 @@ class Sitemap(_Base):
|
|||
self.sitemap_url = url
|
||||
|
||||
def feed(self, excludes=None):
|
||||
links = []
|
||||
if isinstance(self.sitemap_url, str):
|
||||
links += self._get_sitemap(self.sitemap_url, excludes)
|
||||
elif isinstance(self.sitemap_url, list):
|
||||
for url in self.sitemap_url:
|
||||
links += self._get_sitemap(url, excludes)
|
||||
return list(set(links))
|
||||
markup = xml(lambda x: self.sitemap_url)
|
||||
if not markup: return []
|
||||
soup = BeautifulSoup(markup, features='lxml')
|
||||
sitemap = soup.find('urlset').findAll('url')
|
||||
|
||||
def _filter_links(self, links, excludes=None):
|
||||
too_old = datetime.now().timestamp() - settings.MAX_STORY_AGE
|
||||
links = list(filter(None, [a if a.find('loc') else None for a in sitemap]))
|
||||
links = list(filter(None, [a if get_sitemap_date(a) else None for a in links]))
|
||||
links = list(filter(None, [a if unix(get_sitemap_date(a)) > too_old else None for a in links]))
|
||||
links.sort(key=lambda a: unix(get_sitemap_date(a)), reverse=True)
|
||||
|
||||
links = [x.find('loc').text for x in links] or []
|
||||
links = list(set(links))
|
||||
if excludes:
|
||||
links = list(filter(None, [None if any(e in link for e in excludes) else link for link in links]))
|
||||
return links
|
||||
|
||||
def _get_sitemap(self, feed_url, excludes=None):
|
||||
markup = xml(lambda x: feed_url)
|
||||
if not markup: return []
|
||||
soup = BeautifulSoup(markup, features='lxml')
|
||||
links = []
|
||||
feed_urls = []
|
||||
if soup.find('sitemapindex'):
|
||||
sitemap = soup.find('sitemapindex').findAll('sitemap')
|
||||
feed_urls = list(filter(None, [a if a.find('loc') else None for a in sitemap]))
|
||||
if soup.find('urlset'):
|
||||
sitemap = soup.find('urlset').findAll('url')
|
||||
links = list(filter(None, [a if a.find('loc') else None for a in sitemap]))
|
||||
|
||||
feed_urls = self._filter_links(feed_urls, excludes)
|
||||
links = self._filter_links(links, excludes)
|
||||
|
||||
for url in feed_urls:
|
||||
links += self._get_sitemap(url, excludes)
|
||||
return list(set(links))
|
||||
|
||||
class Category(_Base):
|
||||
def __init__(self, url, tz=None):
|
||||
self.tz = tz
|
||||
self.category_url = url
|
||||
self.base_url = '/'.join(url.split('/')[:3])
|
||||
|
||||
def _filter_links(self, links, category_url, excludes=None):
|
||||
links = list(filter(None, [link if link.startswith(category_url) else None for link in links]))
|
||||
links = list(filter(None, [link if link != category_url else None for link in links]))
|
||||
def feed(self, excludes=None):
|
||||
markup = xml(lambda x: self.category_url)
|
||||
if not markup: return []
|
||||
soup = BeautifulSoup(markup, features='html.parser')
|
||||
links = soup.find_all('a', href=True)
|
||||
links = [link.get('href') for link in links]
|
||||
links = [f"{self.base_url}{link}" if link.startswith('/') else link for link in links]
|
||||
links = list(filter(None, [link if link.startswith(self.category_url) else None for link in links]))
|
||||
links = list(filter(None, [link if link != self.category_url else None for link in links]))
|
||||
links = list(set(links))
|
||||
if excludes:
|
||||
links = list(filter(None, [None if any(e in link for e in excludes) else link for link in links]))
|
||||
return links
|
||||
|
||||
def _get_category(self, category_url, excludes=None):
|
||||
base_url = '/'.join(category_url.split('/')[:3])
|
||||
markup = xml(lambda x: category_url)
|
||||
if not markup: return []
|
||||
soup = BeautifulSoup(markup, features='html.parser')
|
||||
links = soup.find_all('a', href=True)
|
||||
links = [link.get('href') for link in links]
|
||||
links = [f"{base_url}{link}" if link.startswith('/') else link for link in links]
|
||||
links = self._filter_links(links, category_url, excludes)
|
||||
return links
|
||||
|
||||
def feed(self, excludes=None):
|
||||
links = []
|
||||
if isinstance(self.category_url, str):
|
||||
links += self._get_category(self.category_url, excludes)
|
||||
elif isinstance(self.category_url, list):
|
||||
for url in self.category_url:
|
||||
links += self._get_category(url, excludes)
|
||||
return list(set(links))
|
||||
|
||||
|
||||
# scratchpad so I can quickly develop the parser
|
||||
if __name__ == '__main__':
|
||||
print("Sitemap: The Spinoff")
|
||||
site = Sitemap("https://thespinoff.co.nz/sitemap.xml")
|
||||
excludes = [
|
||||
'thespinoff.co.nz/sitemap-misc.xml',
|
||||
'thespinoff.co.nz/sitemap-authors.xml',
|
||||
'thespinoff.co.nz/sitemap-tax-category.xml',
|
||||
]
|
||||
posts = site.feed(excludes)
|
||||
print(posts[:5])
|
||||
print(site.story(posts[0]))
|
||||
|
||||
print("Sitemap: Newshub")
|
||||
site = Sitemap([
|
||||
'https://www.newshub.co.nz/home/politics.gnewssitemap.xml',
|
||||
'https://www.newshub.co.nz/home/new-zealand.gnewssitemap.xml',
|
||||
'https://www.newshub.co.nz/home/world.gnewssitemap.xml',
|
||||
'https://www.newshub.co.nz/home/money.gnewssitemap.xml',
|
||||
])
|
||||
print("Category: RadioNZ Te Ao Māori")
|
||||
site = Category("https://www.rnz.co.nz/news/te-manu-korihi/")
|
||||
posts = site.feed()
|
||||
print(posts[:5])
|
||||
print(site.story(posts[0]))
|
||||
|
||||
print("Sitemap: tvnz")
|
||||
site = Sitemap("https://www.tvnz.co.nz/system/tvnz/sitemap.xml")
|
||||
posts = site.feed()
|
||||
print(posts[:5])
|
||||
print(site.story(posts[0]))
|
||||
|
||||
print("Sitemap: Newsroom")
|
||||
site = Sitemap("https://www.newsroom.co.nz/sitemap.xml", tz='Pacific/Auckland')
|
||||
posts = site.feed()
|
||||
print(posts[:5])
|
||||
print(site.story(posts[0]))
|
||||
print(site.story(posts[:-1]))
|
||||
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ def story(ref):
|
|||
s['comments'] = list(filter(bool, s['comments']))
|
||||
s['num_comments'] = r.num_comments
|
||||
|
||||
if s['score'] < settings.REDDIT_SCORE_THRESHOLD and s['num_comments'] < settings.REDDIT_COMMENT_THRESHOLD:
|
||||
if s['score'] < 25 and s['num_comments'] < 10:
|
||||
return False
|
||||
|
||||
if r.selftext:
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
import logging
|
||||
logging.basicConfig(
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
level=logging.DEBUG)
|
||||
import requests
|
||||
|
||||
READ_API = 'http://127.0.0.1:33843/browser/details'
|
||||
READ_COMMENT__API = 'http://127.0.0.1:33843/browser/commentd'
|
||||
TIMEOUT = 60
|
||||
|
||||
|
||||
def get_html(url):
|
||||
logging.info(f"Reader Scraper: {url}")
|
||||
details = get_details(url)
|
||||
if not details:
|
||||
return ''
|
||||
return details['content']
|
||||
|
||||
def get_details(url):
|
||||
try:
|
||||
r = requests.post(READ_API, data=dict(url=url), timeout=TIMEOUT)
|
||||
if r.status_code != 200:
|
||||
raise Exception('Bad response code ' + str(r.status_code))
|
||||
return r.json()
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except BaseException as e:
|
||||
logging.error('Problem Scraping article: {}'.format(str(e)))
|
||||
return None
|
||||
|
||||
def get_comments(url):
|
||||
try:
|
||||
r = requests.post(READ_COMMENT_API, data=dict(url=url), timeout=TIMEOUT)
|
||||
if r.status_code != 200:
|
||||
raise Exception('Bad response code ' + str(r.status_code))
|
||||
return r.json()
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except BaseException as e:
|
||||
logging.error('Problem getting comments for article: {}'.format(str(e)))
|
||||
return None
|
||||
|
|
@ -23,7 +23,7 @@ SUBSTACK = {}
|
|||
CATEGORY = {}
|
||||
# CATEGORY['rnz national'] = { 'url': "https://www.rnz.co.nz/news/national", 'count': 10},
|
||||
|
||||
SCRAPERS = ['browser', 'declutter', 'outline', 'local']
|
||||
SCRAPERS = ['declutter', 'outline', 'local']
|
||||
|
||||
# Reddit account info
|
||||
# leave blank if not using Reddit
|
||||
|
|
@ -31,10 +31,6 @@ REDDIT_CLIENT_ID = ''
|
|||
REDDIT_CLIENT_SECRET = ''
|
||||
REDDIT_USER_AGENT = ''
|
||||
|
||||
# Minimum points or number of comments before including a thread:
|
||||
REDDIT_COMMENT_THRESHOLD = 10
|
||||
REDDIT_SCORE_THRESHOLD = 25
|
||||
|
||||
SUBREDDITS = [
|
||||
'Economics',
|
||||
'AcademicPhilosophy',
|
||||
|
|
|
|||
4
readerserver/constants.js
Normal file
4
readerserver/constants.js
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
module.exports.headers = {
|
||||
'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)',
|
||||
'X-Forwarded-For': '66.249.66.1',
|
||||
};
|
||||
|
|
@ -2,12 +2,12 @@ const port = 33843;
|
|||
const express = require('express');
|
||||
const app = express();
|
||||
const simple = require('./scraper/simple');
|
||||
const browser = require('./scraper/browser');
|
||||
|
||||
app.use(express.urlencoded({ extended: true }));
|
||||
|
||||
app.get('/', (req, res) => {
|
||||
const routes = ['/', '/details', '/browser', '/browser/details', '/browser/comments'];
|
||||
// const routes = ['/', '/details', '/browser', '/browser/details', '/browser/comments'];
|
||||
const routes = ['/', '/details'];
|
||||
|
||||
const html = routes.map(route => `
|
||||
<form method="POST" action="${route}" accept-charset="UTF-8">
|
||||
|
|
@ -21,9 +21,9 @@ app.get('/', (req, res) => {
|
|||
});
|
||||
app.post('/', simple.scrape);
|
||||
app.post('/details', simple.details);
|
||||
app.post('/browser', browser.scrape);
|
||||
app.post('/browser/details', browser.details);
|
||||
app.post('/browser/comments', browser.comments);
|
||||
// app.post('/browser', browser.scrape);
|
||||
// app.post('/browser/details', browser.details);
|
||||
// app.post('/browser/comments', browser.comments);
|
||||
|
||||
app.listen(port, () => {
|
||||
console.log(`Example app listening on port ${port}!`);
|
||||
|
|
|
|||
|
|
@ -4,12 +4,10 @@
|
|||
"main": "main.js",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mozilla/readability": "^0.3.0",
|
||||
"dompurify": "^1.0.11",
|
||||
"express": "^4.17.1",
|
||||
"jsdom": "^15.1.1",
|
||||
"node-fetch": "^2.6.1",
|
||||
"playwright": "^1.5.2",
|
||||
"readability": "https://github.com/mozilla/readability",
|
||||
"request": "^2.88.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
const { firefox } = require("playwright");
|
||||
const { JSDOM } = require("jsdom");
|
||||
const { Readability } = require("@mozilla/readability");
|
||||
|
||||
const { getUserAgent } = require('../../utils/user-agent');
|
||||
const { blockedRegexes, matchUrlDomain } = require("../../utils/sites");
|
||||
|
||||
module.exports.getDetails = async (url) => {
|
||||
const { userAgent, headers } = getUserAgent(url);
|
||||
|
||||
const browser = await firefox.launch({ args: [], headless: true });
|
||||
const tab = await browser.newPage({
|
||||
extraHTTPHeaders: headers,
|
||||
userAgent,
|
||||
viewport: { width: 2000, height: 10000 },
|
||||
});
|
||||
|
||||
try {
|
||||
await tab.route(/.*/, (route) => {
|
||||
const routeUrl = route.request().url();
|
||||
const blockedDomains = Object.keys(blockedRegexes);
|
||||
const domain = matchUrlDomain(blockedDomains, routeUrl);
|
||||
if (domain && routeUrl.match(blockedRegexes[domain])) {
|
||||
return route.abort();
|
||||
}
|
||||
return route.continue();
|
||||
});
|
||||
await tab.addInitScript({ path: "scraper/browser/scripts/bypass-paywalls-chrome/src/js/contentScript.js" });
|
||||
await tab.addInitScript({ path: "scraper/browser/scripts/cosmetic-filters.js" });
|
||||
await tab.addInitScript({ path: "scraper/browser/scripts/fix-relative-links.js" });
|
||||
await tab.goto(url, { timeout: 60000, waitUntil: "domcontentloaded" });
|
||||
await tab.waitForTimeout(2000);
|
||||
|
||||
const body = await tab.content();
|
||||
const doc = new JSDOM(body, { url });
|
||||
const reader = new Readability(doc.window.document);
|
||||
const article = reader.parse();
|
||||
return article;
|
||||
} catch (e) {
|
||||
throw e;
|
||||
} finally {
|
||||
await tab.close();
|
||||
await browser.close();
|
||||
}
|
||||
};
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
const { JSDOM } = require("jsdom");
|
||||
const { firefox } = require("playwright");
|
||||
const { getUserAgent } = require('../../utils/user-agent');
|
||||
const { disqusThread } = require('../../utils/disqus-thread');
|
||||
|
||||
const DISQUS_EMBED = 'https://disqus.com/embed/comments/';
|
||||
|
||||
module.exports.getComments = async (url) => {
|
||||
const { userAgent, headers } = getUserAgent(url);
|
||||
|
||||
const browser = await firefox.launch({ args: [], headless: true });
|
||||
const tab = await browser.newPage({
|
||||
extraHTTPHeaders: headers,
|
||||
userAgent,
|
||||
viewport: { width: 2000, height: 10000 },
|
||||
});
|
||||
|
||||
try {
|
||||
await tab.goto(url, { timeout: 60000, waitUntil: "domcontentloaded" });
|
||||
|
||||
const response = await tab.waitForResponse(response => response.url().includes(DISQUS_EMBED));
|
||||
const text = await response.text();
|
||||
const dom = new JSDOM(text, response.url());
|
||||
const script = dom.window.document.querySelector('#disqus-threadData')
|
||||
const data = JSON.parse(script.innerHTML);
|
||||
|
||||
return disqusThread(data);
|
||||
} catch (e) {
|
||||
throw e;
|
||||
} finally {
|
||||
await tab.close();
|
||||
await browser.close();
|
||||
}
|
||||
};
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
const { getDetails } = require('./_browser');
|
||||
const { getComments } = require('./_comments');
|
||||
|
||||
module.exports.scrape = async (req, res) => {
|
||||
try {
|
||||
const article = await getDetails(req.body.url);
|
||||
if (!article || !article.content) {
|
||||
throw new Error('failed to get details.');
|
||||
}
|
||||
return res.send(article.content);
|
||||
} catch (e) {
|
||||
return res.sendStatus(500);
|
||||
}
|
||||
};
|
||||
|
||||
module.exports.details = async (req, res) => {
|
||||
try {
|
||||
const article = await getDetails(req.body.url);
|
||||
if (!article) {
|
||||
throw new Error('failed to get details.');
|
||||
}
|
||||
return res.send(article);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
return res.sendStatus(500);
|
||||
}
|
||||
};
|
||||
|
||||
module.exports.comments = async (req, res) => {
|
||||
try {
|
||||
const comments = await getComments(req.body.url);
|
||||
if (!comments) {
|
||||
throw new Error('failed to get comments.');
|
||||
}
|
||||
return res.send(comments);
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
return res.sendStatus(500);
|
||||
}
|
||||
};
|
||||
|
|
@ -1 +0,0 @@
|
|||
Subproject commit 44f3d1b114400d73aba2bf2551a34f9f142eda76
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
(function () {
|
||||
removeHiddenElements();
|
||||
|
||||
if (matchDomain("stuff.co.nz")) {
|
||||
removeSelectors([
|
||||
".support-brief-container",
|
||||
'[class*="donation-in-"]',
|
||||
".sics-component__sharebar",
|
||||
".breaking-news-pointer",
|
||||
".bigbyline-container",
|
||||
[
|
||||
".sics-component__html-injector.sics-component__story__paragraph",
|
||||
"READ MORE:",
|
||||
],
|
||||
]);
|
||||
}
|
||||
if (matchDomain("nzherald.co.nz")) {
|
||||
removeSelectors([
|
||||
"[href$='#commenting-widget']",
|
||||
".related-articles",
|
||||
".article__print-button",
|
||||
".share-bar",
|
||||
".c-suggest-links.read-more-links",
|
||||
".website-of-year",
|
||||
".meta-data",
|
||||
".article__kicker",
|
||||
".author__image",
|
||||
]);
|
||||
}
|
||||
if (matchDomain(["rnz.co.nz", "radionz.co.nz"])) {
|
||||
removeSelectors([".c-advert-app", ".c-sub-nav"]);
|
||||
}
|
||||
if (matchDomain(["newsroom.co.nz"])) {
|
||||
removeSelectors([".article_content__section", ".bio"]);
|
||||
}
|
||||
if (matchDomain(["newshub.co.nz"])) {
|
||||
removeSelectors([".c-ArticleHeading-authorPicture", ".relatedarticles"]);
|
||||
}
|
||||
if (matchDomain(["tvnz.co.nz"])) {
|
||||
removeSelectors([".signup-container container"]);
|
||||
}
|
||||
|
||||
function matchDomain(domains) {
|
||||
const hostname = window.location.hostname;
|
||||
if (typeof domains === "string") {
|
||||
domains = [domains];
|
||||
}
|
||||
return domains.some(
|
||||
(domain) => hostname === domain || hostname.endsWith("." + domain)
|
||||
);
|
||||
}
|
||||
|
||||
function removeDOMElement(...elements) {
|
||||
for (const element of elements) {
|
||||
if (element) {
|
||||
element.remove();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function pageContains(selector, text) {
|
||||
const elements = document.querySelectorAll(selector);
|
||||
return Array.prototype.filter.call(elements, function (element) {
|
||||
return RegExp(text).test(element.textContent);
|
||||
});
|
||||
}
|
||||
|
||||
function removeHiddenElements() {
|
||||
window.setTimeout(function () {
|
||||
const selector = "*:not(script):not(head):not(meta):not(link):not(style)";
|
||||
Array.from(document.querySelectorAll(selector))
|
||||
.filter((element) => {
|
||||
const computed = getComputedStyle(element);
|
||||
const displayNone = computed["display"] === "none";
|
||||
const visibilityHidden = computed["visibility"] === "hidden";
|
||||
return displayNone || visibilityHidden;
|
||||
})
|
||||
.forEach((element) => element && element.remove());
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
function removeSelectors(selectors) {
|
||||
window.setTimeout(function () {
|
||||
const elements = selectors.flatMap((s) => {
|
||||
if (typeof s === "string") {
|
||||
return Array.from(document.querySelectorAll(s));
|
||||
}
|
||||
if (s && s.constructor.name === "Array") {
|
||||
return pageContains(...s);
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
removeDOMElement(...elements);
|
||||
}, 1000);
|
||||
}
|
||||
})();
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
(function () {
|
||||
const { host, protocol } = window.location;
|
||||
const url = `${protocol}//${host}`;
|
||||
[
|
||||
['[src^="/"]', 'src'],
|
||||
['[href^="/"]', 'href']
|
||||
].forEach(([selector, attribute]) => {
|
||||
Array.from(document.querySelectorAll(selector))
|
||||
.filter(e => e.attributes[attribute] && /^\/[^\/]/.test(e.attributes[attribute].value))
|
||||
.forEach((e) => {
|
||||
e.attributes[attribute].value = `${url}${e.attributes[attribute].value}`;
|
||||
});
|
||||
});
|
||||
})();
|
||||
|
|
@ -1,8 +1,13 @@
|
|||
const fetch = require('node-fetch');
|
||||
const { JSDOM } = require('jsdom');
|
||||
const { Readability } = require('@mozilla/readability');
|
||||
const request = require('request');
|
||||
const JSDOM = require('jsdom').JSDOM;
|
||||
const { Readability } = require('readability');
|
||||
|
||||
const { getUserAgent } = require('../utils/user-agent');
|
||||
const { headers } = require('../constants');
|
||||
|
||||
const options = url => ({
|
||||
url,
|
||||
headers,
|
||||
});
|
||||
|
||||
const extract = (url, body) => {
|
||||
const doc = new JSDOM(body, { url: url });
|
||||
|
|
@ -10,50 +15,27 @@ const extract = (url, body) => {
|
|||
return reader.parse();
|
||||
};
|
||||
|
||||
module.exports.scrape = async (req, res) => {
|
||||
try {
|
||||
const { userAgent, headers } = getUserAgent(req.body.url);
|
||||
const response = await fetch(req.body.url, {
|
||||
headers: {
|
||||
...headers,
|
||||
'User-Agent': userAgent
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
return res.sendStatus(response.statusCode);
|
||||
}
|
||||
const html = await response.text();
|
||||
const article = await extract(req.body.url, html);
|
||||
if (article && article.content) {
|
||||
return res.send(article.content);
|
||||
}
|
||||
return res.sendStatus(404);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
return res.sendStatus(500);
|
||||
}
|
||||
};
|
||||
|
||||
module.exports.details = async (req, res) => {
|
||||
try {
|
||||
const { userAgent, headers } = getUserAgent(req.body.url);
|
||||
const response = await fetch(req.body.url, {
|
||||
headers: {
|
||||
...headers,
|
||||
'User-Agent': userAgent
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
return res.sendStatus(response.statusCode);
|
||||
}
|
||||
const html = await response.text();
|
||||
const article = await extract(req.body.url, html);
|
||||
if (article) {
|
||||
return res.send(article);
|
||||
}
|
||||
return res.sendStatus(404);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
return res.sendStatus(500);
|
||||
module.exports.scrape = (req, res) => request(options(req.body.url), (error, response, body) => {
|
||||
if (error || response.statusCode != 200) {
|
||||
console.log('Response error:', error ? error.toString() : response.statusCode);
|
||||
return res.sendStatus(response ? response.statusCode : 404);
|
||||
}
|
||||
};
|
||||
const article = extract(req.body.url, body);
|
||||
if (article && article.content) {
|
||||
return res.send(article.content);
|
||||
}
|
||||
return res.sendStatus(404);
|
||||
});
|
||||
|
||||
module.exports.details = (req, res) => request(options(req.body.url), (error, response, body) => {
|
||||
if (error || response.statusCode != 200) {
|
||||
console.log('Response error:', error ? error.toString() : response.statusCode);
|
||||
return res.sendStatus(response ? response.statusCode : 404);
|
||||
}
|
||||
const article = extract(req.body.url, body);
|
||||
if (article) {
|
||||
return res.send(article);
|
||||
}
|
||||
return res.sendStatus(404);
|
||||
});
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
const googleBotUserAgent = 'Googlebot/2.1 (+http://www.google.com/bot.html)';
|
||||
const googleBotIp = '66.249.66.1';
|
||||
|
||||
module.exports.googleBot = {
|
||||
userAgent: googleBotUserAgent,
|
||||
ip: googleBotIp,
|
||||
headers: {
|
||||
'User-Agent': googleBotUserAgent,
|
||||
'X-Forwarded-For': googleBotIp,
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
module.exports.disqusThread = data => {
|
||||
const comments = data.response.posts.reduce((c, post) => ({
|
||||
...c,
|
||||
[post.id.toString()]: {
|
||||
author: post.author.name,
|
||||
authorLink: post.author.profileUrl,
|
||||
date: post.createdAt,
|
||||
text: post.raw_message,
|
||||
score: post.points,
|
||||
children: [],
|
||||
id: post.id.toString(),
|
||||
parent: (post.parent || '').toString(),
|
||||
}
|
||||
}), {});
|
||||
Object.keys(comments).filter(id => !!comments[id].parent).forEach(id => {
|
||||
const comment = comments[id];
|
||||
comments[comment.parent].children.push(comment);
|
||||
});
|
||||
const parents = Object.keys(comments).filter(id => comments[id].parent).map(id => comments[id]);
|
||||
return parents;
|
||||
};
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
module.exports.blockedRegexes = {
|
||||
"adweek.com": /.+\.lightboxcdn\.com\/.+/,
|
||||
"afr.com": /afr\.com\/assets\/vendorsReactRedux_client.+\.js/,
|
||||
"businessinsider.com": /(.+\.tinypass\.com\/.+|cdn\.onesignal\.com\/sdks\/.+\.js)/,
|
||||
"chicagotribune.com": /.+:\/\/.+\.tribdss\.com\//,
|
||||
"economist.com": /(.+\.tinypass\.com\/.+|economist\.com\/engassets\/_next\/static\/chunks\/framework.+\.js)/,
|
||||
"editorialedomani.it": /(js\.pelcro\.com\/.+|editorialedomani.it\/pelcro\.js)/,
|
||||
"foreignpolicy.com": /.+\.tinypass\.com\/.+/,
|
||||
"fortune.com": /.+\.tinypass\.com\/.+/,
|
||||
"haaretz.co.il": /haaretz\.co\.il\/htz\/js\/inter\.js/,
|
||||
"haaretz.com": /haaretz\.com\/hdc\/web\/js\/minified\/header-scripts-int.js.+/,
|
||||
"inquirer.com": /.+\.tinypass\.com\/.+/,
|
||||
"lastampa.it": /.+\.repstatic\.it\/minify\/sites\/lastampa\/.+\/config\.cache\.php\?name=social_js/,
|
||||
"lrb.co.uk": /.+\.tinypass\.com\/.+/,
|
||||
"nzherald.co.nz": /(.+nzherald\.co\.nz\/.+\/subs\/p\.js|.+nzherald\.co\.nz\/.+\/react\.js|.+nzherald\.co\.nz\/.+\/appear\.js|.+nzherald\.co\.nz\/.+\/tracking\/.+|.+nzherald\.co\.nz\/.+\/default\.js|.+\/newsbarscript\.js)/,
|
||||
"medscape.com": /.+\.medscapestatic\.com\/.*medscape-library\.js/,
|
||||
"interest.co.nz": /(.+\.presspatron\.com.+|.+interest\.co\.nz.+pp-ablock-banner\.js)/,
|
||||
"repubblica.it": /scripts\.repubblica\.it\/pw\/pw\.js.+/,
|
||||
"spectator.co.uk": /.+\.tinypass\.com\/.+/,
|
||||
"spectator.com.au": /.+\.tinypass\.com\/.+/,
|
||||
"telegraph.co.uk": /.+telegraph\.co\.uk.+martech.+/,
|
||||
"thecourier.com.au": /.+cdn-au\.piano\.io\/api\/tinypass.+\.js/,
|
||||
"thenation.com": /thenation\.com\/.+\/paywall-script\.php/,
|
||||
"thenational.scot": /(.+\.tinypass\.com\/.+|.+thenational\.scot.+omniture\.js|.+thenational\.scot.+responsive-sync.+)/,
|
||||
"thewrap.com": /thewrap\.com\/.+\/wallkit\.js/,
|
||||
"wsj.com": /cdn\.ampproject\.org\/v\d\/amp-access-.+\.js/,
|
||||
"historyextra.com": /.+\.evolok\.net\/.+\/authorize\/.+/,
|
||||
"barrons.com": /cdn\.ampproject\.org\/v\d\/amp-access-.+\.js/,
|
||||
"irishtimes.com": /cdn\.ampproject\.org\/v\d\/amp-access-.+\.js/,
|
||||
"elmercurio.com": /(merreader\.emol\.cl\/assets\/js\/merPramV2.js|staticmer\.emol\.cl\/js\/inversiones\/PramModal.+\.js)/,
|
||||
"sloanreview.mit.edu": /(.+\.tinypass\.com\/.+|.+\.netdna-ssl\.com\/wp-content\/themes\/smr\/assets\/js\/libs\/welcome-ad\.js)/,
|
||||
"latercera.com": /.+\.cxense\.com\/+/,
|
||||
"lesechos.fr": /.+\.tinypass\.com\/.+/,
|
||||
"washingtonpost.com": /.+\.washingtonpost\.com\/.+\/pwapi-proxy\.min\.js/,
|
||||
"thehindu.com": /ajax\.cloudflare\.com\/cdn-cgi\/scripts\/.+\/cloudflare-static\/rocket-loader\.min\.js/,
|
||||
"technologyreview.com": /.+\.blueconic\.net\/.+/,
|
||||
};
|
||||
|
||||
module.exports.useGoogleBotSites = [
|
||||
"adelaidenow.com.au",
|
||||
"barrons.com",
|
||||
"couriermail.com.au",
|
||||
"dailytelegraph.com.au",
|
||||
"fd.nl",
|
||||
"genomeweb.com",
|
||||
"haaretz.co.il",
|
||||
"haaretz.com",
|
||||
"heraldsun.com.au",
|
||||
"mexiconewsdaily.com",
|
||||
"ntnews.com.au",
|
||||
"quora.com",
|
||||
"seekingalpha.com",
|
||||
"telegraph.co.uk",
|
||||
"theaustralian.com.au",
|
||||
"themarker.com",
|
||||
"themercury.com.au",
|
||||
"thenational.scot",
|
||||
"thetimes.co.uk",
|
||||
"wsj.com",
|
||||
"kansascity.com",
|
||||
"republic.ru",
|
||||
"nzz.ch",
|
||||
"handelsblatt.com",
|
||||
"washingtonpost.com",
|
||||
"df.cl",
|
||||
];
|
||||
|
||||
function matchDomain(domains, hostname) {
|
||||
let matchedDomain = false;
|
||||
if (typeof domains === "string") {
|
||||
domains = [domains];
|
||||
}
|
||||
domains.some(
|
||||
(domain) =>
|
||||
(hostname === domain || hostname.endsWith("." + domain)) &&
|
||||
(matchedDomain = domain)
|
||||
);
|
||||
return matchedDomain;
|
||||
}
|
||||
|
||||
function matchUrlDomain(domains, url) {
|
||||
return matchDomain(domains, urlHost(url));
|
||||
}
|
||||
|
||||
function urlHost(url) {
|
||||
if (url && url.startsWith("http")) {
|
||||
try {
|
||||
return new URL(url).hostname;
|
||||
} catch (e) {
|
||||
console.log(`url not valid: ${url} error: ${e}`);
|
||||
}
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
module.exports.matchDomain = matchDomain;
|
||||
module.exports.matchUrlDomain = matchUrlDomain;
|
||||
module.exports.urlHost = urlHost;
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
const { googleBot } = require('./constants');
|
||||
const { matchUrlDomain, useGoogleBotSites } = require("./sites");
|
||||
|
||||
module.exports.getUserAgent = (url) => {
|
||||
const useGoogleBot = useGoogleBotSites.some(function (item) {
|
||||
return typeof item === "string" && matchUrlDomain(item, url);
|
||||
});
|
||||
|
||||
if (!useGoogleBot) {
|
||||
return {};
|
||||
}
|
||||
return {
|
||||
userAgent: googleBot.userAgent,
|
||||
headers: {
|
||||
"X-Forwarded-For": googleBot.ip
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -2,23 +2,6 @@
|
|||
# yarn lockfile v1
|
||||
|
||||
|
||||
"@mozilla/readability@^0.3.0":
|
||||
version "0.3.0"
|
||||
resolved "https://registry.yarnpkg.com/@mozilla/readability/-/readability-0.3.0.tgz#a473790e8b42ea39f9f03703fa1f17fec7984b60"
|
||||
integrity sha512-q8f1CAZsRKK1j+O0BmikGIlKSK03RpT4woT0PCQwhw0nH0z4+rG026AkxoPcjT7Dsgh1ifGscW8tOpvjoyOjvw==
|
||||
|
||||
"@types/node@*":
|
||||
version "14.14.7"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.7.tgz#8ea1e8f8eae2430cf440564b98c6dfce1ec5945d"
|
||||
integrity sha512-Zw1vhUSQZYw+7u5dAwNbIA9TuTotpzY/OF7sJM9FqPOF3SPjKnxrjoTktXDZgUjybf4cWVBP7O8wvKdSaGHweg==
|
||||
|
||||
"@types/yauzl@^2.9.1":
|
||||
version "2.9.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.9.1.tgz#d10f69f9f522eef3cf98e30afb684a1e1ec923af"
|
||||
integrity sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
abab@^2.0.0:
|
||||
version "2.0.0"
|
||||
resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.0.tgz#aba0ab4c5eee2d4c79d3487d85450fb2376ebb0f"
|
||||
|
|
@ -50,13 +33,6 @@ acorn@^6.0.1, acorn@^6.1.1:
|
|||
resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.3.0.tgz#0087509119ffa4fc0a0041d1e93a417e68cb856e"
|
||||
integrity sha512-/czfa8BwS88b9gWQVhc8eknunSA2DoJpJyTQkhheIf5E48u1N0R4q/YxxsAeqRrmK9TQ/uYfgLDfZo91UlANIA==
|
||||
|
||||
agent-base@6:
|
||||
version "6.0.2"
|
||||
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"
|
||||
integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==
|
||||
dependencies:
|
||||
debug "4"
|
||||
|
||||
ajv@^6.5.5:
|
||||
version "6.10.2"
|
||||
resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.10.2.tgz#d3cea04d6b017b2894ad69040fec8b623eb4bd52"
|
||||
|
|
@ -109,11 +85,6 @@ aws4@^1.8.0:
|
|||
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
|
||||
integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==
|
||||
|
||||
balanced-match@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
|
||||
integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
|
||||
|
||||
bcrypt-pbkdf@^1.0.0:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"
|
||||
|
|
@ -137,24 +108,11 @@ body-parser@1.19.0:
|
|||
raw-body "2.4.0"
|
||||
type-is "~1.6.17"
|
||||
|
||||
brace-expansion@^1.1.7:
|
||||
version "1.1.11"
|
||||
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
|
||||
integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==
|
||||
dependencies:
|
||||
balanced-match "^1.0.0"
|
||||
concat-map "0.0.1"
|
||||
|
||||
browser-process-hrtime@^0.1.2:
|
||||
version "0.1.3"
|
||||
resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-0.1.3.tgz#616f00faef1df7ec1b5bf9cfe2bdc3170f26c7b4"
|
||||
integrity sha512-bRFnI4NnjO6cnyLmOV/7PVoDEMJChlcfN0z4s1YMBY989/SvlfMI1lgCnkFUs53e9gQF+w7qu7XdllSTiSl8Aw==
|
||||
|
||||
buffer-crc32@~0.2.3:
|
||||
version "0.2.13"
|
||||
resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
|
||||
integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=
|
||||
|
||||
bytes@3.1.0:
|
||||
version "3.1.0"
|
||||
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
|
||||
|
|
@ -172,11 +130,6 @@ combined-stream@^1.0.6, combined-stream@~1.0.6:
|
|||
dependencies:
|
||||
delayed-stream "~1.0.0"
|
||||
|
||||
concat-map@0.0.1:
|
||||
version "0.0.1"
|
||||
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
|
||||
integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=
|
||||
|
||||
content-disposition@0.5.3:
|
||||
version "0.5.3"
|
||||
resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd"
|
||||
|
|
@ -239,13 +192,6 @@ debug@2.6.9:
|
|||
dependencies:
|
||||
ms "2.0.0"
|
||||
|
||||
debug@4, debug@^4.1.1:
|
||||
version "4.2.0"
|
||||
resolved "https://registry.yarnpkg.com/debug/-/debug-4.2.0.tgz#7f150f93920e94c58f5574c2fd01a3110effe7f1"
|
||||
integrity sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==
|
||||
dependencies:
|
||||
ms "2.1.2"
|
||||
|
||||
deep-is@~0.1.3:
|
||||
version "0.1.3"
|
||||
resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
|
||||
|
|
@ -296,13 +242,6 @@ encodeurl@~1.0.2:
|
|||
resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
|
||||
integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=
|
||||
|
||||
end-of-stream@^1.1.0:
|
||||
version "1.4.4"
|
||||
resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
|
||||
integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
|
||||
dependencies:
|
||||
once "^1.4.0"
|
||||
|
||||
escape-html@~1.0.3:
|
||||
version "1.0.3"
|
||||
resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
|
||||
|
|
@ -381,17 +320,6 @@ extend@~3.0.2:
|
|||
resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa"
|
||||
integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==
|
||||
|
||||
extract-zip@^2.0.1:
|
||||
version "2.0.1"
|
||||
resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a"
|
||||
integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==
|
||||
dependencies:
|
||||
debug "^4.1.1"
|
||||
get-stream "^5.1.0"
|
||||
yauzl "^2.10.0"
|
||||
optionalDependencies:
|
||||
"@types/yauzl" "^2.9.1"
|
||||
|
||||
extsprintf@1.3.0:
|
||||
version "1.3.0"
|
||||
resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05"
|
||||
|
|
@ -417,13 +345,6 @@ fast-levenshtein@~2.0.4:
|
|||
resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
|
||||
integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=
|
||||
|
||||
fd-slicer@~1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e"
|
||||
integrity sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=
|
||||
dependencies:
|
||||
pend "~1.2.0"
|
||||
|
||||
finalhandler@~1.1.2:
|
||||
version "1.1.2"
|
||||
resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d"
|
||||
|
|
@ -461,18 +382,6 @@ fresh@0.5.2:
|
|||
resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7"
|
||||
integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=
|
||||
|
||||
fs.realpath@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
|
||||
integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=
|
||||
|
||||
get-stream@^5.1.0:
|
||||
version "5.2.0"
|
||||
resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3"
|
||||
integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==
|
||||
dependencies:
|
||||
pump "^3.0.0"
|
||||
|
||||
getpass@^0.1.1:
|
||||
version "0.1.7"
|
||||
resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa"
|
||||
|
|
@ -480,23 +389,6 @@ getpass@^0.1.1:
|
|||
dependencies:
|
||||
assert-plus "^1.0.0"
|
||||
|
||||
glob@^7.1.3:
|
||||
version "7.1.6"
|
||||
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6"
|
||||
integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==
|
||||
dependencies:
|
||||
fs.realpath "^1.0.0"
|
||||
inflight "^1.0.4"
|
||||
inherits "2"
|
||||
minimatch "^3.0.4"
|
||||
once "^1.3.0"
|
||||
path-is-absolute "^1.0.0"
|
||||
|
||||
graceful-fs@^4.1.11:
|
||||
version "4.2.4"
|
||||
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb"
|
||||
integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==
|
||||
|
||||
har-schema@^2.0.0:
|
||||
version "2.0.0"
|
||||
resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
|
||||
|
|
@ -548,14 +440,6 @@ http-signature@~1.2.0:
|
|||
jsprim "^1.2.2"
|
||||
sshpk "^1.7.0"
|
||||
|
||||
https-proxy-agent@^5.0.0:
|
||||
version "5.0.0"
|
||||
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2"
|
||||
integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==
|
||||
dependencies:
|
||||
agent-base "6"
|
||||
debug "4"
|
||||
|
||||
iconv-lite@0.4.24:
|
||||
version "0.4.24"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
|
||||
|
|
@ -563,24 +447,16 @@ iconv-lite@0.4.24:
|
|||
dependencies:
|
||||
safer-buffer ">= 2.1.2 < 3"
|
||||
|
||||
inflight@^1.0.4:
|
||||
version "1.0.6"
|
||||
resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9"
|
||||
integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=
|
||||
dependencies:
|
||||
once "^1.3.0"
|
||||
wrappy "1"
|
||||
|
||||
inherits@2, inherits@2.0.4:
|
||||
version "2.0.4"
|
||||
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
|
||||
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
|
||||
|
||||
inherits@2.0.3:
|
||||
version "2.0.3"
|
||||
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
|
||||
integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
|
||||
|
||||
inherits@2.0.4:
|
||||
version "2.0.4"
|
||||
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
|
||||
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
|
||||
|
||||
ip-regex@^2.1.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
|
||||
|
|
@ -601,11 +477,6 @@ isstream@~0.1.2:
|
|||
resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
|
||||
integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=
|
||||
|
||||
jpeg-js@^0.4.2:
|
||||
version "0.4.2"
|
||||
resolved "https://registry.yarnpkg.com/jpeg-js/-/jpeg-js-0.4.2.tgz#8b345b1ae4abde64c2da2fe67ea216a114ac279d"
|
||||
integrity sha512-+az2gi/hvex7eLTMTlbRLOhH6P6WFdk2ITI8HJsaH2VqYO0I594zXSYEP+tf4FW+8Cy68ScDXoAsQdyQanv3sw==
|
||||
|
||||
jsbn@~0.1.0:
|
||||
version "0.1.1"
|
||||
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
|
||||
|
|
@ -718,18 +589,6 @@ mime@1.6.0:
|
|||
resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
|
||||
integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
|
||||
|
||||
mime@^2.4.6:
|
||||
version "2.4.6"
|
||||
resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.6.tgz#e5b407c90db442f2beb5b162373d07b69affa4d1"
|
||||
integrity sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==
|
||||
|
||||
minimatch@^3.0.4:
|
||||
version "3.0.4"
|
||||
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
|
||||
integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==
|
||||
dependencies:
|
||||
brace-expansion "^1.1.7"
|
||||
|
||||
ms@2.0.0:
|
||||
version "2.0.0"
|
||||
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
|
||||
|
|
@ -740,21 +599,11 @@ ms@2.1.1:
|
|||
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a"
|
||||
integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==
|
||||
|
||||
ms@2.1.2:
|
||||
version "2.1.2"
|
||||
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
|
||||
integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
|
||||
|
||||
negotiator@0.6.2:
|
||||
version "0.6.2"
|
||||
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
|
||||
integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==
|
||||
|
||||
node-fetch@^2.6.1:
|
||||
version "2.6.1"
|
||||
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
|
||||
integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
|
||||
|
||||
nwsapi@^2.1.4:
|
||||
version "2.1.4"
|
||||
resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.1.4.tgz#e006a878db23636f8e8a67d33ca0e4edf61a842f"
|
||||
|
|
@ -772,13 +621,6 @@ on-finished@~2.3.0:
|
|||
dependencies:
|
||||
ee-first "1.1.1"
|
||||
|
||||
once@^1.3.0, once@^1.3.1, once@^1.4.0:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
|
||||
integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E=
|
||||
dependencies:
|
||||
wrappy "1"
|
||||
|
||||
optionator@^0.8.1:
|
||||
version "0.8.2"
|
||||
resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.2.tgz#364c5e409d3f4d6301d6c0b4c05bba50180aeb64"
|
||||
|
|
@ -801,72 +643,26 @@ parseurl@~1.3.3:
|
|||
resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4"
|
||||
integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==
|
||||
|
||||
path-is-absolute@^1.0.0:
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
|
||||
integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18=
|
||||
|
||||
path-to-regexp@0.1.7:
|
||||
version "0.1.7"
|
||||
resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c"
|
||||
integrity sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=
|
||||
|
||||
pend@~1.2.0:
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
|
||||
integrity sha1-elfrVQpng/kRUzH89GY9XI4AelA=
|
||||
|
||||
performance-now@^2.1.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
|
||||
integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
|
||||
|
||||
playwright@^1.5.2:
|
||||
version "1.5.2"
|
||||
resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.5.2.tgz#e127142cba86c918fad9f68315db5e79524af64c"
|
||||
integrity sha512-on7IEui47bDZta0txL86QKMDSgjbxERkLc5N0+lU2zajIfN/Ld6vMl+xiROEUPlT/QtqVekq9pTDGdcc0yScMQ==
|
||||
dependencies:
|
||||
debug "^4.1.1"
|
||||
extract-zip "^2.0.1"
|
||||
https-proxy-agent "^5.0.0"
|
||||
jpeg-js "^0.4.2"
|
||||
mime "^2.4.6"
|
||||
pngjs "^5.0.0"
|
||||
progress "^2.0.3"
|
||||
proper-lockfile "^4.1.1"
|
||||
proxy-from-env "^1.1.0"
|
||||
rimraf "^3.0.2"
|
||||
ws "^7.3.1"
|
||||
|
||||
pn@^1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/pn/-/pn-1.1.0.tgz#e2f4cef0e219f463c179ab37463e4e1ecdccbafb"
|
||||
integrity sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==
|
||||
|
||||
pngjs@^5.0.0:
|
||||
version "5.0.0"
|
||||
resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-5.0.0.tgz#e79dd2b215767fd9c04561c01236df960bce7fbb"
|
||||
integrity sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw==
|
||||
|
||||
prelude-ls@~1.1.2:
|
||||
version "1.1.2"
|
||||
resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"
|
||||
integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=
|
||||
|
||||
progress@^2.0.3:
|
||||
version "2.0.3"
|
||||
resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
|
||||
integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
|
||||
|
||||
proper-lockfile@^4.1.1:
|
||||
version "4.1.1"
|
||||
resolved "https://registry.yarnpkg.com/proper-lockfile/-/proper-lockfile-4.1.1.tgz#284cf9db9e30a90e647afad69deb7cb06881262c"
|
||||
integrity sha512-1w6rxXodisVpn7QYvLk706mzprPTAPCYAqxMvctmPN3ekuRk/kuGkGc82pangZiAt4R3lwSuUzheTTn0/Yb7Zg==
|
||||
dependencies:
|
||||
graceful-fs "^4.1.11"
|
||||
retry "^0.12.0"
|
||||
signal-exit "^3.0.2"
|
||||
|
||||
proxy-addr@~2.0.5:
|
||||
version "2.0.5"
|
||||
resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.5.tgz#34cbd64a2d81f4b1fd21e76f9f06c8a45299ee34"
|
||||
|
|
@ -875,24 +671,11 @@ proxy-addr@~2.0.5:
|
|||
forwarded "~0.1.2"
|
||||
ipaddr.js "1.9.0"
|
||||
|
||||
proxy-from-env@^1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
|
||||
integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
|
||||
|
||||
psl@^1.1.24, psl@^1.1.28:
|
||||
version "1.3.0"
|
||||
resolved "https://registry.yarnpkg.com/psl/-/psl-1.3.0.tgz#e1ebf6a3b5564fa8376f3da2275da76d875ca1bd"
|
||||
integrity sha512-avHdspHO+9rQTLbv1RO+MPYeP/SzsCoxofjVnHanETfQhTJrmB0HlDoW+EiN/R+C0BZ+gERab9NY0lPN2TxNag==
|
||||
|
||||
pump@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
|
||||
integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==
|
||||
dependencies:
|
||||
end-of-stream "^1.1.0"
|
||||
once "^1.3.1"
|
||||
|
||||
punycode@^1.4.1:
|
||||
version "1.4.1"
|
||||
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
|
||||
|
|
@ -928,6 +711,10 @@ raw-body@2.4.0:
|
|||
iconv-lite "0.4.24"
|
||||
unpipe "1.0.0"
|
||||
|
||||
"readability@https://github.com/mozilla/readability":
|
||||
version "0.3.0"
|
||||
resolved "https://github.com/mozilla/readability#d5eea06a0095b3138dbd1f6233f656d690200509"
|
||||
|
||||
request-promise-core@1.1.2:
|
||||
version "1.1.2"
|
||||
resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.2.tgz#339f6aababcafdb31c799ff158700336301d3346"
|
||||
|
|
@ -970,18 +757,6 @@ request@^2.88.0:
|
|||
tunnel-agent "^0.6.0"
|
||||
uuid "^3.3.2"
|
||||
|
||||
retry@^0.12.0:
|
||||
version "0.12.0"
|
||||
resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b"
|
||||
integrity sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=
|
||||
|
||||
rimraf@^3.0.2:
|
||||
version "3.0.2"
|
||||
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a"
|
||||
integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==
|
||||
dependencies:
|
||||
glob "^7.1.3"
|
||||
|
||||
safe-buffer@5.1.2:
|
||||
version "5.1.2"
|
||||
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
|
||||
|
|
@ -1038,11 +813,6 @@ setprototypeof@1.1.1:
|
|||
resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683"
|
||||
integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==
|
||||
|
||||
signal-exit@^3.0.2:
|
||||
version "3.0.3"
|
||||
resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c"
|
||||
integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==
|
||||
|
||||
source-map@~0.6.1:
|
||||
version "0.6.1"
|
||||
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
|
||||
|
|
@ -1225,11 +995,6 @@ wordwrap@~1.0.0:
|
|||
resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb"
|
||||
integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=
|
||||
|
||||
wrappy@1:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
|
||||
integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
|
||||
|
||||
ws@^7.0.0:
|
||||
version "7.1.2"
|
||||
resolved "https://registry.yarnpkg.com/ws/-/ws-7.1.2.tgz#c672d1629de8bb27a9699eb599be47aeeedd8f73"
|
||||
|
|
@ -1237,11 +1002,6 @@ ws@^7.0.0:
|
|||
dependencies:
|
||||
async-limiter "^1.0.0"
|
||||
|
||||
ws@^7.3.1:
|
||||
version "7.4.0"
|
||||
resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.0.tgz#a5dd76a24197940d4a8bb9e0e152bb4503764da7"
|
||||
integrity sha512-kyFwXuV/5ymf+IXhS6f0+eAFvydbaBW3zjpT6hUdAh/hbVjTIB5EHBGi0bPoCLSK2wcuz3BrEkB9LrYv1Nm4NQ==
|
||||
|
||||
xml-name-validator@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
|
||||
|
|
@ -1251,11 +1011,3 @@ xmlchars@^2.1.1:
|
|||
version "2.1.1"
|
||||
resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.1.1.tgz#ef1a81c05bff629c2280007f12daca21bd6f6c93"
|
||||
integrity sha512-7hew1RPJ1iIuje/Y01bGD/mXokXxegAgVS+e+E0wSi2ILHQkYAH1+JXARwTjZSM4Z4Z+c73aKspEcqj+zPPL/w==
|
||||
|
||||
yauzl@^2.10.0:
|
||||
version "2.10.0"
|
||||
resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"
|
||||
integrity sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=
|
||||
dependencies:
|
||||
buffer-crc32 "~0.2.3"
|
||||
fd-slicer "~1.1.0"
|
||||
|
|
|
|||
|
|
@ -5,14 +5,13 @@ import './Style-light.css';
|
|||
import './Style-dark.css';
|
||||
import './fonts/Fonts.css';
|
||||
import { ForwardDot } from './utils.js';
|
||||
import Feed from './Feed.js';
|
||||
import Article from './Article.js';
|
||||
import Comments from './Comments.js';
|
||||
import Search from './Search.js';
|
||||
import Submit from './Submit.js';
|
||||
import Results from './Results.js';
|
||||
import ScrollToTop from './ScrollToTop.js';
|
||||
import Feed from './pages/Feed.js';
|
||||
import Article from './pages/Article.js';
|
||||
import Comments from './pages/Comments.js';
|
||||
import Results from './pages/Results.js';
|
||||
|
||||
|
||||
class App extends React.Component {
|
||||
constructor(props) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import React from 'react';
|
||||
import { Helmet } from 'react-helmet';
|
||||
import localForage from 'localforage';
|
||||
import { sourceLink, infoLine, ToggleDot } from '../utils.js';
|
||||
import { sourceLink, infoLine, ToggleDot } from './utils.js';
|
||||
|
||||
class Article extends React.Component {
|
||||
constructor(props) {
|
||||
|
|
@ -18,7 +18,7 @@ class Article extends React.Component {
|
|||
pConv: [],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
componentDidMount() {
|
||||
const id = this.props.match ? this.props.match.params.id : 'CLOL';
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ class Article extends React.Component {
|
|||
}
|
||||
|
||||
pConvert = (n) => {
|
||||
this.setState({ pConv: [...this.state.pConv, n] });
|
||||
this.setState({ pConv: [...this.state.pConv, n]});
|
||||
}
|
||||
|
||||
render() {
|
||||
|
|
@ -85,10 +85,10 @@ class Article extends React.Component {
|
|||
v.innerHTML.split('\n\n').map(x =>
|
||||
<p dangerouslySetInnerHTML={{ __html: x }} />
|
||||
)
|
||||
:
|
||||
:
|
||||
(v.nodeName === '#text' ?
|
||||
<p>{v.data}</p>
|
||||
:
|
||||
:
|
||||
<>
|
||||
<v.localName dangerouslySetInnerHTML={v.innerHTML ? { __html: v.innerHTML } : null} />
|
||||
{v.localName == 'pre' && <button onClick={() => this.pConvert(k)}>Convert Code to Paragraph</button>}
|
||||
|
|
@ -96,11 +96,11 @@ class Article extends React.Component {
|
|||
)
|
||||
)}
|
||||
</div>
|
||||
:
|
||||
:
|
||||
<p>Problem getting article :(</p>
|
||||
}
|
||||
</div>
|
||||
:
|
||||
:
|
||||
<p>loading...</p>
|
||||
}
|
||||
<ToggleDot id={id} article={false} />
|
||||
|
|
@ -4,7 +4,7 @@ import { HashLink } from 'react-router-hash-link';
|
|||
import { Helmet } from 'react-helmet';
|
||||
import moment from 'moment';
|
||||
import localForage from 'localforage';
|
||||
import { infoLine, ToggleDot } from '../utils.js';
|
||||
import { infoLine, ToggleDot } from './utils.js';
|
||||
|
||||
class Article extends React.Component {
|
||||
constructor(props) {
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
import React from 'react';
|
||||
import { Link } from 'react-router-dom';
|
||||
import { Helmet } from 'react-helmet';
|
||||
import localForage from 'localforage';
|
||||
import { StoryItem } from '../components/StoryItem.js';
|
||||
import { sourceLink, infoLine, getLogoUrl } from './utils.js';
|
||||
|
||||
class Feed extends React.Component {
|
||||
constructor(props) {
|
||||
|
|
@ -55,7 +56,27 @@ class Feed extends React.Component {
|
|||
<title>Feed - QotNews</title>
|
||||
</Helmet>
|
||||
{error && <p>Connection error?</p>}
|
||||
{stories ? stories.map(story => <StoryItem story={story}></StoryItem>) : <p>loading...</p>}
|
||||
{stories ?
|
||||
<div>
|
||||
{stories.map(x =>
|
||||
<div className='item' key={x.id}>
|
||||
<div className='title'>
|
||||
<Link className='link' to={'/' + x.id}>
|
||||
<img className='source-logo' src={getLogoUrl(x)} alt='source logo' /> {x.title}
|
||||
</Link>
|
||||
|
||||
<span className='source'>
|
||||
({sourceLink(x)})
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{infoLine(x)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
:
|
||||
<p>loading...</p>
|
||||
}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
import React from 'react';
|
||||
import { Link } from 'react-router-dom';
|
||||
import { Helmet } from 'react-helmet';
|
||||
import { sourceLink, infoLine, getLogoUrl } from './utils.js';
|
||||
import AbortController from 'abort-controller';
|
||||
import { StoryItem } from '../components/StoryItem.js';
|
||||
|
||||
class Results extends React.Component {
|
||||
constructor(props) {
|
||||
|
|
@ -62,7 +63,25 @@ class Results extends React.Component {
|
|||
<>
|
||||
<p>Search results:</p>
|
||||
<div className='comment lined'>
|
||||
{stories ? stories.map(story => <StoryItem story={story}></StoryItem>) : <p>loading...</p>}
|
||||
{stories.length ?
|
||||
stories.map(x =>
|
||||
<div className='item' key={x.id}>
|
||||
<div className='title'>
|
||||
<Link className='link' to={'/' + x.id}>
|
||||
<img className='source-logo' src={getLogoUrl(x)} alt='source logo' /> {x.title}
|
||||
</Link>
|
||||
|
||||
<span className='source'>
|
||||
({sourceLink(x)})
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{infoLine(x)}
|
||||
</div>
|
||||
)
|
||||
:
|
||||
<p>none</p>
|
||||
}
|
||||
</div>
|
||||
</>
|
||||
:
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
import React from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
import { sourceLink, infoLine, getLogoUrl } from "../utils.js";
|
||||
|
||||
export class StoryItem extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
}
|
||||
|
||||
render() {
|
||||
const story = this.props.story;
|
||||
const { id, title } = story;
|
||||
|
||||
return (
|
||||
<div className="item" key={id}>
|
||||
<div className="title">
|
||||
<Link className="link" to={"/" + id}>
|
||||
<img
|
||||
className="source-logo"
|
||||
src={getLogoUrl(story)}
|
||||
alt="source logo"
|
||||
/>
|
||||
{" "}
|
||||
{title}
|
||||
</Link>
|
||||
|
||||
<span className="source">({sourceLink(story)})</span>
|
||||
</div>
|
||||
|
||||
{infoLine(story)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user