renaming things.

This commit is contained in:
Jason Schwarzenberger
2020-11-17 15:50:31 +13:00
parent 55d50a86d8
commit 3b885e4327
13 changed files with 34 additions and 28 deletions

View File

@@ -2,12 +2,18 @@ const port = 33843;
const express = require('express');
const app = express();
const simple = require('./scraper/simple');
const browser = require('./scraper/browser');
const headless = require('./scraper/headless');
app.use(express.urlencoded({ extended: true }));
app.get('/', (req, res) => {
const routes = ['/', '/details', '/browser', '/browser/details', '/browser/comments'];
const routes = [
'/simple',
'/simple/details',
'/headless',
'/headless/details',
'/headless/comments'
];
const html = routes.map(route => `
<form method="POST" action="${route}" accept-charset="UTF-8">
@@ -19,11 +25,11 @@ app.get('/', (req, res) => {
</form>`).join('<hr />');
res.send(html);
});
app.post('/', simple.scrape);
app.post('/details', simple.details);
app.post('/browser', browser.scrape);
app.post('/browser/details', browser.details);
app.post('/browser/comments', browser.comments);
app.post('/simple/', simple.scrape);
app.post('/simple/details', simple.details);
app.post('/headless', headless.scrape);
app.post('/headless/details', headless.details);
app.post('/headless/comments', headless.comments);
app.listen(port, () => {
console.log(`Example app listening on port ${port}!`);

View File

@@ -25,9 +25,9 @@ module.exports.getDetails = async (url) => {
}
return route.continue();
});
await tab.addInitScript({ path: "scraper/browser/scripts/bypass-paywalls-chrome/src/js/contentScript.js" });
await tab.addInitScript({ path: "scraper/browser/scripts/cosmetic-filters.js" });
await tab.addInitScript({ path: "scraper/browser/scripts/fix-relative-links.js" });
await tab.addInitScript({ path: "scraper/headless/scripts/bypass-paywalls-chrome/src/js/contentScript.js" });
await tab.addInitScript({ path: "scraper/headless/scripts/cosmetic-filters.js" });
await tab.addInitScript({ path: "scraper/headless/scripts/fix-relative-links.js" });
await tab.goto(url, { timeout: 60000, waitUntil: "domcontentloaded" });
await tab.waitForTimeout(2000);