From c9da2a078bbe57093aa5b2b7f23c89080b374684 Mon Sep 17 00:00:00 2001 From: Jason Schwarzenberger Date: Wed, 18 Nov 2020 10:06:45 +1300 Subject: [PATCH] increase setTimeouts. --- .gitmodules | 1 + readerserver/scraper/headless/_browser.js | 2 +- readerserver/scraper/headless/scripts/bypass-paywalls-chrome | 2 +- readerserver/scraper/headless/scripts/cosmetic-filters.js | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index be8a66a..46f9335 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "readerserver/scraper/headless/scripts/bypass-paywalls-chrome"] path = readerserver/scraper/headless/scripts/bypass-paywalls-chrome url = https://github.com/iamadamdev/bypass-paywalls-chrome/ + branch = master \ No newline at end of file diff --git a/readerserver/scraper/headless/_browser.js b/readerserver/scraper/headless/_browser.js index 308134c..c9e3661 100644 --- a/readerserver/scraper/headless/_browser.js +++ b/readerserver/scraper/headless/_browser.js @@ -28,7 +28,7 @@ module.exports.getDetails = async (url) => { await tab.addInitScript({ path: "scraper/headless/scripts/bypass-paywalls-chrome/src/js/contentScript.js" }); await tab.addInitScript({ path: "scraper/headless/scripts/cosmetic-filters.js" }); await tab.addInitScript({ path: "scraper/headless/scripts/fix-relative-links.js" }); - await tab.goto(url, { timeout: 60000, waitUntil: "domcontentloaded" }); + await tab.goto(url, { timeout: 90000, waitUntil: "domcontentloaded" }); await tab.waitForTimeout(2000); const body = await tab.content(); diff --git a/readerserver/scraper/headless/scripts/bypass-paywalls-chrome b/readerserver/scraper/headless/scripts/bypass-paywalls-chrome index fff7f48..fb1b09f 160000 --- a/readerserver/scraper/headless/scripts/bypass-paywalls-chrome +++ b/readerserver/scraper/headless/scripts/bypass-paywalls-chrome @@ -1 +1 @@ -Subproject commit fff7f483db947e690977bfc80955a53329d3d349 +Subproject commit fb1b09fccbb64f1d782753cc5c425eb0723e596f diff --git a/readerserver/scraper/headless/scripts/cosmetic-filters.js b/readerserver/scraper/headless/scripts/cosmetic-filters.js index 7a1941a..cfc0cd3 100644 --- a/readerserver/scraper/headless/scripts/cosmetic-filters.js +++ b/readerserver/scraper/headless/scripts/cosmetic-filters.js @@ -88,7 +88,7 @@ return displayNone || visibilityHidden; }) .forEach((element) => element && element.remove()); - }, 500); + }, 1500); } function removeSelectors(selectors) {