tanner/qotnews
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

try stop bad pages.

Browse Source
This commit is contained in:
Jason Schwarzenberger 2020-11-04 16:34:31 +13:00
parent 939f4775a7
commit 16b59f6c67
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apiserver/feeds/news.py
View File

@ -14,7 +14,6 @@ import extruct
from utils import clean from utils import clean
OUTLINE_API = 'https://api.outline.com/v3/parse_article'
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0' USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0'
def unix(date_str): def unix(date_str):
@ -123,6 +122,9 @@ class Sitemap:
data = extruct.extract(markup) data = extruct.extract(markup)
s = parse_extruct(s, data) s = parse_extruct(s, data)
if not s['date']:
return False
return s return s
class Category: class Category:
@ -138,6 +140,7 @@ class Category:
links = [link.get('href') for link in links] links = [link.get('href') for link in links]
links = [f"{self.base_url}{link}" if link.startswith('/') else link for link in links] links = [f"{self.base_url}{link}" if link.startswith('/') else link for link in links]
links = list(filter(None, [link if link.startswith(self.category_url) else None for link in links])) links = list(filter(None, [link if link.startswith(self.category_url) else None for link in links]))
links = list(filter(None, [link if link != self.category_url else None for link in links]))
return links return links
def story(self, ref): def story(self, ref):
@ -156,6 +159,9 @@ class Category:
data = extruct.extract(markup) data = extruct.extract(markup)
s = parse_extruct(s, data) s = parse_extruct(s, data)
if not s['date']:
return False
return s return s
# scratchpad so I can quickly develop the parser # scratchpad so I can quickly develop the parser