From 9fff1b9e4615112658185d69331272742b94ac36 Mon Sep 17 00:00:00 2001 From: Jason Schwarzenberger Date: Wed, 4 Nov 2020 17:14:42 +1300 Subject: [PATCH] avoid duplicate articles listed on the category page --- apiserver/feeds/news.py | 84 +++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/apiserver/feeds/news.py b/apiserver/feeds/news.py index e0306aa..e0fae4e 100644 --- a/apiserver/feeds/news.py +++ b/apiserver/feeds/news.py @@ -16,6 +16,7 @@ from utils import clean USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0' + def unix(date_str): date_tzfix = date_str if ":" == date_tzfix[-3]: @@ -33,6 +34,7 @@ def unix(date_str): pass return 0 + def xml(route, ref=None): try: headers = {'User-Agent': USER_AGENT, 'X-Forwarded-For': '66.249.66.1'} @@ -46,6 +48,7 @@ def xml(route, ref=None): logging.error('Problem hitting URL: {}'.format(str(e))) return False + def parse_extruct(s, data): for rdfa in data['rdfa']: for key, props in rdfa.items(): @@ -94,6 +97,7 @@ def parse_extruct(s, data): return s + class Sitemap: def __init__(self, url): self.sitemap_url = url @@ -104,43 +108,8 @@ class Sitemap: soup = BeautifulSoup(markup, features='lxml') articles = soup.find('urlset').findAll('url') articles = list(filter(None, [a if a.find('lastmod') is not None else None for a in articles])) - return [x.find('loc').text for x in articles] or [] - - def story(self, ref): - markup = xml(lambda x: ref) - if not markup: - return False - - s = {} - s['author_link'] = '' - s['score'] = 0 - s['comments'] = [] - s['num_comments'] = 0 - s['link'] = ref - s['url'] = ref - s['date'] = 0 - - data = extruct.extract(markup) - s = parse_extruct(s, data) - - if not s['date']: - return False - return s - -class Category: - def __init__(self, url): - self.category_url = url - self.base_url = '/'.join(url.split('/')[:3]) - - def feed(self): - markup = xml(lambda x: self.category_url) - if not markup: return [] - soup = BeautifulSoup(markup, features='html.parser') - links = soup.find_all('a', href=True) - links = [link.get('href') for link in links] - links = [f"{self.base_url}{link}" if link.startswith('/') else link for link in links] - links = list(filter(None, [link if link.startswith(self.category_url) else None for link in links])) - links = list(filter(None, [link if link != self.category_url else None for link in links])) + links = [x.find('loc').text for x in articles] or [] + links = list(set(links)) return links def story(self, ref): @@ -164,6 +133,47 @@ class Category: return False return s + +class Category: + def __init__(self, url): + self.category_url = url + self.base_url = '/'.join(url.split('/')[:3]) + + def feed(self): + markup = xml(lambda x: self.category_url) + if not markup: return [] + soup = BeautifulSoup(markup, features='html.parser') + links = soup.find_all('a', href=True) + links = [link.get('href') for link in links] + links = [f"{self.base_url}{link}" if link.startswith('/') else link for link in links] + links = list(filter(None, [link if link.startswith(self.category_url) else None for link in links])) + links = list(filter(None, [link if link != self.category_url else None for link in links])) + links = list(set(links)) + + return links + + def story(self, ref): + markup = xml(lambda x: ref) + if not markup: + return False + + s = {} + s['author_link'] = '' + s['score'] = 0 + s['comments'] = [] + s['num_comments'] = 0 + s['link'] = ref + s['url'] = ref + s['date'] = 0 + + data = extruct.extract(markup) + s = parse_extruct(s, data) + + if not s['date']: + return False + return s + + # scratchpad so I can quickly develop the parser if __name__ == '__main__': print("Sitemap: Stuff")