avoid duplicate articles listed on the category page

apiserver/feeds/news.py

@@ -16,6 +16,7 @@ from utils import clean
 
 USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0'
 
+
 def unix(date_str):
     date_tzfix = date_str
     if ":" == date_tzfix[-3]:
@@ -33,6 +34,7 @@ def unix(date_str):
             pass
     return 0
 
+
 def xml(route, ref=None):
     try:
         headers = {'User-Agent': USER_AGENT, 'X-Forwarded-For': '66.249.66.1'}
@@ -46,6 +48,7 @@ def xml(route, ref=None):
         logging.error('Problem hitting URL: {}'.format(str(e)))
         return False
 
+
 def parse_extruct(s, data):
     for rdfa in data['rdfa']:
             for key, props in rdfa.items():
@@ -94,6 +97,7 @@ def parse_extruct(s, data):
 
     return s
 
+
 class Sitemap:
     def __init__(self, url):
         self.sitemap_url = url
@@ -104,7 +108,9 @@ class Sitemap:
         soup = BeautifulSoup(markup, features='lxml')
         articles = soup.find('urlset').findAll('url')
         articles = list(filter(None, [a if a.find('lastmod') is not None else None for a in articles]))
-        return [x.find('loc').text for x in articles] or []
+        links = [x.find('loc').text for x in articles] or []
+        links = list(set(links))
+        return links
 
     def story(self, ref):
         markup = xml(lambda x: ref)
@@ -127,6 +133,7 @@ class Sitemap:
             return False
         return s
 
+
 class Category:
     def __init__(self, url):
         self.category_url = url
@@ -141,6 +148,8 @@ class Category:
         links = [f"{self.base_url}{link}" if link.startswith('/') else link for link in links]
         links = list(filter(None, [link if link.startswith(self.category_url) else None for link in links]))
         links = list(filter(None, [link if link != self.category_url else None for link in links]))
+        links = list(set(links))
+
         return links
 
     def story(self, ref):
@@ -164,6 +173,7 @@ class Category:
             return False
         return s
 
+
 # scratchpad so I can quickly develop the parser
 if __name__ == '__main__':
     print("Sitemap: Stuff")