tanner/qotnews
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Archive Bloomberg articles first

Browse Source
This commit is contained in:
Tanner Collin 2019-10-08 08:00:50 +00:00
parent 5caa4542d8
commit 19e9a80be1
1 changed files with 27 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
apiserver/feed.py
View File

@ -9,6 +9,7 @@ import time
from feeds import hackernews, reddit, tildes from feeds import hackernews, reddit, tildes
OUTLINE_API = 'https://outlineapi.com/article' OUTLINE_API = 'https://outlineapi.com/article'
ARCHIVE_API = 'https://archive.fo/submit/'
READ_API = 'http://127.0.0.1:33843' READ_API = 'http://127.0.0.1:33843'
INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif'] INVALID_FILES = ['.pdf', '.png', '.jpg', '.gif']
@ -22,6 +23,22 @@ def list():
return feed return feed
def get_article(url): def get_article(url):
if 'bloomberg.com' in url:
try:
logging.info('Article from Bloomberg, archiving first...')
data = {'submitid': '9tjtS1EYe5wy8AJiYgVfH9P97uHU1IHG4lO67hsQpHOC3KKJrhqVIoQG2U7Rg%2Fpr', 'url': url}
r = requests.post(ARCHIVE_API, data=data, timeout=20, allow_redirects=False)
if r.status_code == 200:
logging.error('Submitted for archiving. Skipping to wait...')
return ''
elif 'location' in r.headers:
url = r.headers['location']
else:
raise Exception('Bad response code ' + str(r.status_code))
except BaseException as e:
logging.error('Problem archiving article: {}'.format(str(e)))
return ''
try: try:
params = {'source_url': url} params = {'source_url': url}
headers = {'Referer': 'https://outline.com/'} headers = {'Referer': 'https://outline.com/'}
@ -86,11 +103,14 @@ def update_story(story):
return True return True
if __name__ == '__main__': if __name__ == '__main__':
test_news_cache = {} #test_news_cache = {}
nid = 'jean' #nid = 'jean'
ref = 20802050 #ref = 20802050
source = 'hackernews' #source = 'hackernews'
test_news_cache[nid] = dict(id=nid, ref=ref, source=source) #test_news_cache[nid] = dict(id=nid, ref=ref, source=source)
news_story = test_news_cache[nid] #news_story = test_news_cache[nid]
update_story(news_story) #update_story(news_story)
print(get_article('https://www.bloomberg.com/news/articles/2019-09-23/xi-s-communists-under-pressure-as-high-prices-hit-china-workers'))
print('done') print('done')