forked from tanner/qotnews
Tanner Collin
c65fb69092
Too lazy to debug this:
2023-08-29 12:56:35,111 - root - INFO - Updating lobsters story: yktkwr, index: 55
Traceback (most recent call last):
File "src/gevent/greenlet.py", line 854, in gevent._gevent_cgreenlet.Greenlet.run
File "/home/tanner/qotnews/apiserver/server.py", line 194, in feed_thread
valid = feed.update_story(story)
File "/home/tanner/qotnews/apiserver/feed.py", line 74, in update_story
res = lobsters.story(story['ref'])
File "/home/tanner/qotnews/apiserver/feeds/lobsters.py", line 103, in story
s['comments'] = iter_comments(r['comments'])
File "/home/tanner/qotnews/apiserver/feeds/lobsters.py", line 76, in iter_comments
parent_stack = parent_stack[:indent-1]
TypeError: unsupported operand type(s) for -: 'NoneType' and 'int'
2023-08-29T12:56:35Z <Greenlet at 0x7f92ad840ae0: feed_thread> failed with TypeError
123 lines
3.3 KiB
Python
123 lines
3.3 KiB
Python
import logging
|
|
logging.basicConfig(
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
level=logging.DEBUG)
|
|
|
|
if __name__ == '__main__':
|
|
import sys
|
|
sys.path.insert(0,'.')
|
|
|
|
import requests
|
|
from datetime import datetime
|
|
|
|
from utils import clean
|
|
|
|
API_HOTTEST = lambda x: 'https://lobste.rs/hottest.json'
|
|
API_ITEM = lambda x : 'https://lobste.rs/s/{}.json'.format(x)
|
|
|
|
SITE_LINK = lambda x : 'https://lobste.rs/s/{}'.format(x)
|
|
SITE_AUTHOR_LINK = lambda x : 'https://lobste.rs/u/{}'.format(x)
|
|
|
|
def api(route, ref=None):
|
|
try:
|
|
r = requests.get(route(ref), timeout=5)
|
|
if r.status_code != 200:
|
|
raise Exception('Bad response code ' + str(r.status_code))
|
|
return r.json()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except BaseException as e:
|
|
logging.error('Problem hitting lobsters API: {}, trying again'.format(str(e)))
|
|
|
|
try:
|
|
r = requests.get(route(ref), timeout=15)
|
|
if r.status_code != 200:
|
|
raise Exception('Bad response code ' + str(r.status_code))
|
|
return r.json()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except BaseException as e:
|
|
logging.error('Problem hitting lobsters API: {}'.format(str(e)))
|
|
return False
|
|
|
|
def feed():
|
|
return [x['short_id'] for x in api(API_HOTTEST) or []]
|
|
|
|
def unix(date_str):
|
|
date_str = date_str.replace(':', '')
|
|
return int(datetime.strptime(date_str, '%Y-%m-%dT%H%M%S.%f%z').timestamp())
|
|
|
|
def make_comment(i):
|
|
c = {}
|
|
try:
|
|
c['author'] = i['commenting_user']['username']
|
|
except KeyError:
|
|
c['author'] = ''
|
|
c['score'] = i.get('score', 0)
|
|
try:
|
|
c['date'] = unix(i['created_at'])
|
|
except KeyError:
|
|
c['date'] = 0
|
|
c['text'] = clean(i.get('comment', '') or '')
|
|
c['comments'] = []
|
|
return c
|
|
|
|
def iter_comments(flat_comments):
|
|
nested_comments = []
|
|
parent_stack = []
|
|
for comment in flat_comments:
|
|
c = make_comment(comment)
|
|
indent = comment['indent_level']
|
|
|
|
if indent == 1:
|
|
nested_comments.append(c)
|
|
parent_stack = [c]
|
|
else:
|
|
parent_stack = parent_stack[:indent-1]
|
|
p = parent_stack[-1]
|
|
p['comments'].append(c)
|
|
parent_stack.append(c)
|
|
return nested_comments
|
|
|
|
def story(ref):
|
|
r = api(API_ITEM, ref)
|
|
if not r:
|
|
logging.info('Bad Lobsters API response.')
|
|
return False
|
|
|
|
s = {}
|
|
try:
|
|
s['author'] = r['submitter_user']['username']
|
|
s['author_link'] = SITE_AUTHOR_LINK(s['author'])
|
|
except KeyError:
|
|
s['author'] = ''
|
|
s['author_link'] = ''
|
|
s['score'] = r.get('score', 0)
|
|
try:
|
|
s['date'] = unix(r['created_at'])
|
|
except KeyError:
|
|
s['date'] = 0
|
|
s['title'] = r.get('title', '')
|
|
s['link'] = SITE_LINK(ref)
|
|
s['url'] = r.get('url', '')
|
|
|
|
try:
|
|
s['comments'] = iter_comments(r['comments'])
|
|
except TypeError:
|
|
logging.exception('TypeError iterating lobsters comments, skipping article.')
|
|
return False
|
|
|
|
s['num_comments'] = r['comment_count']
|
|
|
|
if 'description' in r and r['description']:
|
|
s['text'] = clean(r['description'] or '')
|
|
|
|
return s
|
|
|
|
# scratchpad so I can quickly develop the parser
|
|
if __name__ == '__main__':
|
|
#print(feed())
|
|
import json
|
|
print(json.dumps(story('fzvd1v')))
|
|
#print(story(20802050))
|