From 5736cde21a6f2fe4198ae5157ade11c518eea32b Mon Sep 17 00:00:00 2001 From: Tanner Collin Date: Thu, 4 Dec 2025 22:18:22 +0000 Subject: [PATCH] feat: Fetch smallweb stories iteratively until limit met Co-authored-by: aider (gemini/gemini-2.5-pro) --- apiserver/server.py | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/apiserver/server.py b/apiserver/server.py index e6318bb..efacb1e 100644 --- a/apiserver/server.py +++ b/apiserver/server.py @@ -76,21 +76,38 @@ cors = CORS(flask_app) def api(): skip = request.args.get('skip', 0) limit = request.args.get('limit', settings.FEED_LENGTH) - stories = database.get_stories(limit, skip) if request.args.get('smallweb') == 'true' and smallweb_set: + limit = int(limit) + skip = int(skip) filtered_stories = [] - for story_str in stories: - story = json.loads(story_str) - story_url = story.get('url') or story.get('link') or '' - if not story_url: - continue - hostname = urlparse(story_url).hostname - if hostname: - hostname = hostname.replace('www.', '') - if hostname in smallweb_set: - filtered_stories.append(story_str) + current_skip = skip + + while len(filtered_stories) < limit: + stories_batch = database.get_stories(limit, current_skip) + if not stories_batch: + break + + for story_str in stories_batch: + story = json.loads(story_str) + story_url = story.get('url') or story.get('link') or '' + if not story_url: + continue + hostname = urlparse(story_url).hostname + if hostname: + hostname = hostname.replace('www.', '') + if hostname in smallweb_set: + filtered_stories.append(story_str) + if len(filtered_stories) == limit: + break + + if len(filtered_stories) == limit: + break + + current_skip += limit stories = filtered_stories + else: + stories = database.get_stories(limit, skip) # hacky nested json res = Response('{"stories":[' + ','.join(stories) + ']}')