feat: Fetch smallweb stories iteratively until limit met

Co-authored-by: aider (gemini/gemini-2.5-pro) <aider@aider.chat>

apiserver/server.py

@@ -76,21 +76,38 @@ cors = CORS(flask_app)
 def api():
     skip = request.args.get('skip', 0)
     limit = request.args.get('limit', settings.FEED_LENGTH)
-    stories = database.get_stories(limit, skip)
 
     if request.args.get('smallweb') == 'true' and smallweb_set:
+        limit = int(limit)
+        skip = int(skip)
         filtered_stories = []
-        for story_str in stories:
+        current_skip = skip
-            story = json.loads(story_str)
+
-            story_url = story.get('url') or story.get('link') or ''
+        while len(filtered_stories) < limit:
-            if not story_url:
+            stories_batch = database.get_stories(limit, current_skip)
-                continue
+            if not stories_batch:
-            hostname = urlparse(story_url).hostname
+                break
-            if hostname:
+
-                hostname = hostname.replace('www.', '')
+            for story_str in stories_batch:
-                if hostname in smallweb_set:
+                story = json.loads(story_str)
-                    filtered_stories.append(story_str)
+                story_url = story.get('url') or story.get('link') or ''
+                if not story_url:
+                    continue
+                hostname = urlparse(story_url).hostname
+                if hostname:
+                    hostname = hostname.replace('www.', '')
+                    if hostname in smallweb_set:
+                        filtered_stories.append(story_str)
+                        if len(filtered_stories) == limit:
+                            break
+            
+            if len(filtered_stories) == limit:
+                break
+            
+            current_skip += limit
         stories = filtered_stories
+    else:
+        stories = database.get_stories(limit, skip)
 
     # hacky nested json
     res = Response('{"stories":[' + ','.join(stories) + ']}')