feat: Add smallweb filter checkbox and server-side filtering

Co-authored-by: aider (gemini/gemini-2.5-pro) <aider@aider.chat>

apiserver/server.py

@@ -16,6 +16,7 @@ import traceback
 import time
 import datetime
 import humanize
+import urllib.request
 from urllib.parse import urlparse, parse_qs
 
 import settings
@@ -28,6 +29,21 @@ from flask import abort, Flask, request, render_template, stream_with_context, R
 from werkzeug.exceptions import NotFound
 from flask_cors import CORS
 
+smallweb_set = set()
+def load_smallweb_list():
+    global smallweb_set
+    try:
+        url = 'https://raw.githubusercontent.com/kagisearch/smallweb/refs/heads/main/smallweb.txt'
+        with urllib.request.urlopen(url, timeout=10) as response:
+            urls = response.read().decode('utf-8').splitlines()
+            hosts = {urlparse(u).hostname for u in urls if u and urlparse(u).hostname}
+            smallweb_set = {h.replace('www.', '') for h in hosts}
+            logging.info('Loaded {} smallweb domains.'.format(len(smallweb_set)))
+    except Exception as e:
+        logging.error('Failed to load smallweb list: {}'.format(e))
+
+load_smallweb_list()
+
 database.init()
 search.init()
 
@@ -57,6 +73,21 @@ def api():
     skip = request.args.get('skip', 0)
     limit = request.args.get('limit', settings.FEED_LENGTH)
     stories = database.get_stories(limit, skip)
+
+    if request.args.get('smallweb') == 'true' and smallweb_set:
+        filtered_stories = []
+        for story_str in stories:
+            story = json.loads(story_str)
+            story_url = story.get('url') or story.get('link') or ''
+            if not story_url:
+                continue
+            hostname = urlparse(story_url).hostname
+            if hostname:
+                hostname = hostname.replace('www.', '')
+                if hostname in smallweb_set:
+                    filtered_stories.append(story_str)
+        stories = filtered_stories
+
     # hacky nested json
     res = Response('{"stories":[' + ','.join(stories) + ']}')
     res.headers['content-type'] = 'application/json'

webclient/src/Feed.js

@@ -8,9 +8,17 @@ function Feed({ updateCache }) {
 	const [stories, setStories] = useState(() => JSON.parse(localStorage.getItem('stories')) || false);
 	const [error, setError] = useState('');
 	const [loadingStatus, setLoadingStatus] = useState(null);
+	const [filterSmallweb, setFilterSmallweb] = useState(() => localStorage.getItem('filterSmallweb') === 'true');
+
+	const handleFilterChange = e => {
+		const isChecked = e.target.checked;
+		setFilterSmallweb(isChecked);
+		localStorage.setItem('filterSmallweb', isChecked);
+		setStories(false);
+	};
 
 	useEffect(() => {
-		fetch('/api')
+		fetch(filterSmallweb ? '/api?smallweb=true' : '/api')
 			.then(res => {
 				if (!res.ok) {
 					throw new Error(`Server responded with ${res.status} ${res.statusText}`);
@@ -93,7 +101,7 @@ function Feed({ updateCache }) {
 					setError(errorMessage);
 				}
 			);
-	}, [updateCache]);
+	}, [updateCache, filterSmallweb]);
 
 	return (
 		<div className='container'>
@@ -102,6 +110,11 @@ function Feed({ updateCache }) {
 				<meta name="robots" content="index" />
 			</Helmet>
 
+			<div style={{marginBottom: '1rem'}}>
+				<input type="checkbox" id="filter-smallweb" checked={filterSmallweb} onChange={handleFilterChange} />
+				<label htmlFor="filter-smallweb" style={{paddingLeft: '0.5rem'}}>Filter Smallweb</label>
+			</div>
+
 			{error &&
 				<details style={{marginBottom: '1rem'}}>
 					<summary>Connection error? Click to expand.</summary>