diff --git a/apiserver/server.py b/apiserver/server.py index f0a0937..e0d7193 100644 --- a/apiserver/server.py +++ b/apiserver/server.py @@ -16,6 +16,7 @@ import traceback import time import datetime import humanize +import urllib.request from urllib.parse import urlparse, parse_qs import settings @@ -28,6 +29,21 @@ from flask import abort, Flask, request, render_template, stream_with_context, R from werkzeug.exceptions import NotFound from flask_cors import CORS +smallweb_set = set() +def load_smallweb_list(): + global smallweb_set + try: + url = 'https://raw.githubusercontent.com/kagisearch/smallweb/refs/heads/main/smallweb.txt' + with urllib.request.urlopen(url, timeout=10) as response: + urls = response.read().decode('utf-8').splitlines() + hosts = {urlparse(u).hostname for u in urls if u and urlparse(u).hostname} + smallweb_set = {h.replace('www.', '') for h in hosts} + logging.info('Loaded {} smallweb domains.'.format(len(smallweb_set))) + except Exception as e: + logging.error('Failed to load smallweb list: {}'.format(e)) + +load_smallweb_list() + database.init() search.init() @@ -57,6 +73,21 @@ def api(): skip = request.args.get('skip', 0) limit = request.args.get('limit', settings.FEED_LENGTH) stories = database.get_stories(limit, skip) + + if request.args.get('smallweb') == 'true' and smallweb_set: + filtered_stories = [] + for story_str in stories: + story = json.loads(story_str) + story_url = story.get('url') or story.get('link') or '' + if not story_url: + continue + hostname = urlparse(story_url).hostname + if hostname: + hostname = hostname.replace('www.', '') + if hostname in smallweb_set: + filtered_stories.append(story_str) + stories = filtered_stories + # hacky nested json res = Response('{"stories":[' + ','.join(stories) + ']}') res.headers['content-type'] = 'application/json' diff --git a/webclient/src/Feed.js b/webclient/src/Feed.js index 6129008..9d3dc60 100644 --- a/webclient/src/Feed.js +++ b/webclient/src/Feed.js @@ -8,9 +8,17 @@ function Feed({ updateCache }) { const [stories, setStories] = useState(() => JSON.parse(localStorage.getItem('stories')) || false); const [error, setError] = useState(''); const [loadingStatus, setLoadingStatus] = useState(null); + const [filterSmallweb, setFilterSmallweb] = useState(() => localStorage.getItem('filterSmallweb') === 'true'); + + const handleFilterChange = e => { + const isChecked = e.target.checked; + setFilterSmallweb(isChecked); + localStorage.setItem('filterSmallweb', isChecked); + setStories(false); + }; useEffect(() => { - fetch('/api') + fetch(filterSmallweb ? '/api?smallweb=true' : '/api') .then(res => { if (!res.ok) { throw new Error(`Server responded with ${res.status} ${res.statusText}`); @@ -93,7 +101,7 @@ function Feed({ updateCache }) { setError(errorMessage); } ); - }, [updateCache]); + }, [updateCache, filterSmallweb]); return (