Removed deduplication of tokens

2025-07-22 21:19:21 +02:00
parent c25cc8e62b
commit 1fa6898bbe
1 changed files with 4 additions and 1 deletions
--- a/src/search/tokenizer.ts
+++ b/src/search/tokenizer.ts
@@ -39,7 +39,10 @@ export class Tokenizer {
      }

      // Remove duplicates
-      tokens = [...new Set(tokens)]
+      // tokens = [...new Set(tokens)]
+      
+      // Remove empty tokens
+      tokens = tokens.filter(Boolean)

      return tokens
    } catch (e) {