From 1fa6898bbe6385217a178c3e4c7739b654d24fd0 Mon Sep 17 00:00:00 2001
From: Simon Cambier <simon.cambier@protonmail.com>
Date: Tue, 22 Jul 2025 21:19:21 +0200
Subject: [PATCH] Removed deduplication of tokens

---
 src/search/tokenizer.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/search/tokenizer.ts b/src/search/tokenizer.ts
index 84f0b73..f67e946 100644
--- a/src/search/tokenizer.ts
+++ b/src/search/tokenizer.ts
@@ -39,7 +39,10 @@ export class Tokenizer {
       }
 
       // Remove duplicates
-      tokens = [...new Set(tokens)]
+      // tokens = [...new Set(tokens)]
+      
+      // Remove empty tokens
+      tokens = tokens.filter(Boolean)
 
       return tokens
     } catch (e) {