perf: combine token split into 1 pass (#427)

2025-01-03 19:27:19 +08:00
parent c4fa419aa0
commit 0ebe903bb9
1 changed files with 5 additions and 9 deletions
--- a/src/search/tokenizer.ts
+++ b/src/search/tokenizer.ts
@@ -27,15 +27,11 @@ export class Tokenizer {
      }

      let tokens = this.tokenizeTokens(text, { skipChs: true })
-
-      // Split hyphenated tokens
-      tokens = [...tokens, ...tokens.flatMap(splitHyphens)]
-
-      // Split camelCase tokens into "camel" and "case
-      tokens = [...tokens, ...tokens.flatMap(splitCamelCase)]
-
-      // Add whole words (aka "not tokens")
-      tokens = [...tokens, ...words]
+      tokens = [...tokens.flatMap(token => [
+        token,
+        ...splitHyphens(token),
+        ...splitCamelCase(token),
+      ]), ...words]

      // Add urls
      if (urls.length) {