diff --git a/src/search/tokenizer.ts b/src/search/tokenizer.ts index 6149070..84f0b73 100644 --- a/src/search/tokenizer.ts +++ b/src/search/tokenizer.ts @@ -27,15 +27,11 @@ export class Tokenizer { } let tokens = this.tokenizeTokens(text, { skipChs: true }) - - // Split hyphenated tokens - tokens = [...tokens, ...tokens.flatMap(splitHyphens)] - - // Split camelCase tokens into "camel" and "case - tokens = [...tokens, ...tokens.flatMap(splitCamelCase)] - - // Add whole words (aka "not tokens") - tokens = [...tokens, ...words] + tokens = [...tokens.flatMap(token => [ + token, + ...splitHyphens(token), + ...splitCamelCase(token), + ]), ...words] // Add urls if (urls.length) {