tanner/obsidian-tannersearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: combine token split into 1 pass (#427)

Browse Source
This commit is contained in:
YuNing Chen
2025-01-03 19:27:19 +08:00
committed by GitHub
parent c4fa419aa0
commit 0ebe903bb9
1 changed files with 5 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/search/tokenizer.ts
View File

@@ -27,15 +27,11 @@ export class Tokenizer {
} }
let tokens = this.tokenizeTokens(text, { skipChs: true }) let tokens = this.tokenizeTokens(text, { skipChs: true })
tokens = [...tokens.flatMap(token => [
// Split hyphenated tokens token,
tokens = [...tokens, ...tokens.flatMap(splitHyphens)] ...splitHyphens(token),
...splitCamelCase(token),
// Split camelCase tokens into "camel" and "case ]), ...words]
tokens = [...tokens, ...tokens.flatMap(splitCamelCase)]
// Add whole words (aka "not tokens")
tokens = [...tokens, ...words]
// Add urls // Add urls
if (urls.length) { if (urls.length) {