For real this time
This commit is contained in:
Simon Cambier
2022-12-12 21:24:33 +01:00
parent 3bdbfe12bf
commit a8199db570
3 changed files with 8 additions and 5 deletions

View File

@@ -5,7 +5,7 @@ import type {
ResultNote,
SearchMatch,
} from '../globals'
import { chsRegex, chsSegmenter, SPACE_OR_PUNCTUATION } from '../globals'
import { chsRegex, getChsSegmenter, SPACE_OR_PUNCTUATION } from '../globals'
import { settings } from '../settings'
import {
chunkArray,
@@ -19,7 +19,7 @@ import { cacheManager } from '../cache-manager'
const tokenize = (text: string): string[] => {
const tokens = text.split(SPACE_OR_PUNCTUATION)
const chsSegmenter = getChsSegmenter()
if (chsSegmenter) {
return tokens.flatMap(word =>
chsRegex.test(word) ? chsSegmenter.cut(word) : [word]