Fix: issue#190 (#317)

* Fixed search results with diacritics

- Caches are now stored with diacritics regardless of settings.ignoreDiacritics
- Modified getMatches() behavior to return results with correct form
- Modified ResultItemVault.svelte

* Fixed highlighting words with comma and period

- remove commas and periods from matches

* Fixed highlighting of Cyrillic characters

- changed highlight regexp determination to be based on character type

* Fixed highlighting problem of Japanese and Korean

- marked some Japanese diacritics to escape removal
- added NFC normalization to keep right form of Korean character

* Fixed highlighting of words with punctuation

- deleted space/punctuation list from stringsToRegex()
- it seems to be working correctly with words with punctuation and hyphenated words AFAIK

* Deleted some unused imports

* Modified the comment

* Added comment

* Fixed highlighting issue with comma and period

* Fixed highlighting issue with caret and other symbols

- Added `^` to separators
- Changed regex to use separators
- Added escape of `^` from diacritics removal

src/cache-manager.ts

@@ -17,7 +17,6 @@ import {
   isFilenameIndexable,
   logDebug,
   makeMD5,
-  removeDiacritics,
 } from './tools/utils'
 import type { CanvasData } from 'obsidian/canvas'
 import type { AsPlainObject } from 'minisearch'
@@ -115,7 +114,6 @@ async function getAndMapIndexedDocument(
     console.warn(`Omnisearch: ${content} content for file`, file.path)
     content = ''
   }
-  content = removeDiacritics(content)
   const metadata = app.metadataCache.getFileCache(file)
 
   // Look for links that lead to non-existing files,
@@ -143,7 +141,7 @@ async function getAndMapIndexedDocument(
 
   const tags = getTagsFromMetadata(metadata)
   return {
-    basename: removeDiacritics(file.basename),
+    basename: file.basename,
     content,
     path: file.path,
     mtime: file.stat.mtime,

src/components/ResultItemVault.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import { settings, showExcerpt } from 'src/settings'
+  import { showExcerpt } from 'src/settings'
   import type { ResultNote } from '../globals'
   import {
     getExtension,
@@ -7,12 +7,16 @@
     isFileImage,
     isFilePDF,
     pathWithoutFilename,
-    removeDiacritics,
   } from '../tools/utils'
   import ResultItemContainer from './ResultItemContainer.svelte'
   import { TFile, setIcon, App } from 'obsidian'
   import { cloneDeep } from 'lodash-es'
-  import { stringsToRegex, getMatches, makeExcerpt, highlightText } from 'src/tools/text-processing'
+  import {
+    stringsToRegex,
+    getMatches,
+    makeExcerpt,
+    highlightText,
+  } from 'src/tools/text-processing'
 
   export let selected = false
   export let note: ResultNote
@@ -44,9 +48,6 @@
   $: {
     title = note.basename
     notePath = pathWithoutFilename(note.path)
-    if (settings.ignoreDiacritics) {
-      title = removeDiacritics(title)
-    }
 
     // Icons
     if (elFolderPathIcon) {

src/globals.ts

@@ -119,10 +119,9 @@ export function isCacheEnabled(): boolean {
   return !Platform.isIosApp && settings.useCache
 }
 
-const separators =
+export const SEPARATORS =
-  /[|\t\n\r= -#%-*,.`\/<>:;?@[-\]_{}\u00A0\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u09FD\u0A76\u0AF0\u0C77\u0C84\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166E\u1680\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2000-\u200A\u2010-\u2029\u202F-\u2043\u2045-\u2051\u2053-\u205F\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E4F\u3000-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]/
+  /[|\t\n\r\^= -#%-*,.`\/<>:;?@[-\]_{}\u00A0\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u09FD\u0A76\u0AF0\u0C77\u0C84\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166E\u1680\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2000-\u200A\u2010-\u2029\u202F-\u2043\u2045-\u2051\u2053-\u205F\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E4F\u3000-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]/
     .toString()
     .slice(1, -1)
-export const SPACE_OR_PUNCTUATION_UNIQUE = new RegExp(`${separators}`, 'u')
+export const SPACE_OR_PUNCTUATION = new RegExp(`${SEPARATORS}+`, 'u')
-export const SPACE_OR_PUNCTUATION = new RegExp(`${separators}+`, 'u')
 export const BRACKETS_AND_SPACE = /[|\[\]\(\)<>\{\} \t\n\r]/u

src/tools/text-processing.ts

@@ -3,14 +3,13 @@ import {
   type SearchMatch,
   regexLineSplit,
   regexYaml,
-  getChsSegmenter,
-  SPACE_OR_PUNCTUATION_UNIQUE,
   regexStripQuotes,
   excerptAfter,
   excerptBefore,
+  SEPARATORS,
 } from 'src/globals'
 import { settings } from 'src/settings'
-import { warnDebug } from './utils'
+import { removeDiacritics, warnDebug } from './utils'
 import type { Query } from 'src/search/query'
 import { Notice } from 'obsidian'
 import { escapeRegExp } from 'lodash-es'
@@ -32,17 +31,18 @@ export function highlightText(text: string, matches: SearchMatch[]): string {
   if (!matches.length) {
     return text
   }
-  const chsSegmenter = getChsSegmenter()
+    try {
-  try {
     // Text to highlight
     const src = new RegExp(
       matches
         .map(
           // This regex will match the word (with \b word boundary)
-          // and, if ChsSegmenter is active, the simple string (without word boundary)
+          // \b doesn't detect non-alphabetical character's word boundary, so we need to escape it
           matchItem =>
             `\\b${escapeRegExp(matchItem.match)}\\b${
-              chsSegmenter ? `|${escapeRegExp(matchItem.match)}` : ''
+              !/[a-zA-Z]/.test(matchItem.match)
+                ? `|${escapeRegExp(matchItem.match)}`
+                : ''
             }`
         )
         .join('|'),
@@ -55,7 +55,7 @@ export function highlightText(text: string, matches: SearchMatch[]): string {
         match.match(
           new RegExp(
             `\\b${escapeRegExp(info.match)}\\b${
-              chsSegmenter ? `|${escapeRegExp(info.match)}` : ''
+              !/[a-zA-Z]/.test(info.match) ? `|${escapeRegExp(info.match)}` : ''
             }`,
             'giu'
           )
@@ -89,7 +89,7 @@ export function splitLines(text: string): string[] {
 }
 
 export function removeFrontMatter(text: string): string {
-  // Regex to recognize YAML Front Matter (at beginning of file, 3 hyphens, than any charecter, including newlines, then 3 hyphens).
+  // Regex to recognize YAML Front Matter (at beginning of file, 3 hyphens, than any character, including newlines, then 3 hyphens).
   return text.replace(regexYaml, '')
 }
 
@@ -102,18 +102,7 @@ export function stringsToRegex(strings: string[]): RegExp {
   // sort strings by decreasing length, so that longer strings are matched first
   strings.sort((a, b) => b.length - a.length)
 
-  const joined =
+  const joined =`(${strings.map(s => escapeRegExp(s)).join('|')})`
-    '(' +
-    // Default word split is not applied if the user uses the cm-chs-patch plugin
-    (getChsSegmenter()
-      ? ''
-      : // Split on start of line, spaces, punctuation, or capital letters (for camelCase)
-      // We also add the hyphen to the list of characters that can split words
-      settings.splitCamelCase
-      ? `^|${SPACE_OR_PUNCTUATION_UNIQUE.source}|\-|[A-Z]`
-      : `^|${SPACE_OR_PUNCTUATION_UNIQUE.source}|\-`) +
-    ')' +
-    `(${strings.map(s => escapeRegExp(s)).join('|')})`
 
   return new RegExp(`${joined}`, 'gui')
 }
@@ -123,7 +112,12 @@ export function getMatches(
   reg: RegExp,
   query?: Query
 ): SearchMatch[] {
-  text = text.toLowerCase()
+  const separatorRegExp = new RegExp(SEPARATORS, 'gu')
+  const originalText = text
+  text = text.toLowerCase().replace(separatorRegExp, ' ')
+  if (settings.ignoreDiacritics) {
+    text = removeDiacritics(text)
+  }
   const startTime = new Date().getTime()
   let match: RegExpExecArray | null = null
   let matches: SearchMatch[] = []
@@ -134,9 +128,13 @@ export function getMatches(
       warnDebug('Stopped getMatches at', count, 'results')
       break
     }
-    const m = match[2]
+    const matchStartIndex = match.index
-    if (m && match.index >= 0) {
+    const matchEndIndex = matchStartIndex + match[0].length
-      matches.push({ match: m, offset: match.index + 1 })
+    const originalMatch = originalText
+      .substring(matchStartIndex, matchEndIndex)
+      .trim()
+    if (originalMatch && match.index >= 0) {
+      matches.push({ match: originalMatch, offset: match.index + 1 })
     }
   }
 

src/tools/utils.ts

@@ -115,14 +115,23 @@ export function getTagsFromMetadata(metadata: CachedMetadata | null): string[] {
  * https://stackoverflow.com/a/37511463
  */
 export function removeDiacritics(str: string): string {
+  // Japanese diacritics that should be distinguished
+  const excludeDiacritics: string[] = ['\\u30FC', '\\u309A', '\\u3099']
+  const regexpExclude: string = excludeDiacritics.join('|')
+  const regexp: RegExp = new RegExp(`(?!${regexpExclude})\\p{Diacritic}`, 'gu')
+
   if (str === null || str === undefined) {
     return ''
   }
   // Keep backticks for code blocks, because otherwise they are removed by the .normalize() function
   // https://stackoverflow.com/a/36100275
   str = str.replaceAll('`', '[__omnisearch__backtick__]')
-  str = str.normalize('NFD').replace(/\p{Diacritic}/gu, '')
+  // Keep caret same as above
+  str = str.replaceAll('^', '[__omnisearch__caret__]')
+  // To keep right form of Korean character, NFC normalization is necessary
+  str = str.normalize('NFD').replace(regexp, '').normalize('NFC')
   str = str.replaceAll('[__omnisearch__backtick__]', '`')
+  str = str.replaceAll('[__omnisearch__caret__]', '^')
   return str
 }