Fix: issue#190 (#317)

* Fixed search results with diacritics - Caches are now stored with diacritics regardless of settings.ignoreDiacritics - Modified getMatches() behavior to return results with correct form - Modified ResultItemVault.svelte * Fixed highlighting words with comma and period - remove commas and periods from matches * Fixed highlighting of Cyrillic characters - changed highlight regexp determination to be based on character type * Fixed highlighting problem of Japanese and Korean - marked some Japanese diacritics to escape removal - added NFC normalization to keep right form of Korean character * Fixed highlighting of words with punctuation - deleted space/punctuation list from stringsToRegex() - it seems to be working correctly with words with punctuation and hyphenated words AFAIK * Deleted some unused imports * Modified the comment * Added comment * Fixed highlighting issue with comma and period * Fixed highlighting issue with caret and other symbols - Added `^` to separators - Changed regex to use separators - Added escape of `^` from diacritics removal
2023-11-29 15:32:25 +09:00
parent 115886af78
commit 9cac7c0581
5 changed files with 44 additions and 39 deletions
--- a/src/cache-manager.ts
+++ b/src/cache-manager.ts
@@ -17,7 +17,6 @@ import {
  isFilenameIndexable,
  logDebug,
  makeMD5,
-  removeDiacritics,
 } from './tools/utils'
 import type { CanvasData } from 'obsidian/canvas'
 import type { AsPlainObject } from 'minisearch'
@@ -115,7 +114,6 @@ async function getAndMapIndexedDocument(
    console.warn(`Omnisearch: ${content} content for file`, file.path)
    content = ''
  }
-  content = removeDiacritics(content)
  const metadata = app.metadataCache.getFileCache(file)

  // Look for links that lead to non-existing files,
@@ -143,7 +141,7 @@ async function getAndMapIndexedDocument(

  const tags = getTagsFromMetadata(metadata)
  return {
-    basename: removeDiacritics(file.basename),
+    basename: file.basename,
    content,
    path: file.path,
    mtime: file.stat.mtime,
--- a/src/components/ResultItemVault.svelte
+++ b/src/components/ResultItemVault.svelte
@@ -1,5 +1,5 @@
 <script lang="ts">
-  import { settings, showExcerpt } from 'src/settings'
+  import { showExcerpt } from 'src/settings'
  import type { ResultNote } from '../globals'
  import {
    getExtension,
@@ -7,12 +7,16 @@
    isFileImage,
    isFilePDF,
    pathWithoutFilename,
-    removeDiacritics,
  } from '../tools/utils'
  import ResultItemContainer from './ResultItemContainer.svelte'
  import { TFile, setIcon, App } from 'obsidian'
  import { cloneDeep } from 'lodash-es'
-  import { stringsToRegex, getMatches, makeExcerpt, highlightText } from 'src/tools/text-processing'
+  import {
+    stringsToRegex,
+    getMatches,
+    makeExcerpt,
+    highlightText,
+  } from 'src/tools/text-processing'

  export let selected = false
  export let note: ResultNote
@@ -44,9 +48,6 @@
  $: {
    title = note.basename
    notePath = pathWithoutFilename(note.path)
-    if (settings.ignoreDiacritics) {
-      title = removeDiacritics(title)
-    }

    // Icons
    if (elFolderPathIcon) {
--- a/src/globals.ts
+++ b/src/globals.ts
@@ -119,10 +119,9 @@ export function isCacheEnabled(): boolean {
  return !Platform.isIosApp && settings.useCache
 }

-const separators =
-  /[|\t\n\r= -#%-*,.`\/<>:;?@[-\]_{}\u00A0\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u09FD\u0A76\u0AF0\u0C77\u0C84\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166E\u1680\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2000-\u200A\u2010-\u2029\u202F-\u2043\u2045-\u2051\u2053-\u205F\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E4F\u3000-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]/
+export const SEPARATORS =
+  /[|\t\n\r\^= -#%-*,.`\/<>:;?@[-\]_{}\u00A0\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u09FD\u0A76\u0AF0\u0C77\u0C84\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166E\u1680\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2000-\u200A\u2010-\u2029\u202F-\u2043\u2045-\u2051\u2053-\u205F\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E4F\u3000-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]/
    .toString()
    .slice(1, -1)
-export const SPACE_OR_PUNCTUATION_UNIQUE = new RegExp(`${separators}`, 'u')
-export const SPACE_OR_PUNCTUATION = new RegExp(`${separators}+`, 'u')
+export const SPACE_OR_PUNCTUATION = new RegExp(`${SEPARATORS}+`, 'u')
 export const BRACKETS_AND_SPACE = /[|\[\]\(\)<>\{\} \t\n\r]/u
--- a/src/tools/text-processing.ts
+++ b/src/tools/text-processing.ts
@@ -3,14 +3,13 @@ import {
  type SearchMatch,
  regexLineSplit,
  regexYaml,
-  getChsSegmenter,
-  SPACE_OR_PUNCTUATION_UNIQUE,
  regexStripQuotes,
  excerptAfter,
  excerptBefore,
+  SEPARATORS,
 } from 'src/globals'
 import { settings } from 'src/settings'
-import { warnDebug } from './utils'
+import { removeDiacritics, warnDebug } from './utils'
 import type { Query } from 'src/search/query'
 import { Notice } from 'obsidian'
 import { escapeRegExp } from 'lodash-es'
@@ -32,17 +31,18 @@ export function highlightText(text: string, matches: SearchMatch[]): string {
  if (!matches.length) {
    return text
  }
-  const chsSegmenter = getChsSegmenter()
-  try {
+    try {
    // Text to highlight
    const src = new RegExp(
      matches
        .map(
          // This regex will match the word (with \b word boundary)
-          // and, if ChsSegmenter is active, the simple string (without word boundary)
+          // \b doesn't detect non-alphabetical character's word boundary, so we need to escape it
          matchItem =>
            `\\b${escapeRegExp(matchItem.match)}\\b${
-              chsSegmenter ? `|${escapeRegExp(matchItem.match)}` : ''
+              !/[a-zA-Z]/.test(matchItem.match)
+                ? `|${escapeRegExp(matchItem.match)}`
+                : ''
            }`
        )
        .join('|'),
@@ -55,7 +55,7 @@ export function highlightText(text: string, matches: SearchMatch[]): string {
        match.match(
          new RegExp(
            `\\b${escapeRegExp(info.match)}\\b${
-              chsSegmenter ? `|${escapeRegExp(info.match)}` : ''
+              !/[a-zA-Z]/.test(info.match) ? `|${escapeRegExp(info.match)}` : ''
            }`,
            'giu'
          )
@@ -89,7 +89,7 @@ export function splitLines(text: string): string[] {
 }

 export function removeFrontMatter(text: string): string {
-  // Regex to recognize YAML Front Matter (at beginning of file, 3 hyphens, than any charecter, including newlines, then 3 hyphens).
+  // Regex to recognize YAML Front Matter (at beginning of file, 3 hyphens, than any character, including newlines, then 3 hyphens).
  return text.replace(regexYaml, '')
 }

@@ -102,18 +102,7 @@ export function stringsToRegex(strings: string[]): RegExp {
  // sort strings by decreasing length, so that longer strings are matched first
  strings.sort((a, b) => b.length - a.length)

-  const joined =
-    '(' +
-    // Default word split is not applied if the user uses the cm-chs-patch plugin
-    (getChsSegmenter()
-      ? ''
-      : // Split on start of line, spaces, punctuation, or capital letters (for camelCase)
-      // We also add the hyphen to the list of characters that can split words
-      settings.splitCamelCase
-      ? `^|${SPACE_OR_PUNCTUATION_UNIQUE.source}|\-|[A-Z]`
-      : `^|${SPACE_OR_PUNCTUATION_UNIQUE.source}|\-`) +
-    ')' +
-    `(${strings.map(s => escapeRegExp(s)).join('|')})`
+  const joined =`(${strings.map(s => escapeRegExp(s)).join('|')})`

  return new RegExp(`${joined}`, 'gui')
 }
@@ -123,7 +112,12 @@ export function getMatches(
  reg: RegExp,
  query?: Query
 ): SearchMatch[] {
-  text = text.toLowerCase()
+  const separatorRegExp = new RegExp(SEPARATORS, 'gu')
+  const originalText = text
+  text = text.toLowerCase().replace(separatorRegExp, ' ')
+  if (settings.ignoreDiacritics) {
+    text = removeDiacritics(text)
+  }
  const startTime = new Date().getTime()
  let match: RegExpExecArray | null = null
  let matches: SearchMatch[] = []
@@ -134,9 +128,13 @@ export function getMatches(
      warnDebug('Stopped getMatches at', count, 'results')
      break
    }
-    const m = match[2]
-    if (m && match.index >= 0) {
-      matches.push({ match: m, offset: match.index + 1 })
+    const matchStartIndex = match.index
+    const matchEndIndex = matchStartIndex + match[0].length
+    const originalMatch = originalText
+      .substring(matchStartIndex, matchEndIndex)
+      .trim()
+    if (originalMatch && match.index >= 0) {
+      matches.push({ match: originalMatch, offset: match.index + 1 })
    }
  }

--- a/src/tools/utils.ts
+++ b/src/tools/utils.ts
@@ -115,14 +115,23 @@ export function getTagsFromMetadata(metadata: CachedMetadata | null): string[] {
 * https://stackoverflow.com/a/37511463
 */
 export function removeDiacritics(str: string): string {
+  // Japanese diacritics that should be distinguished
+  const excludeDiacritics: string[] = ['\\u30FC', '\\u309A', '\\u3099']
+  const regexpExclude: string = excludeDiacritics.join('|')
+  const regexp: RegExp = new RegExp(`(?!${regexpExclude})\\p{Diacritic}`, 'gu')
+
  if (str === null || str === undefined) {
    return ''
  }
  // Keep backticks for code blocks, because otherwise they are removed by the .normalize() function
  // https://stackoverflow.com/a/36100275
  str = str.replaceAll('`', '[__omnisearch__backtick__]')
-  str = str.normalize('NFD').replace(/\p{Diacritic}/gu, '')
+  // Keep caret same as above
+  str = str.replaceAll('^', '[__omnisearch__caret__]')
+  // To keep right form of Korean character, NFC normalization is necessary
+  str = str.normalize('NFD').replace(regexp, '').normalize('NFC')
  str = str.replaceAll('[__omnisearch__backtick__]', '`')
+  str = str.replaceAll('[__omnisearch__caret__]', '^')
  return str
 }