Merge branch 'feature/225-hyphen-split' into develop

src/globals.ts

@@ -106,4 +106,4 @@ export function isCacheEnabled(): boolean {
 }
 
 export const SPACE_OR_PUNCTUATION =
-  /[|\n\r -#%-*,-/:;?@[-\]_{}\u00A0\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u09FD\u0A76\u0AF0\u0C77\u0C84\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166E\u1680\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2000-\u200A\u2010-\u2029\u202F-\u2043\u2045-\u2051\u2053-\u205F\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E4F\u3000-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]+/u
+  /[|\n\r -#%-*,.\/:;?@[-\]_{}\u00A0\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u09FD\u0A76\u0AF0\u0C77\u0C84\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166E\u1680\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2000-\u200A\u2010-\u2029\u202F-\u2043\u2045-\u2051\u2053-\u205F\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E4F\u3000-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]+/u

src/search/omnisearch.ts

@@ -12,6 +12,7 @@ import {
   logDebug,
   removeDiacritics,
   splitCamelCase,
+  splitHyphens,
   stringsToRegex,
   stripMarkdownCharacters,
   warnDebug,
@@ -22,17 +23,22 @@ import { cacheManager } from '../cache-manager'
 import { sortBy } from 'lodash-es'
 
 const tokenize = (text: string): string[] => {
-  const tokens = text.split(SPACE_OR_PUNCTUATION)
+  let tokens = text.split(SPACE_OR_PUNCTUATION)
+
+  // When enabled, we only use the chsSegmenter,
+  // and not the other custom tokenizers
   const chsSegmenter = getChsSegmenter()
   if (chsSegmenter) {
-    return tokens.flatMap(word =>
+    tokens = tokens.flatMap(word =>
       chsRegex.test(word) ? chsSegmenter.cut(word) : [word]
     )
   } else {
-    if (settings.splitCamelCase)
+    // Split camelCase tokens into "camel" and "case
-      return [...tokens, ...tokens.flatMap(splitCamelCase)]
+    tokens = [...tokens, ...tokens.flatMap(splitCamelCase)]
-    return tokens
+    // Split hyphenated tokens
+    tokens = [...tokens, ...tokens.flatMap(splitHyphens)]
   }
+  return tokens
 }
 
 export class Omnisearch {

src/tools/utils.ts

@@ -26,14 +26,10 @@ export function highlighter(str: string): string {
   return `<span class="${highlightClass}">${str}</span>`
 }
 
-export function highlighterGroups(...args: any[]) {
+export function highlighterGroups(substring: string, ...args: any[]) {
-  if (
+  // args[0] is the single char preceding args[1], which is the word we want to highlight
-    args[1] !== null &&
+  if (!!args[1].trim())
-    args[1] !== undefined &&
+    return `<span>${args[0]}</span><span class="${highlightClass}">${args[1]}</span>`
-    args[2] !== null &&
-    args[2] !== undefined
-  )
-    return `<span>${args[1]}</span><span class="${highlightClass}">${args[2]}</span>`
   return '&lt;no content&gt;'
 }
 
@@ -91,15 +87,20 @@ export function getAllIndices(text: string, regex: RegExp): SearchMatch[] {
  */
 export function stringsToRegex(strings: string[]): RegExp {
   if (!strings.length) return /^$/g
+
+  // sort strings by decreasing length, so that longer strings are matched first
+  strings.sort((a, b) => b.length - a.length)
+
   const joined =
     '(' +
     // Default word split is not applied if the user uses the cm-chs-patch plugin
     (getChsSegmenter()
       ? ''
       : // Split on start of line, spaces, punctuation, or capital letters (for camelCase)
+      // We also add the hyphen to the list of characters that can split words
       settings.splitCamelCase
-      ? `^|${SPACE_OR_PUNCTUATION.source}|[A-Z]`
+      ? `^|${SPACE_OR_PUNCTUATION.source}|\-|[A-Z]`
-      : `^|${SPACE_OR_PUNCTUATION.source}`) +
+      : `^|${SPACE_OR_PUNCTUATION.source}|\-`) +
     ')' +
     `(${strings.map(s => escapeRegex(s)).join('|')})`
 
@@ -318,11 +319,32 @@ export function chunkArray<T>(arr: T[], len: number): T[][] {
 }
 
 /**
- * Converts a 'fooBarBAZLorem' into ['foo', 'Bar', 'BAZ', 'Lorem]
+ * Converts a 'fooBarBAZLorem' into ['foo', 'Bar', 'BAZ', 'Lorem']
+ * If the string isn't camelCase, returns an empty array
  * @param text
  */
 export function splitCamelCase(text: string): string[] {
-  return text.replace(/([a-z](?=[A-Z]))/g, '$1 ').split(' ')
+  const split = text
+    .replace(/([a-z](?=[A-Z]))/g, '$1 ')
+    .split(' ')
+    .filter(t => t)
+  if (split.length > 1) {
+    return split
+  }
+  return []
+}
+
+/**
+ * Converts a 'foo-bar-baz' into ['foo', 'bar', 'baz']
+ * If the string isn't hyphenated, returns an empty array
+ * @param text
+ */
+export function splitHyphens(text: string): string[] {
+  const split = text.split('-').filter(t => t)
+  if (split.length > 1) {
+    return split
+  }
+  return []
 }
 
 export function logDebug(...args: any[]): void {