Add tests and search simulator TUI

jest.config.js

@@ -82,7 +82,11 @@ module.exports = {
   ],
 
   // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
-  // moduleNameMapper: {},
+  moduleNameMapper: {
+    '^obsidian$': '<rootDir>/tests/obsidian-mock.ts',
+    '^svelte/store$': '<rootDir>/tests/svelte-store-mock.ts',
+    '^lodash-es$': 'lodash',
+  },
 
   // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
   // modulePathIgnorePatterns: [],

package.json

@@ -8,7 +8,8 @@
     "build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
     "check": "svelte-check --tsconfig ./tsconfig.json",
     "version": "node version-bump.mjs",
-    "test": "jest"
+    "test": "jest",
+    "tui": "node tools/search-tui-launcher.js"
   },
   "keywords": [],
   "author": "Simon Cambier",

src/tools/text-processing.ts

@@ -26,7 +26,7 @@ export class TextProcessor {
       return text.replace(
         new RegExp(
           `(${matches
-            .map(item => escapeRegExp(escapeHTML(item.match)))
+            .map(item => escapeRegExp(item.match))
             .join('|')})`,
           'giu'
         ),
@@ -72,13 +72,41 @@ export class TextProcessor {
     if (this.plugin.settings.ignoreDiacritics) {
       text = removeDiacritics(text, this.plugin.settings.ignoreArabicDiacritics)
     }
-    const startTime = new Date().getTime()
     let match: RegExpExecArray | null = null
     let matches: SearchMatch[] = []
     let count = 0
+
+    // Build the normalized-to-original offset map once. Re-normalizing every
+    // prefix for every match becomes extremely expensive in large notes.
+    const normalizedToOriginal: number[] = [0]
+    if (this.plugin.settings.ignoreDiacritics) {
+      for (let index = 0; index < originalText.length;) {
+        const codePoint = String.fromCodePoint(originalText.codePointAt(index)!)
+        const normalized = removeDiacritics(
+          codePoint,
+          this.plugin.settings.ignoreArabicDiacritics
+        )
+        const normalizedStart = normalizedToOriginal.length - 1
+        for (let offset = 0; offset < normalized.length; offset++) {
+          normalizedToOriginal[normalizedStart + offset] = index
+        }
+        normalizedToOriginal.push(index + codePoint.length)
+        index += codePoint.length
+      }
+    }
+
+    const originalOffset = (normalizedOffset: number): number => {
+      if (!this.plugin.settings.ignoreDiacritics) return normalizedOffset
+      if (normalizedOffset <= 0) return 0
+      return normalizedToOriginal[Math.min(normalizedOffset, normalizedToOriginal.length - 1)] ?? originalText.length
+    }
+
     while ((match = reg.exec(text)) !== null) {
-      // Avoid infinite loops, stop looking after 100 matches or if we're taking too much time
+      // Avoid infinite loops and unbounded result lists.
-      if (++count >= 100 || new Date().getTime() - startTime > 50) {
+      // The regex scan is linear and the 100-match cap keeps large notes safe;
+      // a wall-clock cutoff can incorrectly discard a valid late match in
+      // very large notes.
+      if (++count >= 100) {
         warnVerbose('Stopped getMatches at', count, 'results')
         break
       }
@@ -88,9 +116,13 @@ export class TextProcessor {
       // If `ignoreDiacritics` is on, `text` may have a different length than `originalText`,
       // making `match.index` unreliable for `originalText`.
       // We use `match[0]`, which is the matched term (but without diacritics).
-      const originalMatchBeforeTrim = this.plugin.settings.ignoreDiacritics
+      // Match offsets refer to the normalized string. Translate both ends
-        ? match[0]
+      // back to the original text so excerpts and highlighting retain
-        : originalText.substring(matchStartIndex, matchEndIndex)
+      // diacritics, without repeatedly normalizing large prefixes.
+      const originalMatchBeforeTrim = originalText.substring(
+        originalOffset(matchStartIndex),
+        originalOffset(matchEndIndex)
+      )
 
       const originalMatch = originalMatchBeforeTrim.trim()
 

tests/fork-features.test.ts

@@ -0,0 +1,146 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+jest.mock('markdown-link-extractor', () => () => [])
+jest.mock('obsidian', () => {
+  const nodePath = require('node:path')
+  class TFile {
+    path = ''
+    basename = ''
+    stat = { mtime: 1 }
+    constructor(filePath: string) {
+      this.path = filePath
+      this.basename = nodePath.basename(filePath, nodePath.extname(filePath))
+    }
+  }
+
+  return {
+    TFile,
+    Notice: jest.fn(),
+    Platform: { isMacOS: false },
+    normalizePath: (value: string) => value.replaceAll('\\', '/'),
+    getAllTags: (metadata: any) => metadata?.tags ?? [],
+    parseFrontMatterAliases: (frontmatter: any) => frontmatter?.aliases ?? [],
+  }
+})
+
+import { Query } from '../src/search/query'
+import { Tokenizer } from '../src/search/tokenizer'
+import { TextProcessor } from '../src/tools/text-processing'
+import { removeDiacritics } from '../src/tools/utils'
+import { DocumentsRepository } from '../src/repositories/documents-repository'
+
+const notesDir = path.join(__dirname, 'notes')
+const readNote = (name: string) =>
+  fs.readFileSync(path.join(notesDir, name), 'utf8')
+
+const pluginFor = (overrides: Record<string, unknown> = {}) => ({
+  settings: {
+    highlight: true,
+    ignoreDiacritics: true,
+    ignoreArabicDiacritics: false,
+    renderLineReturnInExcerpts: false,
+    ...overrides,
+  },
+  getChsSegmenter: () => undefined,
+} as any)
+
+describe('fork search behavior', () => {
+  test('keeps apostrophes together in a query and highlights them', () => {
+    const query = new Query("Sun's BBQ", {
+      ignoreDiacritics: false,
+      ignoreArabicDiacritics: false,
+    })
+    expect(query.query.text).toEqual(["sun's", 'bbq'])
+
+    const processor = new TextProcessor(pluginFor())
+    const matches = processor.getMatches("Sun's BBQ", ["Sun's", 'BBQ'], query)
+    expect(matches[0].match).toBe("Sun's BBQ")
+    expect(processor.highlightText("Sun's BBQ", matches)).toContain(
+      'omnisearch-highlight'
+    )
+    expect(processor.highlightText("Sun's BBQ", matches)).toContain("Sun's BBQ")
+  })
+
+  test('filters words shorter than three characters and the documented stop words', () => {
+    const tokenizer = new Tokenizer(pluginFor())
+    const result = tokenizer.tokenizeForSearch('a an the and or but if in on at by for with to from of is it that this useful') as any
+    expect(result.queries.length).toBeGreaterThan(0)
+    expect(result.queries.flatMap((query: any) => query.queries)).toContain('useful')
+    for (const ignored of ['a', 'an', 'the', 'and', 'or', 'but', 'if', 'in', 'on', 'at', 'by', 'for', 'with', 'to', 'from', 'of', 'is', 'it', 'that', 'this']) {
+      expect(result.queries.flatMap((query: any) => query.queries)).not.toContain(ignored)
+    }
+    const onlyIgnored = tokenizer.tokenizeForSearch('x an') as any
+    expect(onlyIgnored.queries.every((query: any) => query.queries.length === 0 || query.queries[0] === 'x an')).toBe(true)
+  })
+
+  test('matches diacritic-insensitive text and preserves the original match', () => {
+    const processor = new TextProcessor(pluginFor())
+    const matches = processor.getMatches('The café serves crème brûlée.', ['cafe', 'brulee'])
+    expect(matches.map(match => match.match)).toEqual(['café', 'brûlée'])
+    expect(removeDiacritics('café crème brûlée')).toBe('cafe creme brulee')
+  })
+
+  test('renders excerpts with blank lines as readable HTML', () => {
+    const processor = new TextProcessor(pluginFor({ renderLineReturnInExcerpts: true }))
+    const excerpt = processor.makeExcerpt(readNote('fork-features.md'), readNote('fork-features.md').indexOf('café'))
+    expect(excerpt).toContain('<br>')
+    expect(excerpt).not.toMatch(/<br>\s*<br>/)
+    expect(excerpt).toContain('café')
+  })
+})
+
+describe('indexed heading metadata', () => {
+  test('indexes H1, H2, H3, and promotes only valid colon and AKA headings', async () => {
+    const { TFile } = require('obsidian')
+    const files = new Map<string, string>([
+      ['tests/notes/headers-and-lists.md', readNote('headers-and-lists.md')],
+      ['tests/notes/fork-features.md', readNote('fork-features.md')],
+      ['tests/notes/colon-variants.md', readNote('colon-variants.md')],
+      ['tests/notes/aka-variants.md', readNote('aka-variants.md')],
+    ])
+    const metadata = new Map<string, any>([
+      ['tests/notes/headers-and-lists.md', { headings: [
+        { level: 1, heading: 'H1 Heading' },
+        { level: 2, heading: 'H2 Heading' },
+        { level: 3, heading: 'H3 Heading' },
+        { level: 4, heading: 'H4 Heading' },
+        { level: 5, heading: 'H5 Heading' },
+        { level: 6, heading: 'H6 Heading' },
+      ] }],
+      ['tests/notes/fork-features.md', { headings: [] }],
+      ['tests/notes/colon-variants.md', { headings: [] }],
+      ['tests/notes/aka-variants.md', { headings: [] }],
+    ])
+    const plugin = pluginFor({ indexedFileTypes: ['md'], unsupportedFilesIndexing: 'no' })
+    plugin.notesIndexer = { isFilePlaintext: () => true, isFilenameIndexable: () => true }
+    plugin.getTextExtractor = () => undefined
+    plugin.getAIImageAnalyzer = () => undefined
+    plugin.app = {
+      vault: {
+        getAbstractFileByPath: (filePath: string) => new TFile(filePath),
+        cachedRead: async (file: any) => files.get(file.path),
+      },
+      metadataCache: {
+        getFileCache: (file: any) => metadata.get(file.path),
+      },
+    }
+
+    const repository = new DocumentsRepository(plugin)
+    const getDocument = (repository as any).getAndMapIndexedDocument.bind(repository)
+    const headings = await getDocument('tests/notes/headers-and-lists.md')
+    expect(headings.headings1).toBe('H1 Heading')
+    expect(headings.headings2).toBe('H2 Heading')
+    expect(headings.headings3).toBe('H3 Heading')
+
+    const fork = await getDocument('tests/notes/fork-features.md')
+    expect(fork.headings1).toBe('Packing List')
+    expect(fork.headings3).toBe('Japan trip')
+
+    const colon = await getDocument('tests/notes/colon-variants.md')
+    expect(colon.headings3).toBe('Colon heading')
+
+    const aka = await getDocument('tests/notes/aka-variants.md')
+    expect(aka.headings1).toBe('lowercase alias')
+  })
+})

tests/notes/aka-variants.md

@@ -0,0 +1,9 @@
+aka lowercase alias
+
+Content for lowercase alias.
+
+---
+
+AKA: uppercase alias
+
+This later alias must not be promoted because only the first paragraph is inspected.

tests/notes/colon-variants.md

@@ -0,0 +1,13 @@
+Introductory paragraph.
+
+Colon heading:
+- item one
+- item two
+
+Not a heading: this is inline content.
+
+Another colon heading:
+
+Its content starts after a blank line.
+
+Trailing text.

tests/notes/fork-features.md

@@ -0,0 +1,16 @@
+# Fork Feature Fixture
+Aka: Packing List
+
+This body contains Sun's BBQ and a blank-line excerpt target.
+
+
+The café serves crème brûlée in Montréal.
+
+Japan trip:
+- passport
+- cash
+- umbrella
+
+Words such as a an the and or but if in on at by for with to from of is it that this should not be useful search terms.
+
+The apostrophe phrase Sun's BBQ appears again for highlighting.

tests/notes/headers-and-lists.md

@@ -0,0 +1,27 @@
+# H1 Heading
+
+This note has a first-level heading and useful body content.
+
+## H2 Heading
+
+The second-level section has searchable material.
+
+### H3 Heading
+
+- first H3 list item
+- second H3 list item
+
+#### H4 Heading
+
+H4 content is intentionally present too.
+
+##### H5 Heading
+
+H5 content and a [link](https://example.com).
+
+###### H6 Heading
+
+- nested item
+  - nested child
+
+A final paragraph mentions heading terms in ordinary content.

tests/notes/title-priority.md

@@ -0,0 +1,9 @@
+# Title Priority Fixture
+
+The title-priority term appears in body content after the heading.
+
+## Heading Priority Fixture
+
+The heading-priority term appears in ordinary content too.
+
+Content-only match appears here.

tests/obsidian-mock.ts

@@ -0,0 +1,23 @@
+import path from 'node:path'
+
+export class TFile {
+  path = ''
+  basename = ''
+  stat = { mtime: 1 }
+  constructor(filePath = '') {
+    this.path = filePath
+    this.basename = path.basename(filePath, path.extname(filePath))
+  }
+}
+
+export const Notice = jest.fn()
+export const Platform = { isMacOS: false }
+export const normalizePath = (value: string) => value.replaceAll('\\', '/')
+export const getAllTags = (metadata: any) => metadata?.tags ?? []
+export const parseFrontMatterAliases = (frontmatter: any) => {
+  const aliases = frontmatter?.aliases
+  if (!aliases) return []
+  return Array.isArray(aliases)
+    ? aliases
+    : String(aliases).split(',').map(alias => alias.trim()).filter(Boolean)
+}

tests/svelte-store-mock.ts

@@ -0,0 +1,12 @@
+export const writable = <T>(value: T) => ({
+  subscribe(run: (value: T) => void) {
+    run(value)
+    return () => undefined
+  },
+  set(next: T) {
+    value = next
+  },
+  update(updateValue: (value: T) => T) {
+    value = updateValue(value)
+  },
+})

tools/obsidian-shim.ts

@@ -0,0 +1,15 @@
+export class TFile {
+  path: string
+  basename: string
+  stat = { mtime: 1 }
+  constructor(path: string) {
+    this.path = path
+    this.basename = path.split('/').pop()!.replace(/\.md$/, '')
+  }
+}
+export class Notice { constructor(..._args: unknown[]) {} }
+export class MarkdownView {}
+export const Platform = { isMacOS: false }
+export const getAllTags = (metadata: any) => metadata?.tags ?? []
+export const parseFrontMatterAliases = (frontmatter: any) => frontmatter?.aliases ?? []
+export const normalizePath = (value: string) => value.replaceAll('\\', '/')

tools/plugin-search.ts

@@ -0,0 +1,94 @@
+import { Query } from '../src/search/query'
+import { SearchEngine } from '../src/search/search-engine'
+import { DocumentsRepository } from '../src/repositories/documents-repository'
+import { TFile } from 'obsidian'
+import { TextProcessor } from '../src/tools/text-processing'
+import type { IndexedDocument } from '../src/globals'
+
+type Note = { path: string; basename: string; text: string }
+
+const defaults: any = {
+  fuzziness: '1', weightBasename: 10, weightDirectory: 7, weightH1: 6,
+  weightH2: 5, weightH3: 4, weightUnmarkedTags: 2, recencyBoost: '0',
+  downrankedFoldersFilters: [], hideExcluded: false, weightCustomProperties: [],
+  ignoreDiacritics: true, ignoreArabicDiacritics: false, simpleSearch: false,
+  maxEmbeds: 5, renderLineReturnInExcerpts: true, highlight: true,
+}
+
+function indexDocument(note: Note): IndexedDocument {
+  const lines = note.text.split('\n')
+  const headings1: string[] = [], headings2: string[] = [], headings3: string[] = []
+  for (const line of lines) {
+    const match = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/)
+    if (match && match[1].length <= 3) {
+      ;[headings1, headings2, headings3][match[1].length - 1].push(match[2])
+    }
+  }
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim()
+    const previous = i > 0 ? lines[i - 1].trim() : null
+    const next = i < lines.length - 1 ? lines[i + 1].trim() : null
+    if (line.endsWith(':') && previous === '' && next !== null && next !== '') {
+      headings3.push(line.slice(0, -1).trim())
+    }
+  }
+  return {
+    path: note.path, basename: note.basename, displayTitle: '', mtime: 1,
+    content: note.text, cleanedContent: note.text, aliases: '', tags: [],
+    unmarkedTags: [], headings1: headings1.join(' '), headings2: headings2.join(' '),
+    headings3: headings3.join(' '),
+  }
+}
+
+export async function createPluginSearch(notes: Note[]) {
+  const noteMap = new Map(notes.map(note => [note.path, note]))
+  const metadata = new Map(notes.map(note => [note.path, {
+    headings: note.text.split('\n').flatMap((line, index) => {
+      const match = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/)
+      return match ? [{ level: match[1].length, heading: match[2], position: {
+        start: { offset: note.text.split('\n').slice(0, index).join('\n').length + (index ? 1 : 0) },
+        end: { offset: note.text.length },
+      } }] : []
+    }),
+    links: [],
+  }]))
+  const plugin: any = {
+    settings: defaults,
+    app: {
+      vault: {
+        getAbstractFileByPath: (path: string) => new (TFile as any)(path),
+        cachedRead: async (file: TFile) => noteMap.get(file.path)?.text ?? '',
+      },
+      metadataCache: {
+        getCache: (path: string) => metadata.get(path),
+        getFileCache: (file: TFile) => metadata.get(file.path),
+        isUserIgnored: () => false,
+      },
+    },
+    notesIndexer: { isFilePlaintext: () => true, isFilenameIndexable: () => true },
+    getChsSegmenter: () => undefined,
+    getTextExtractor: () => undefined,
+    getAIImageAnalyzer: () => undefined,
+    embedsRepository: { refreshEmbedsForNote: () => undefined, getEmbeds: () => [] },
+  }
+  plugin.textProcessor = new TextProcessor(plugin)
+  plugin.documentsRepository = new DocumentsRepository(plugin)
+  const engine = new SearchEngine(plugin)
+  await engine.addFromPaths(notes.map(note => note.path))
+  return async (text: string) => {
+    const query = new Query(text, { ignoreDiacritics: true, ignoreArabicDiacritics: false })
+    const results = await engine.getSuggestions(query)
+    return results.map(result => {
+      const offset = result.matches[0]?.offset ?? -1
+      const excerpt = plugin.textProcessor.makeExcerpt(result.content, offset)
+      return {
+        id: result.path,
+        terms: result.foundWords,
+        matches: result.matches,
+        score: result.score,
+        offset,
+        excerpt,
+      }
+    })
+  }
+}

tools/search-tui-launcher.js

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+const os = require('node:os')
+const path = require('node:path')
+const { buildSync } = require('esbuild')
+
+const output = path.join(os.tmpdir(), `tannersearch-plugin-search-${process.pid}.cjs`)
+buildSync({
+  entryPoints: [path.join(__dirname, 'plugin-search.ts')],
+  bundle: true,
+  platform: 'node',
+  format: 'cjs',
+  outfile: output,
+  alias: { obsidian: path.join(__dirname, 'obsidian-shim.ts') },
+  logLevel: 'silent',
+})
+process.env.TUI_PLUGIN_SEARCH = output
+require('./search-tui.js')

tools/search-tui.js

@@ -0,0 +1,441 @@
+#!/usr/bin/env node
+
+/**
+ * Small terminal harness for exercising the search experience without Obsidian.
+ * It intentionally uses only Node's standard library so it can be run from a
+ * checkout with: npm run tui [directory]
+ */
+const fs = require('node:fs')
+const path = require('node:path')
+const MiniSearch = require('minisearch')
+
+const DEFAULT_DIRECTORY = path.resolve(__dirname, '..', 'tests', 'notes')
+const STOP_WORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'but', 'if', 'in', 'on', 'at', 'by', 'for',
+  'with', 'to', 'from', 'of', 'is', 'it', 'that', 'this',
+])
+const ANSI = {
+  clear: '\x1b[2J\x1b[H',
+  bold: '\x1b[1m',
+  dim: '\x1b[2m',
+  cyan: '\x1b[36m',
+  yellow: '\x1b[33m',
+  green: '\x1b[32m',
+  inverse: '\x1b[7m',
+  hideCursor: '\x1b[?25l',
+  showCursor: '\x1b[?25h',
+  reset: '\x1b[0m',
+}
+
+function walk(directory) {
+  const notes = []
+  for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
+    const filePath = path.join(directory, entry.name)
+    if (entry.isDirectory()) notes.push(...walk(filePath))
+    else if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) notes.push(filePath)
+  }
+  return notes
+}
+
+function removeDiacritics(value) {
+  return value.normalize('NFD').replace(/\p{Diacritic}/gu, '').normalize('NFC')
+}
+
+// Mirrors the plugin's indexing tokenizer. In particular, apostrophes are
+// preserved as part of a token rather than being treated as separators.
+function tokenize(text) {
+  return text
+    .split(/[|\[\]()<>{} \t\n\r]+/u)
+    .flatMap(word => [word, ...word.split(/[.,:;!?/\\_#%&*=^@-]+/u)])
+    .filter(Boolean)
+}
+
+function processTerm(term) {
+  const processed = removeDiacritics(term).toLowerCase()
+  return processed.length < 3 || STOP_WORDS.has(processed) ? null : processed
+}
+
+function headingFields(text) {
+  const lines = text.split(/\r?\n/)
+  const fields = { headings1: [], headings2: [], headings3: [], markdownHeadings: [] }
+  for (let index = 0; index < lines.length; index++) {
+    const match = lines[index].match(/^(#{1,6})\s+(.+?)\s*#*\s*$/)
+    if (!match) continue
+    const level = match[1].length
+    if (level <= 3) fields[`headings${level}`].push(match[2])
+    fields.markdownHeadings.push(match[2])
+  }
+  // Match DocumentsRepository's colon-heading promotion rule.
+  for (let index = 0; index < lines.length; index++) {
+    const line = lines[index].trim()
+    const previous = index > 0 ? lines[index - 1].trim() : null
+    const next = index < lines.length - 1 ? lines[index + 1].trim() : null
+    if (line.endsWith(':') && previous === '' && next !== null && next !== '') {
+      fields.headings3.push(line.slice(0, -1).trim())
+    }
+  }
+  return fields
+}
+
+function buildIndex(notes) {
+  const index = new MiniSearch({
+    idField: 'path',
+    fields: ['basename', 'directory', 'aliases', 'content', 'headings1', 'headings2', 'headings3'],
+    tokenize,
+    processTerm,
+  })
+  index.addAll(notes.map(note => {
+    const fields = headingFields(note.text)
+    return {
+      path: note.path,
+      basename: note.basename,
+      directory: path.dirname(note.relative),
+      aliases: '',
+      content: note.text,
+      headings1: fields.headings1.join(' '),
+      headings2: fields.headings2.join(' '),
+      headings3: fields.headings3.join(' '),
+    }
+  }))
+  return index
+}
+
+function search(index, notesByPath, query) {
+  const terms = tokenize(query).map(processTerm).filter(Boolean)
+  if (!terms.length) return []
+  const results = index.search(terms.join(' '), {
+    prefix: term => term.length >= 1,
+    bm25: { b: 0.2, d: 0.5, k: 1.2 },
+    fuzzy: term => term.length <= 4 ? 0 : term.length <= 5 ? 0.05 : 0.1,
+    boost: { basename: 10, directory: 7, headings1: 6, headings2: 5, headings3: 4 },
+  })
+  return results.map(result => {
+    const note = notesByPath.get(result.id)
+    const lower = note.text.toLocaleLowerCase()
+    const fields = headingFields(note.text)
+    const titleTerm = terms.find(term => note.basename.toLocaleLowerCase().includes(term))
+    const headingTerm = result.terms.find(term => fields.markdownHeadings.some(heading => heading.toLowerCase().includes(term)))
+    const offset = titleTerm ? 0 : headingTerm
+      ? lower.indexOf(headingTerm)
+      : Math.max(0, lower.indexOf(result.terms[0] || terms[0]))
+    return { note, terms: result.terms.length ? result.terms : terms, score: result.score, offset }
+  })
+}
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+function htmlExcerptToTerminal(html) {
+  const text = html
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<span[^>]*>([\s\S]*?)<\/span>/gi, `${ANSI.yellow}${ANSI.bold}$1${ANSI.reset}${ANSI.dim}`)
+    .replace(/<[^>]+>/g, '')
+    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"').replace(/&#039;/g, "'")
+  const unindented = text.split('\n').map(line => line.replace(/^\s+/, ''))
+  return ANSI.dim + unindented.slice(0, 3).join('\n') + ANSI.reset
+}
+
+function highlight(text, terms, restore = '') {
+  if (!terms.length) return text
+  const expression = new RegExp(`(${terms.sort((a, b) => b.length - a.length).map(escapeRegExp).join('|')})`, 'giu')
+  return text.replace(expression, `${ANSI.yellow}${ANSI.bold}$1${ANSI.reset}${restore}`)
+}
+
+// Use the exact matches returned by TextProcessor rather than MiniSearch's
+// terms. This preserves complete matches such as "OCD" and "OCD's".
+function highlightMatches(text, matches, additionalTerms = []) {
+  const terms = [...new Set([
+    ...matches.map(match => match.match),
+    ...additionalTerms,
+  ])].sort((a, b) => b.length - a.length)
+  if (!terms.length) return text
+  const expression = new RegExp(`(${terms.map(escapeRegExp).join('|')})`, 'giu')
+  return text.replace(expression, `${ANSI.yellow}${ANSI.bold}$1${ANSI.reset}${ANSI.dim}`)
+}
+
+function excerpt(note, result) {
+  // Obsidian's result preview is short and keeps the note's line breaks.
+  const lines = note.text.split(/\r?\n/)
+  let offset = 0
+  let matchLine = 0
+  for (let index = 0; index < lines.length; index++) {
+    if (result.offset <= offset + lines[index].length) {
+      matchLine = index
+      break
+    }
+    offset += lines[index].length + 1
+  }
+  const firstLine = Math.max(0, matchLine - 1)
+  const lastLine = Math.min(lines.length, firstLine + 3)
+  const visible = lines.slice(firstLine, lastLine)
+  const prefix = firstLine ? '…' : ''
+  const suffix = lastLine < lines.length ? '…' : ''
+  return ANSI.dim + prefix + visible.map(line => highlight(line, result.terms, ANSI.dim)).join('\n') + suffix + ANSI.reset
+}
+
+function loadNotes(directory) {
+  return walk(directory).map(filePath => ({
+    path: filePath,
+    basename: path.basename(filePath, '.md'),
+    relative: path.relative(directory, filePath),
+    text: fs.readFileSync(filePath, 'utf8'),
+  }))
+}
+
+class SearchTUI {
+  constructor(directory) {
+    this.directory = directory
+    this.pluginSearch = null
+    this.searchGeneration = 0
+    this.notes = loadNotes(directory)
+    this.notesByPath = new Map(this.notes.map(note => [note.path, note]))
+    this.index = buildIndex(this.notes)
+    this.query = ''
+    this.results = []
+    this.selected = 0
+    this.scrollOffset = 0
+    this.mode = 'search'
+    this.openResult = null
+    this.noteScrollLine = 0
+  }
+
+  async init() {
+    if (process.env.TUI_PLUGIN_SEARCH) {
+      const { createPluginSearch } = require(process.env.TUI_PLUGIN_SEARCH)
+      this.pluginSearch = await createPluginSearch(this.notes.map(note => ({
+        path: note.relative,
+        basename: note.basename,
+        text: note.text,
+      })))
+    }
+    return this
+  }
+
+  start() {
+    if (!process.stdin.isTTY || !process.stdout.isTTY) {
+      throw new Error('The search TUI requires an interactive terminal (TTY).')
+    }
+    process.stdin.setRawMode(true)
+    process.stdout.write(ANSI.hideCursor)
+    process.stdin.resume()
+    process.stdin.setEncoding('utf8')
+    process.stdin.on('data', key => this.onKey(key))
+    this.render()
+  }
+
+  stop() {
+    process.stdin.setRawMode(false)
+    process.stdin.pause()
+    process.stdin.removeAllListeners('data')
+    // DocumentsRepository owns a repeating cache timer, so merely pausing
+    // stdin leaves the process alive. Restore the terminal and exit explicitly.
+    process.stdout.write(`${ANSI.showCursor}\n`)
+    process.exit(0)
+  }
+
+  onKey(key) {
+    if (key === '\u0003') return this.stop() // Ctrl-C
+    if (this.mode === 'note') {
+      if (key === 'q' || key === '\u001b' || key === '\u001b\u001b') {
+        this.mode = 'search'
+        this.render()
+      } else if (key === '\u001b[A') {
+        this.noteScrollLine = Math.max(0, this.noteScrollLine - 1)
+        this.render()
+      } else if (key === '\u001b[B') {
+        this.noteScrollLine += 1
+        this.ensureNoteScrollBounds()
+        this.render()
+      } else if (key === '\u001b[5~') {
+        this.noteScrollLine = Math.max(0, this.noteScrollLine - this.visibleNoteLines())
+        this.render()
+      } else if (key === '\u001b[6~') {
+        this.noteScrollLine += this.visibleNoteLines()
+        this.ensureNoteScrollBounds()
+        this.render()
+      } else if (key === '\u001b[H' || key === '\u001b[1~') {
+        this.noteScrollLine = 0
+        this.render()
+      } else if (key === '\u001b[F' || key === '\u001b[4~') {
+        this.noteScrollLine = Number.MAX_SAFE_INTEGER
+        this.ensureNoteScrollBounds()
+        this.render()
+      }
+      return
+    }
+    if (key === '\u0008' || key === '\u007f') {
+      this.query = this.query.slice(0, -1)
+      this.updateResults()
+    } else if (key === '\r' || key === '\n') {
+      if (this.results.length) {
+        this.openResult = this.results[this.selected]
+        this.mode = 'note'
+        this.noteScrollLine = 0
+        this.ensureNoteCursorVisible()
+        this.render()
+      }
+    } else if (key === '\u001b[A') {
+      this.selected = Math.max(0, this.selected - 1)
+      this.ensureSelectionVisible()
+      this.render()
+    } else if (key === '\u001b[B') {
+      this.selected = Math.min(Math.max(0, this.results.length - 1), this.selected + 1)
+      this.ensureSelectionVisible()
+      this.render()
+    } else if (key === '\u001b[5~') {
+      this.selected = Math.max(0, this.selected - this.visibleResultCount())
+      this.ensureSelectionVisible()
+      this.render()
+    } else if (key === '\u001b[6~') {
+      this.selected = Math.min(Math.max(0, this.results.length - 1), this.selected + this.visibleResultCount())
+      this.ensureSelectionVisible()
+      this.render()
+    } else if (key === '\u001b[H' || key === '\u001b[1~') {
+      this.selected = 0
+      this.ensureSelectionVisible()
+      this.render()
+    } else if (key === '\u001b[F' || key === '\u001b[4~') {
+      this.selected = Math.max(0, this.results.length - 1)
+      this.ensureSelectionVisible()
+      this.render()
+    } else if (key === '\u001b') {
+      this.query = ''
+      this.updateResults()
+    } else if (key >= ' ' && key !== '\u007f') {
+      this.query += key
+      this.updateResults()
+    }
+  }
+
+  visibleNoteLines() {
+    return Math.max(1, (process.stdout.rows || 24) - 6)
+  }
+
+  noteCursorPosition() {
+    if (!this.openResult) return { line: 0, column: 0 }
+    let remaining = this.openResult.offset
+    const lines = this.openResult.note.text.split(/\r?\n/)
+    for (let line = 0; line < lines.length; line++) {
+      if (remaining <= lines[line].length) return { line, column: Math.max(0, remaining) }
+      remaining -= lines[line].length + 1
+    }
+    return { line: lines.length - 1, column: lines.at(-1)?.length ?? 0 }
+  }
+
+  ensureNoteScrollBounds() {
+    if (!this.openResult) return
+    const lineCount = this.openResult.note.text.split(/\r?\n/).length
+    this.noteScrollLine = Math.max(0, Math.min(this.noteScrollLine, Math.max(0, lineCount - this.visibleNoteLines())))
+  }
+
+  ensureNoteCursorVisible() {
+    const cursor = this.noteCursorPosition()
+    const visible = this.visibleNoteLines()
+    this.noteScrollLine = cursor.line - Math.floor(visible / 2)
+    this.ensureNoteScrollBounds()
+  }
+
+  visibleResultCount() {
+    // Each result occupies a title line, three excerpt lines, and a spacer.
+    // Keep the header and help text visible while scrolling.
+    return Math.max(1, Math.floor(((process.stdout.rows || 24) - 6) / 5))
+  }
+
+  ensureSelectionVisible() {
+    const visible = this.visibleResultCount()
+    if (this.selected < this.scrollOffset) this.scrollOffset = this.selected
+    if (this.selected >= this.scrollOffset + visible) {
+      this.scrollOffset = this.selected - visible + 1
+    }
+    this.scrollOffset = Math.max(0, Math.min(
+      this.scrollOffset,
+      Math.max(0, this.results.length - visible),
+    ))
+  }
+
+  async updateResults() {
+    const generation = ++this.searchGeneration
+    if (this.pluginSearch) {
+      const ranked = await this.pluginSearch(this.query)
+      if (generation !== this.searchGeneration) return
+      this.results = ranked.map(result => {
+        const note = this.notes.find(candidate => candidate.relative === result.id)
+        return {
+          note,
+          terms: result.terms,
+          matches: result.matches,
+          score: result.score,
+          offset: result.offset,
+          excerpt: highlightMatches(
+            htmlExcerptToTerminal(result.excerpt),
+            result.matches,
+            result.terms,
+          ),
+        }
+      }).filter(result => result.note)
+    } else {
+      this.results = search(this.index, this.notesByPath, this.query)
+    }
+    this.selected = Math.min(this.selected, Math.max(0, this.results.length - 1))
+    this.ensureSelectionVisible()
+    this.render()
+  }
+
+  render() {
+    process.stdout.write(ANSI.clear)
+    if (this.mode === 'note') return this.renderNote()
+    process.stdout.write(`${ANSI.bold}${ANSI.cyan}Tannersearch TUI${ANSI.reset}  ${ANSI.dim}${this.directory}${ANSI.reset}\n\n`)
+    process.stdout.write(`${ANSI.bold}Search:${ANSI.reset} ${this.query}${ANSI.cyan}▌${ANSI.reset}\n`)
+    process.stdout.write(`${ANSI.dim}Type to search · ↑/↓ select · Enter open · Esc clear · Ctrl-C quit${ANSI.reset}\n\n`)
+    if (!this.query) {
+      process.stdout.write(`${ANSI.dim}Start typing to search ${this.notes.length} Markdown notes.${ANSI.reset}\n`)
+      return
+    }
+    if (!this.results.length) {
+      process.stdout.write(`${ANSI.yellow}No matching notes.${ANSI.reset}\n`)
+      return
+    }
+    const visible = this.visibleResultCount()
+    this.ensureSelectionVisible()
+    const shown = this.results.slice(this.scrollOffset, this.scrollOffset + visible)
+    shown.forEach((result, visibleIndex) => {
+      const index = this.scrollOffset + visibleIndex
+      const marker = index === this.selected ? `${ANSI.inverse}>${ANSI.reset}` : ' '
+      process.stdout.write(`${marker} ${ANSI.bold}${result.note.basename}${ANSI.reset} ${ANSI.dim}(${result.note.relative})${ANSI.reset}\n`)
+      process.stdout.write(`${result.excerpt || excerpt(result.note, result)}\n\n`)
+    })
+    if (this.results.length > visible) {
+      process.stdout.write(`${ANSI.dim}Showing ${this.scrollOffset + 1}-${Math.min(this.scrollOffset + visible, this.results.length)} of ${this.results.length} · PageUp/PageDown to scroll${ANSI.reset}\n`)
+    }
+  }
+
+  renderNote() {
+    const result = this.openResult
+    const note = result.note
+    process.stdout.write(`${ANSI.bold}${ANSI.green}${note.basename}${ANSI.reset} ${ANSI.dim}— fake cursor at match${ANSI.reset}\n`)
+    process.stdout.write(`${ANSI.dim}Press q or Esc to return to results.${ANSI.reset}\n\n`)
+    const lines = note.text.split(/\r?\n/)
+    const cursor = this.noteCursorPosition()
+    const visible = lines.slice(this.noteScrollLine, this.noteScrollLine + this.visibleNoteLines())
+    visible.forEach((line, index) => {
+      const lineNumber = this.noteScrollLine + index
+      if (lineNumber === cursor.line) {
+        process.stdout.write(`${highlight(line.slice(0, cursor.column), result.terms)}${ANSI.green}▌${ANSI.reset}${highlight(line.slice(cursor.column), result.terms)}\n`)
+        process.stdout.write(`${' '.repeat(cursor.column)}${ANSI.green}^ cursor${ANSI.reset}\n`)
+      } else {
+        process.stdout.write(`${highlight(line, result.terms)}\n`)
+      }
+    })
+  }
+}
+
+const directory = path.resolve(process.argv[2] || DEFAULT_DIRECTORY)
+try {
+  if (!fs.statSync(directory).isDirectory()) throw new Error(`Not a directory: ${directory}`)
+  new SearchTUI(directory).init().then(tui => tui.start())
+} catch (error) {
+  console.error(`search-tui: ${error.message}`)
+  process.exitCode = 1
+}