Add tests and search simulator TUI

tests/fork-features.test.ts

@@ -0,0 +1,146 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+jest.mock('markdown-link-extractor', () => () => [])
+jest.mock('obsidian', () => {
+  const nodePath = require('node:path')
+  class TFile {
+    path = ''
+    basename = ''
+    stat = { mtime: 1 }
+    constructor(filePath: string) {
+      this.path = filePath
+      this.basename = nodePath.basename(filePath, nodePath.extname(filePath))
+    }
+  }
+
+  return {
+    TFile,
+    Notice: jest.fn(),
+    Platform: { isMacOS: false },
+    normalizePath: (value: string) => value.replaceAll('\\', '/'),
+    getAllTags: (metadata: any) => metadata?.tags ?? [],
+    parseFrontMatterAliases: (frontmatter: any) => frontmatter?.aliases ?? [],
+  }
+})
+
+import { Query } from '../src/search/query'
+import { Tokenizer } from '../src/search/tokenizer'
+import { TextProcessor } from '../src/tools/text-processing'
+import { removeDiacritics } from '../src/tools/utils'
+import { DocumentsRepository } from '../src/repositories/documents-repository'
+
+const notesDir = path.join(__dirname, 'notes')
+const readNote = (name: string) =>
+  fs.readFileSync(path.join(notesDir, name), 'utf8')
+
+const pluginFor = (overrides: Record<string, unknown> = {}) => ({
+  settings: {
+    highlight: true,
+    ignoreDiacritics: true,
+    ignoreArabicDiacritics: false,
+    renderLineReturnInExcerpts: false,
+    ...overrides,
+  },
+  getChsSegmenter: () => undefined,
+} as any)
+
+describe('fork search behavior', () => {
+  test('keeps apostrophes together in a query and highlights them', () => {
+    const query = new Query("Sun's BBQ", {
+      ignoreDiacritics: false,
+      ignoreArabicDiacritics: false,
+    })
+    expect(query.query.text).toEqual(["sun's", 'bbq'])
+
+    const processor = new TextProcessor(pluginFor())
+    const matches = processor.getMatches("Sun's BBQ", ["Sun's", 'BBQ'], query)
+    expect(matches[0].match).toBe("Sun's BBQ")
+    expect(processor.highlightText("Sun's BBQ", matches)).toContain(
+      'omnisearch-highlight'
+    )
+    expect(processor.highlightText("Sun's BBQ", matches)).toContain("Sun's BBQ")
+  })
+
+  test('filters words shorter than three characters and the documented stop words', () => {
+    const tokenizer = new Tokenizer(pluginFor())
+    const result = tokenizer.tokenizeForSearch('a an the and or but if in on at by for with to from of is it that this useful') as any
+    expect(result.queries.length).toBeGreaterThan(0)
+    expect(result.queries.flatMap((query: any) => query.queries)).toContain('useful')
+    for (const ignored of ['a', 'an', 'the', 'and', 'or', 'but', 'if', 'in', 'on', 'at', 'by', 'for', 'with', 'to', 'from', 'of', 'is', 'it', 'that', 'this']) {
+      expect(result.queries.flatMap((query: any) => query.queries)).not.toContain(ignored)
+    }
+    const onlyIgnored = tokenizer.tokenizeForSearch('x an') as any
+    expect(onlyIgnored.queries.every((query: any) => query.queries.length === 0 || query.queries[0] === 'x an')).toBe(true)
+  })
+
+  test('matches diacritic-insensitive text and preserves the original match', () => {
+    const processor = new TextProcessor(pluginFor())
+    const matches = processor.getMatches('The café serves crème brûlée.', ['cafe', 'brulee'])
+    expect(matches.map(match => match.match)).toEqual(['café', 'brûlée'])
+    expect(removeDiacritics('café crème brûlée')).toBe('cafe creme brulee')
+  })
+
+  test('renders excerpts with blank lines as readable HTML', () => {
+    const processor = new TextProcessor(pluginFor({ renderLineReturnInExcerpts: true }))
+    const excerpt = processor.makeExcerpt(readNote('fork-features.md'), readNote('fork-features.md').indexOf('café'))
+    expect(excerpt).toContain('<br>')
+    expect(excerpt).not.toMatch(/<br>\s*<br>/)
+    expect(excerpt).toContain('café')
+  })
+})
+
+describe('indexed heading metadata', () => {
+  test('indexes H1, H2, H3, and promotes only valid colon and AKA headings', async () => {
+    const { TFile } = require('obsidian')
+    const files = new Map<string, string>([
+      ['tests/notes/headers-and-lists.md', readNote('headers-and-lists.md')],
+      ['tests/notes/fork-features.md', readNote('fork-features.md')],
+      ['tests/notes/colon-variants.md', readNote('colon-variants.md')],
+      ['tests/notes/aka-variants.md', readNote('aka-variants.md')],
+    ])
+    const metadata = new Map<string, any>([
+      ['tests/notes/headers-and-lists.md', { headings: [
+        { level: 1, heading: 'H1 Heading' },
+        { level: 2, heading: 'H2 Heading' },
+        { level: 3, heading: 'H3 Heading' },
+        { level: 4, heading: 'H4 Heading' },
+        { level: 5, heading: 'H5 Heading' },
+        { level: 6, heading: 'H6 Heading' },
+      ] }],
+      ['tests/notes/fork-features.md', { headings: [] }],
+      ['tests/notes/colon-variants.md', { headings: [] }],
+      ['tests/notes/aka-variants.md', { headings: [] }],
+    ])
+    const plugin = pluginFor({ indexedFileTypes: ['md'], unsupportedFilesIndexing: 'no' })
+    plugin.notesIndexer = { isFilePlaintext: () => true, isFilenameIndexable: () => true }
+    plugin.getTextExtractor = () => undefined
+    plugin.getAIImageAnalyzer = () => undefined
+    plugin.app = {
+      vault: {
+        getAbstractFileByPath: (filePath: string) => new TFile(filePath),
+        cachedRead: async (file: any) => files.get(file.path),
+      },
+      metadataCache: {
+        getFileCache: (file: any) => metadata.get(file.path),
+      },
+    }
+
+    const repository = new DocumentsRepository(plugin)
+    const getDocument = (repository as any).getAndMapIndexedDocument.bind(repository)
+    const headings = await getDocument('tests/notes/headers-and-lists.md')
+    expect(headings.headings1).toBe('H1 Heading')
+    expect(headings.headings2).toBe('H2 Heading')
+    expect(headings.headings3).toBe('H3 Heading')
+
+    const fork = await getDocument('tests/notes/fork-features.md')
+    expect(fork.headings1).toBe('Packing List')
+    expect(fork.headings3).toBe('Japan trip')
+
+    const colon = await getDocument('tests/notes/colon-variants.md')
+    expect(colon.headings3).toBe('Colon heading')
+
+    const aka = await getDocument('tests/notes/aka-variants.md')
+    expect(aka.headings1).toBe('lowercase alias')
+  })
+})

tests/notes/aka-variants.md

@@ -0,0 +1,9 @@
+aka lowercase alias
+
+Content for lowercase alias.
+
+---
+
+AKA: uppercase alias
+
+This later alias must not be promoted because only the first paragraph is inspected.

tests/notes/colon-variants.md

@@ -0,0 +1,13 @@
+Introductory paragraph.
+
+Colon heading:
+- item one
+- item two
+
+Not a heading: this is inline content.
+
+Another colon heading:
+
+Its content starts after a blank line.
+
+Trailing text.

tests/notes/fork-features.md

@@ -0,0 +1,16 @@
+# Fork Feature Fixture
+Aka: Packing List
+
+This body contains Sun's BBQ and a blank-line excerpt target.
+
+
+The café serves crème brûlée in Montréal.
+
+Japan trip:
+- passport
+- cash
+- umbrella
+
+Words such as a an the and or but if in on at by for with to from of is it that this should not be useful search terms.
+
+The apostrophe phrase Sun's BBQ appears again for highlighting.

tests/notes/headers-and-lists.md

@@ -0,0 +1,27 @@
+# H1 Heading
+
+This note has a first-level heading and useful body content.
+
+## H2 Heading
+
+The second-level section has searchable material.
+
+### H3 Heading
+
+- first H3 list item
+- second H3 list item
+
+#### H4 Heading
+
+H4 content is intentionally present too.
+
+##### H5 Heading
+
+H5 content and a [link](https://example.com).
+
+###### H6 Heading
+
+- nested item
+  - nested child
+
+A final paragraph mentions heading terms in ordinary content.

tests/notes/title-priority.md

@@ -0,0 +1,9 @@
+# Title Priority Fixture
+
+The title-priority term appears in body content after the heading.
+
+## Heading Priority Fixture
+
+The heading-priority term appears in ordinary content too.
+
+Content-only match appears here.

tests/obsidian-mock.ts

@@ -0,0 +1,23 @@
+import path from 'node:path'
+
+export class TFile {
+  path = ''
+  basename = ''
+  stat = { mtime: 1 }
+  constructor(filePath = '') {
+    this.path = filePath
+    this.basename = path.basename(filePath, path.extname(filePath))
+  }
+}
+
+export const Notice = jest.fn()
+export const Platform = { isMacOS: false }
+export const normalizePath = (value: string) => value.replaceAll('\\', '/')
+export const getAllTags = (metadata: any) => metadata?.tags ?? []
+export const parseFrontMatterAliases = (frontmatter: any) => {
+  const aliases = frontmatter?.aliases
+  if (!aliases) return []
+  return Array.isArray(aliases)
+    ? aliases
+    : String(aliases).split(',').map(alias => alias.trim()).filter(Boolean)
+}

tests/svelte-store-mock.ts

@@ -0,0 +1,12 @@
+export const writable = <T>(value: T) => ({
+  subscribe(run: (value: T) => void) {
+    run(value)
+    return () => undefined
+  },
+  set(next: T) {
+    value = next
+  },
+  update(updateValue: (value: T) => T) {
+    value = updateValue(value)
+  },
+})