diff --git a/jest.config.js b/jest.config.js index 6a9cbef..bcbcc90 100644 --- a/jest.config.js +++ b/jest.config.js @@ -82,7 +82,11 @@ module.exports = { ], // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module - // moduleNameMapper: {}, + moduleNameMapper: { + '^obsidian$': '/tests/obsidian-mock.ts', + '^svelte/store$': '/tests/svelte-store-mock.ts', + '^lodash-es$': 'lodash', + }, // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader // modulePathIgnorePatterns: [], diff --git a/package.json b/package.json index 3beacc4..0d2e582 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production", "check": "svelte-check --tsconfig ./tsconfig.json", "version": "node version-bump.mjs", - "test": "jest" + "test": "jest", + "tui": "node tools/search-tui-launcher.js" }, "keywords": [], "author": "Simon Cambier", diff --git a/src/tools/text-processing.ts b/src/tools/text-processing.ts index b9ea0f0..ecd94c1 100644 --- a/src/tools/text-processing.ts +++ b/src/tools/text-processing.ts @@ -26,7 +26,7 @@ export class TextProcessor { return text.replace( new RegExp( `(${matches - .map(item => escapeRegExp(escapeHTML(item.match))) + .map(item => escapeRegExp(item.match)) .join('|')})`, 'giu' ), @@ -72,13 +72,41 @@ export class TextProcessor { if (this.plugin.settings.ignoreDiacritics) { text = removeDiacritics(text, this.plugin.settings.ignoreArabicDiacritics) } - const startTime = new Date().getTime() let match: RegExpExecArray | null = null let matches: SearchMatch[] = [] let count = 0 + + // Build the normalized-to-original offset map once. Re-normalizing every + // prefix for every match becomes extremely expensive in large notes. + const normalizedToOriginal: number[] = [0] + if (this.plugin.settings.ignoreDiacritics) { + for (let index = 0; index < originalText.length;) { + const codePoint = String.fromCodePoint(originalText.codePointAt(index)!) + const normalized = removeDiacritics( + codePoint, + this.plugin.settings.ignoreArabicDiacritics + ) + const normalizedStart = normalizedToOriginal.length - 1 + for (let offset = 0; offset < normalized.length; offset++) { + normalizedToOriginal[normalizedStart + offset] = index + } + normalizedToOriginal.push(index + codePoint.length) + index += codePoint.length + } + } + + const originalOffset = (normalizedOffset: number): number => { + if (!this.plugin.settings.ignoreDiacritics) return normalizedOffset + if (normalizedOffset <= 0) return 0 + return normalizedToOriginal[Math.min(normalizedOffset, normalizedToOriginal.length - 1)] ?? originalText.length + } + while ((match = reg.exec(text)) !== null) { - // Avoid infinite loops, stop looking after 100 matches or if we're taking too much time - if (++count >= 100 || new Date().getTime() - startTime > 50) { + // Avoid infinite loops and unbounded result lists. + // The regex scan is linear and the 100-match cap keeps large notes safe; + // a wall-clock cutoff can incorrectly discard a valid late match in + // very large notes. + if (++count >= 100) { warnVerbose('Stopped getMatches at', count, 'results') break } @@ -88,9 +116,13 @@ export class TextProcessor { // If `ignoreDiacritics` is on, `text` may have a different length than `originalText`, // making `match.index` unreliable for `originalText`. // We use `match[0]`, which is the matched term (but without diacritics). - const originalMatchBeforeTrim = this.plugin.settings.ignoreDiacritics - ? match[0] - : originalText.substring(matchStartIndex, matchEndIndex) + // Match offsets refer to the normalized string. Translate both ends + // back to the original text so excerpts and highlighting retain + // diacritics, without repeatedly normalizing large prefixes. + const originalMatchBeforeTrim = originalText.substring( + originalOffset(matchStartIndex), + originalOffset(matchEndIndex) + ) const originalMatch = originalMatchBeforeTrim.trim() diff --git a/tests/fork-features.test.ts b/tests/fork-features.test.ts new file mode 100644 index 0000000..3cbff1f --- /dev/null +++ b/tests/fork-features.test.ts @@ -0,0 +1,146 @@ +import fs from 'node:fs' +import path from 'node:path' + +jest.mock('markdown-link-extractor', () => () => []) +jest.mock('obsidian', () => { + const nodePath = require('node:path') + class TFile { + path = '' + basename = '' + stat = { mtime: 1 } + constructor(filePath: string) { + this.path = filePath + this.basename = nodePath.basename(filePath, nodePath.extname(filePath)) + } + } + + return { + TFile, + Notice: jest.fn(), + Platform: { isMacOS: false }, + normalizePath: (value: string) => value.replaceAll('\\', '/'), + getAllTags: (metadata: any) => metadata?.tags ?? [], + parseFrontMatterAliases: (frontmatter: any) => frontmatter?.aliases ?? [], + } +}) + +import { Query } from '../src/search/query' +import { Tokenizer } from '../src/search/tokenizer' +import { TextProcessor } from '../src/tools/text-processing' +import { removeDiacritics } from '../src/tools/utils' +import { DocumentsRepository } from '../src/repositories/documents-repository' + +const notesDir = path.join(__dirname, 'notes') +const readNote = (name: string) => + fs.readFileSync(path.join(notesDir, name), 'utf8') + +const pluginFor = (overrides: Record = {}) => ({ + settings: { + highlight: true, + ignoreDiacritics: true, + ignoreArabicDiacritics: false, + renderLineReturnInExcerpts: false, + ...overrides, + }, + getChsSegmenter: () => undefined, +} as any) + +describe('fork search behavior', () => { + test('keeps apostrophes together in a query and highlights them', () => { + const query = new Query("Sun's BBQ", { + ignoreDiacritics: false, + ignoreArabicDiacritics: false, + }) + expect(query.query.text).toEqual(["sun's", 'bbq']) + + const processor = new TextProcessor(pluginFor()) + const matches = processor.getMatches("Sun's BBQ", ["Sun's", 'BBQ'], query) + expect(matches[0].match).toBe("Sun's BBQ") + expect(processor.highlightText("Sun's BBQ", matches)).toContain( + 'omnisearch-highlight' + ) + expect(processor.highlightText("Sun's BBQ", matches)).toContain("Sun's BBQ") + }) + + test('filters words shorter than three characters and the documented stop words', () => { + const tokenizer = new Tokenizer(pluginFor()) + const result = tokenizer.tokenizeForSearch('a an the and or but if in on at by for with to from of is it that this useful') as any + expect(result.queries.length).toBeGreaterThan(0) + expect(result.queries.flatMap((query: any) => query.queries)).toContain('useful') + for (const ignored of ['a', 'an', 'the', 'and', 'or', 'but', 'if', 'in', 'on', 'at', 'by', 'for', 'with', 'to', 'from', 'of', 'is', 'it', 'that', 'this']) { + expect(result.queries.flatMap((query: any) => query.queries)).not.toContain(ignored) + } + const onlyIgnored = tokenizer.tokenizeForSearch('x an') as any + expect(onlyIgnored.queries.every((query: any) => query.queries.length === 0 || query.queries[0] === 'x an')).toBe(true) + }) + + test('matches diacritic-insensitive text and preserves the original match', () => { + const processor = new TextProcessor(pluginFor()) + const matches = processor.getMatches('The café serves crème brûlée.', ['cafe', 'brulee']) + expect(matches.map(match => match.match)).toEqual(['café', 'brûlée']) + expect(removeDiacritics('café crème brûlée')).toBe('cafe creme brulee') + }) + + test('renders excerpts with blank lines as readable HTML', () => { + const processor = new TextProcessor(pluginFor({ renderLineReturnInExcerpts: true })) + const excerpt = processor.makeExcerpt(readNote('fork-features.md'), readNote('fork-features.md').indexOf('café')) + expect(excerpt).toContain('
') + expect(excerpt).not.toMatch(/
\s*
/) + expect(excerpt).toContain('café') + }) +}) + +describe('indexed heading metadata', () => { + test('indexes H1, H2, H3, and promotes only valid colon and AKA headings', async () => { + const { TFile } = require('obsidian') + const files = new Map([ + ['tests/notes/headers-and-lists.md', readNote('headers-and-lists.md')], + ['tests/notes/fork-features.md', readNote('fork-features.md')], + ['tests/notes/colon-variants.md', readNote('colon-variants.md')], + ['tests/notes/aka-variants.md', readNote('aka-variants.md')], + ]) + const metadata = new Map([ + ['tests/notes/headers-and-lists.md', { headings: [ + { level: 1, heading: 'H1 Heading' }, + { level: 2, heading: 'H2 Heading' }, + { level: 3, heading: 'H3 Heading' }, + { level: 4, heading: 'H4 Heading' }, + { level: 5, heading: 'H5 Heading' }, + { level: 6, heading: 'H6 Heading' }, + ] }], + ['tests/notes/fork-features.md', { headings: [] }], + ['tests/notes/colon-variants.md', { headings: [] }], + ['tests/notes/aka-variants.md', { headings: [] }], + ]) + const plugin = pluginFor({ indexedFileTypes: ['md'], unsupportedFilesIndexing: 'no' }) + plugin.notesIndexer = { isFilePlaintext: () => true, isFilenameIndexable: () => true } + plugin.getTextExtractor = () => undefined + plugin.getAIImageAnalyzer = () => undefined + plugin.app = { + vault: { + getAbstractFileByPath: (filePath: string) => new TFile(filePath), + cachedRead: async (file: any) => files.get(file.path), + }, + metadataCache: { + getFileCache: (file: any) => metadata.get(file.path), + }, + } + + const repository = new DocumentsRepository(plugin) + const getDocument = (repository as any).getAndMapIndexedDocument.bind(repository) + const headings = await getDocument('tests/notes/headers-and-lists.md') + expect(headings.headings1).toBe('H1 Heading') + expect(headings.headings2).toBe('H2 Heading') + expect(headings.headings3).toBe('H3 Heading') + + const fork = await getDocument('tests/notes/fork-features.md') + expect(fork.headings1).toBe('Packing List') + expect(fork.headings3).toBe('Japan trip') + + const colon = await getDocument('tests/notes/colon-variants.md') + expect(colon.headings3).toBe('Colon heading') + + const aka = await getDocument('tests/notes/aka-variants.md') + expect(aka.headings1).toBe('lowercase alias') + }) +}) diff --git a/tests/notes/aka-variants.md b/tests/notes/aka-variants.md new file mode 100644 index 0000000..9ccf55f --- /dev/null +++ b/tests/notes/aka-variants.md @@ -0,0 +1,9 @@ +aka lowercase alias + +Content for lowercase alias. + +--- + +AKA: uppercase alias + +This later alias must not be promoted because only the first paragraph is inspected. \ No newline at end of file diff --git a/tests/notes/colon-variants.md b/tests/notes/colon-variants.md new file mode 100644 index 0000000..aabb293 --- /dev/null +++ b/tests/notes/colon-variants.md @@ -0,0 +1,13 @@ +Introductory paragraph. + +Colon heading: +- item one +- item two + +Not a heading: this is inline content. + +Another colon heading: + +Its content starts after a blank line. + +Trailing text. \ No newline at end of file diff --git a/tests/notes/fork-features.md b/tests/notes/fork-features.md new file mode 100644 index 0000000..7c7eba5 --- /dev/null +++ b/tests/notes/fork-features.md @@ -0,0 +1,16 @@ +# Fork Feature Fixture +Aka: Packing List + +This body contains Sun's BBQ and a blank-line excerpt target. + + +The café serves crème brûlée in Montréal. + +Japan trip: +- passport +- cash +- umbrella + +Words such as a an the and or but if in on at by for with to from of is it that this should not be useful search terms. + +The apostrophe phrase Sun's BBQ appears again for highlighting. \ No newline at end of file diff --git a/tests/notes/headers-and-lists.md b/tests/notes/headers-and-lists.md new file mode 100644 index 0000000..cb28a17 --- /dev/null +++ b/tests/notes/headers-and-lists.md @@ -0,0 +1,27 @@ +# H1 Heading + +This note has a first-level heading and useful body content. + +## H2 Heading + +The second-level section has searchable material. + +### H3 Heading + +- first H3 list item +- second H3 list item + +#### H4 Heading + +H4 content is intentionally present too. + +##### H5 Heading + +H5 content and a [link](https://example.com). + +###### H6 Heading + +- nested item + - nested child + +A final paragraph mentions heading terms in ordinary content. \ No newline at end of file diff --git a/tests/notes/title-priority.md b/tests/notes/title-priority.md new file mode 100644 index 0000000..3a2453a --- /dev/null +++ b/tests/notes/title-priority.md @@ -0,0 +1,9 @@ +# Title Priority Fixture + +The title-priority term appears in body content after the heading. + +## Heading Priority Fixture + +The heading-priority term appears in ordinary content too. + +Content-only match appears here. \ No newline at end of file diff --git a/tests/obsidian-mock.ts b/tests/obsidian-mock.ts new file mode 100644 index 0000000..d6e1ad1 --- /dev/null +++ b/tests/obsidian-mock.ts @@ -0,0 +1,23 @@ +import path from 'node:path' + +export class TFile { + path = '' + basename = '' + stat = { mtime: 1 } + constructor(filePath = '') { + this.path = filePath + this.basename = path.basename(filePath, path.extname(filePath)) + } +} + +export const Notice = jest.fn() +export const Platform = { isMacOS: false } +export const normalizePath = (value: string) => value.replaceAll('\\', '/') +export const getAllTags = (metadata: any) => metadata?.tags ?? [] +export const parseFrontMatterAliases = (frontmatter: any) => { + const aliases = frontmatter?.aliases + if (!aliases) return [] + return Array.isArray(aliases) + ? aliases + : String(aliases).split(',').map(alias => alias.trim()).filter(Boolean) +} diff --git a/tests/svelte-store-mock.ts b/tests/svelte-store-mock.ts new file mode 100644 index 0000000..8e18dac --- /dev/null +++ b/tests/svelte-store-mock.ts @@ -0,0 +1,12 @@ +export const writable = (value: T) => ({ + subscribe(run: (value: T) => void) { + run(value) + return () => undefined + }, + set(next: T) { + value = next + }, + update(updateValue: (value: T) => T) { + value = updateValue(value) + }, +}) diff --git a/tools/obsidian-shim.ts b/tools/obsidian-shim.ts new file mode 100644 index 0000000..d8431dd --- /dev/null +++ b/tools/obsidian-shim.ts @@ -0,0 +1,15 @@ +export class TFile { + path: string + basename: string + stat = { mtime: 1 } + constructor(path: string) { + this.path = path + this.basename = path.split('/').pop()!.replace(/\.md$/, '') + } +} +export class Notice { constructor(..._args: unknown[]) {} } +export class MarkdownView {} +export const Platform = { isMacOS: false } +export const getAllTags = (metadata: any) => metadata?.tags ?? [] +export const parseFrontMatterAliases = (frontmatter: any) => frontmatter?.aliases ?? [] +export const normalizePath = (value: string) => value.replaceAll('\\', '/') diff --git a/tools/plugin-search.ts b/tools/plugin-search.ts new file mode 100644 index 0000000..10ee1f8 --- /dev/null +++ b/tools/plugin-search.ts @@ -0,0 +1,94 @@ +import { Query } from '../src/search/query' +import { SearchEngine } from '../src/search/search-engine' +import { DocumentsRepository } from '../src/repositories/documents-repository' +import { TFile } from 'obsidian' +import { TextProcessor } from '../src/tools/text-processing' +import type { IndexedDocument } from '../src/globals' + +type Note = { path: string; basename: string; text: string } + +const defaults: any = { + fuzziness: '1', weightBasename: 10, weightDirectory: 7, weightH1: 6, + weightH2: 5, weightH3: 4, weightUnmarkedTags: 2, recencyBoost: '0', + downrankedFoldersFilters: [], hideExcluded: false, weightCustomProperties: [], + ignoreDiacritics: true, ignoreArabicDiacritics: false, simpleSearch: false, + maxEmbeds: 5, renderLineReturnInExcerpts: true, highlight: true, +} + +function indexDocument(note: Note): IndexedDocument { + const lines = note.text.split('\n') + const headings1: string[] = [], headings2: string[] = [], headings3: string[] = [] + for (const line of lines) { + const match = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/) + if (match && match[1].length <= 3) { + ;[headings1, headings2, headings3][match[1].length - 1].push(match[2]) + } + } + for (let i = 0; i < lines.length; i++) { + const line = lines[i].trim() + const previous = i > 0 ? lines[i - 1].trim() : null + const next = i < lines.length - 1 ? lines[i + 1].trim() : null + if (line.endsWith(':') && previous === '' && next !== null && next !== '') { + headings3.push(line.slice(0, -1).trim()) + } + } + return { + path: note.path, basename: note.basename, displayTitle: '', mtime: 1, + content: note.text, cleanedContent: note.text, aliases: '', tags: [], + unmarkedTags: [], headings1: headings1.join(' '), headings2: headings2.join(' '), + headings3: headings3.join(' '), + } +} + +export async function createPluginSearch(notes: Note[]) { + const noteMap = new Map(notes.map(note => [note.path, note])) + const metadata = new Map(notes.map(note => [note.path, { + headings: note.text.split('\n').flatMap((line, index) => { + const match = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/) + return match ? [{ level: match[1].length, heading: match[2], position: { + start: { offset: note.text.split('\n').slice(0, index).join('\n').length + (index ? 1 : 0) }, + end: { offset: note.text.length }, + } }] : [] + }), + links: [], + }])) + const plugin: any = { + settings: defaults, + app: { + vault: { + getAbstractFileByPath: (path: string) => new (TFile as any)(path), + cachedRead: async (file: TFile) => noteMap.get(file.path)?.text ?? '', + }, + metadataCache: { + getCache: (path: string) => metadata.get(path), + getFileCache: (file: TFile) => metadata.get(file.path), + isUserIgnored: () => false, + }, + }, + notesIndexer: { isFilePlaintext: () => true, isFilenameIndexable: () => true }, + getChsSegmenter: () => undefined, + getTextExtractor: () => undefined, + getAIImageAnalyzer: () => undefined, + embedsRepository: { refreshEmbedsForNote: () => undefined, getEmbeds: () => [] }, + } + plugin.textProcessor = new TextProcessor(plugin) + plugin.documentsRepository = new DocumentsRepository(plugin) + const engine = new SearchEngine(plugin) + await engine.addFromPaths(notes.map(note => note.path)) + return async (text: string) => { + const query = new Query(text, { ignoreDiacritics: true, ignoreArabicDiacritics: false }) + const results = await engine.getSuggestions(query) + return results.map(result => { + const offset = result.matches[0]?.offset ?? -1 + const excerpt = plugin.textProcessor.makeExcerpt(result.content, offset) + return { + id: result.path, + terms: result.foundWords, + matches: result.matches, + score: result.score, + offset, + excerpt, + } + }) + } +} diff --git a/tools/search-tui-launcher.js b/tools/search-tui-launcher.js new file mode 100644 index 0000000..e82e83c --- /dev/null +++ b/tools/search-tui-launcher.js @@ -0,0 +1,17 @@ +#!/usr/bin/env node +const os = require('node:os') +const path = require('node:path') +const { buildSync } = require('esbuild') + +const output = path.join(os.tmpdir(), `tannersearch-plugin-search-${process.pid}.cjs`) +buildSync({ + entryPoints: [path.join(__dirname, 'plugin-search.ts')], + bundle: true, + platform: 'node', + format: 'cjs', + outfile: output, + alias: { obsidian: path.join(__dirname, 'obsidian-shim.ts') }, + logLevel: 'silent', +}) +process.env.TUI_PLUGIN_SEARCH = output +require('./search-tui.js') diff --git a/tools/search-tui.js b/tools/search-tui.js new file mode 100644 index 0000000..dc40986 --- /dev/null +++ b/tools/search-tui.js @@ -0,0 +1,441 @@ +#!/usr/bin/env node + +/** + * Small terminal harness for exercising the search experience without Obsidian. + * It intentionally uses only Node's standard library so it can be run from a + * checkout with: npm run tui [directory] + */ +const fs = require('node:fs') +const path = require('node:path') +const MiniSearch = require('minisearch') + +const DEFAULT_DIRECTORY = path.resolve(__dirname, '..', 'tests', 'notes') +const STOP_WORDS = new Set([ + 'a', 'an', 'the', 'and', 'or', 'but', 'if', 'in', 'on', 'at', 'by', 'for', + 'with', 'to', 'from', 'of', 'is', 'it', 'that', 'this', +]) +const ANSI = { + clear: '\x1b[2J\x1b[H', + bold: '\x1b[1m', + dim: '\x1b[2m', + cyan: '\x1b[36m', + yellow: '\x1b[33m', + green: '\x1b[32m', + inverse: '\x1b[7m', + hideCursor: '\x1b[?25l', + showCursor: '\x1b[?25h', + reset: '\x1b[0m', +} + +function walk(directory) { + const notes = [] + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const filePath = path.join(directory, entry.name) + if (entry.isDirectory()) notes.push(...walk(filePath)) + else if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) notes.push(filePath) + } + return notes +} + +function removeDiacritics(value) { + return value.normalize('NFD').replace(/\p{Diacritic}/gu, '').normalize('NFC') +} + +// Mirrors the plugin's indexing tokenizer. In particular, apostrophes are +// preserved as part of a token rather than being treated as separators. +function tokenize(text) { + return text + .split(/[|\[\]()<>{} \t\n\r]+/u) + .flatMap(word => [word, ...word.split(/[.,:;!?/\\_#%&*=^@-]+/u)]) + .filter(Boolean) +} + +function processTerm(term) { + const processed = removeDiacritics(term).toLowerCase() + return processed.length < 3 || STOP_WORDS.has(processed) ? null : processed +} + +function headingFields(text) { + const lines = text.split(/\r?\n/) + const fields = { headings1: [], headings2: [], headings3: [], markdownHeadings: [] } + for (let index = 0; index < lines.length; index++) { + const match = lines[index].match(/^(#{1,6})\s+(.+?)\s*#*\s*$/) + if (!match) continue + const level = match[1].length + if (level <= 3) fields[`headings${level}`].push(match[2]) + fields.markdownHeadings.push(match[2]) + } + // Match DocumentsRepository's colon-heading promotion rule. + for (let index = 0; index < lines.length; index++) { + const line = lines[index].trim() + const previous = index > 0 ? lines[index - 1].trim() : null + const next = index < lines.length - 1 ? lines[index + 1].trim() : null + if (line.endsWith(':') && previous === '' && next !== null && next !== '') { + fields.headings3.push(line.slice(0, -1).trim()) + } + } + return fields +} + +function buildIndex(notes) { + const index = new MiniSearch({ + idField: 'path', + fields: ['basename', 'directory', 'aliases', 'content', 'headings1', 'headings2', 'headings3'], + tokenize, + processTerm, + }) + index.addAll(notes.map(note => { + const fields = headingFields(note.text) + return { + path: note.path, + basename: note.basename, + directory: path.dirname(note.relative), + aliases: '', + content: note.text, + headings1: fields.headings1.join(' '), + headings2: fields.headings2.join(' '), + headings3: fields.headings3.join(' '), + } + })) + return index +} + +function search(index, notesByPath, query) { + const terms = tokenize(query).map(processTerm).filter(Boolean) + if (!terms.length) return [] + const results = index.search(terms.join(' '), { + prefix: term => term.length >= 1, + bm25: { b: 0.2, d: 0.5, k: 1.2 }, + fuzzy: term => term.length <= 4 ? 0 : term.length <= 5 ? 0.05 : 0.1, + boost: { basename: 10, directory: 7, headings1: 6, headings2: 5, headings3: 4 }, + }) + return results.map(result => { + const note = notesByPath.get(result.id) + const lower = note.text.toLocaleLowerCase() + const fields = headingFields(note.text) + const titleTerm = terms.find(term => note.basename.toLocaleLowerCase().includes(term)) + const headingTerm = result.terms.find(term => fields.markdownHeadings.some(heading => heading.toLowerCase().includes(term))) + const offset = titleTerm ? 0 : headingTerm + ? lower.indexOf(headingTerm) + : Math.max(0, lower.indexOf(result.terms[0] || terms[0])) + return { note, terms: result.terms.length ? result.terms : terms, score: result.score, offset } + }) +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +function htmlExcerptToTerminal(html) { + const text = html + .replace(//gi, '\n') + .replace(/]*>([\s\S]*?)<\/span>/gi, `${ANSI.yellow}${ANSI.bold}$1${ANSI.reset}${ANSI.dim}`) + .replace(/<[^>]+>/g, '') + .replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>') + .replace(/"/g, '"').replace(/'/g, "'") + const unindented = text.split('\n').map(line => line.replace(/^\s+/, '')) + return ANSI.dim + unindented.slice(0, 3).join('\n') + ANSI.reset +} + +function highlight(text, terms, restore = '') { + if (!terms.length) return text + const expression = new RegExp(`(${terms.sort((a, b) => b.length - a.length).map(escapeRegExp).join('|')})`, 'giu') + return text.replace(expression, `${ANSI.yellow}${ANSI.bold}$1${ANSI.reset}${restore}`) +} + +// Use the exact matches returned by TextProcessor rather than MiniSearch's +// terms. This preserves complete matches such as "OCD" and "OCD's". +function highlightMatches(text, matches, additionalTerms = []) { + const terms = [...new Set([ + ...matches.map(match => match.match), + ...additionalTerms, + ])].sort((a, b) => b.length - a.length) + if (!terms.length) return text + const expression = new RegExp(`(${terms.map(escapeRegExp).join('|')})`, 'giu') + return text.replace(expression, `${ANSI.yellow}${ANSI.bold}$1${ANSI.reset}${ANSI.dim}`) +} + +function excerpt(note, result) { + // Obsidian's result preview is short and keeps the note's line breaks. + const lines = note.text.split(/\r?\n/) + let offset = 0 + let matchLine = 0 + for (let index = 0; index < lines.length; index++) { + if (result.offset <= offset + lines[index].length) { + matchLine = index + break + } + offset += lines[index].length + 1 + } + const firstLine = Math.max(0, matchLine - 1) + const lastLine = Math.min(lines.length, firstLine + 3) + const visible = lines.slice(firstLine, lastLine) + const prefix = firstLine ? '…' : '' + const suffix = lastLine < lines.length ? '…' : '' + return ANSI.dim + prefix + visible.map(line => highlight(line, result.terms, ANSI.dim)).join('\n') + suffix + ANSI.reset +} + +function loadNotes(directory) { + return walk(directory).map(filePath => ({ + path: filePath, + basename: path.basename(filePath, '.md'), + relative: path.relative(directory, filePath), + text: fs.readFileSync(filePath, 'utf8'), + })) +} + +class SearchTUI { + constructor(directory) { + this.directory = directory + this.pluginSearch = null + this.searchGeneration = 0 + this.notes = loadNotes(directory) + this.notesByPath = new Map(this.notes.map(note => [note.path, note])) + this.index = buildIndex(this.notes) + this.query = '' + this.results = [] + this.selected = 0 + this.scrollOffset = 0 + this.mode = 'search' + this.openResult = null + this.noteScrollLine = 0 + } + + async init() { + if (process.env.TUI_PLUGIN_SEARCH) { + const { createPluginSearch } = require(process.env.TUI_PLUGIN_SEARCH) + this.pluginSearch = await createPluginSearch(this.notes.map(note => ({ + path: note.relative, + basename: note.basename, + text: note.text, + }))) + } + return this + } + + start() { + if (!process.stdin.isTTY || !process.stdout.isTTY) { + throw new Error('The search TUI requires an interactive terminal (TTY).') + } + process.stdin.setRawMode(true) + process.stdout.write(ANSI.hideCursor) + process.stdin.resume() + process.stdin.setEncoding('utf8') + process.stdin.on('data', key => this.onKey(key)) + this.render() + } + + stop() { + process.stdin.setRawMode(false) + process.stdin.pause() + process.stdin.removeAllListeners('data') + // DocumentsRepository owns a repeating cache timer, so merely pausing + // stdin leaves the process alive. Restore the terminal and exit explicitly. + process.stdout.write(`${ANSI.showCursor}\n`) + process.exit(0) + } + + onKey(key) { + if (key === '\u0003') return this.stop() // Ctrl-C + if (this.mode === 'note') { + if (key === 'q' || key === '\u001b' || key === '\u001b\u001b') { + this.mode = 'search' + this.render() + } else if (key === '\u001b[A') { + this.noteScrollLine = Math.max(0, this.noteScrollLine - 1) + this.render() + } else if (key === '\u001b[B') { + this.noteScrollLine += 1 + this.ensureNoteScrollBounds() + this.render() + } else if (key === '\u001b[5~') { + this.noteScrollLine = Math.max(0, this.noteScrollLine - this.visibleNoteLines()) + this.render() + } else if (key === '\u001b[6~') { + this.noteScrollLine += this.visibleNoteLines() + this.ensureNoteScrollBounds() + this.render() + } else if (key === '\u001b[H' || key === '\u001b[1~') { + this.noteScrollLine = 0 + this.render() + } else if (key === '\u001b[F' || key === '\u001b[4~') { + this.noteScrollLine = Number.MAX_SAFE_INTEGER + this.ensureNoteScrollBounds() + this.render() + } + return + } + if (key === '\u0008' || key === '\u007f') { + this.query = this.query.slice(0, -1) + this.updateResults() + } else if (key === '\r' || key === '\n') { + if (this.results.length) { + this.openResult = this.results[this.selected] + this.mode = 'note' + this.noteScrollLine = 0 + this.ensureNoteCursorVisible() + this.render() + } + } else if (key === '\u001b[A') { + this.selected = Math.max(0, this.selected - 1) + this.ensureSelectionVisible() + this.render() + } else if (key === '\u001b[B') { + this.selected = Math.min(Math.max(0, this.results.length - 1), this.selected + 1) + this.ensureSelectionVisible() + this.render() + } else if (key === '\u001b[5~') { + this.selected = Math.max(0, this.selected - this.visibleResultCount()) + this.ensureSelectionVisible() + this.render() + } else if (key === '\u001b[6~') { + this.selected = Math.min(Math.max(0, this.results.length - 1), this.selected + this.visibleResultCount()) + this.ensureSelectionVisible() + this.render() + } else if (key === '\u001b[H' || key === '\u001b[1~') { + this.selected = 0 + this.ensureSelectionVisible() + this.render() + } else if (key === '\u001b[F' || key === '\u001b[4~') { + this.selected = Math.max(0, this.results.length - 1) + this.ensureSelectionVisible() + this.render() + } else if (key === '\u001b') { + this.query = '' + this.updateResults() + } else if (key >= ' ' && key !== '\u007f') { + this.query += key + this.updateResults() + } + } + + visibleNoteLines() { + return Math.max(1, (process.stdout.rows || 24) - 6) + } + + noteCursorPosition() { + if (!this.openResult) return { line: 0, column: 0 } + let remaining = this.openResult.offset + const lines = this.openResult.note.text.split(/\r?\n/) + for (let line = 0; line < lines.length; line++) { + if (remaining <= lines[line].length) return { line, column: Math.max(0, remaining) } + remaining -= lines[line].length + 1 + } + return { line: lines.length - 1, column: lines.at(-1)?.length ?? 0 } + } + + ensureNoteScrollBounds() { + if (!this.openResult) return + const lineCount = this.openResult.note.text.split(/\r?\n/).length + this.noteScrollLine = Math.max(0, Math.min(this.noteScrollLine, Math.max(0, lineCount - this.visibleNoteLines()))) + } + + ensureNoteCursorVisible() { + const cursor = this.noteCursorPosition() + const visible = this.visibleNoteLines() + this.noteScrollLine = cursor.line - Math.floor(visible / 2) + this.ensureNoteScrollBounds() + } + + visibleResultCount() { + // Each result occupies a title line, three excerpt lines, and a spacer. + // Keep the header and help text visible while scrolling. + return Math.max(1, Math.floor(((process.stdout.rows || 24) - 6) / 5)) + } + + ensureSelectionVisible() { + const visible = this.visibleResultCount() + if (this.selected < this.scrollOffset) this.scrollOffset = this.selected + if (this.selected >= this.scrollOffset + visible) { + this.scrollOffset = this.selected - visible + 1 + } + this.scrollOffset = Math.max(0, Math.min( + this.scrollOffset, + Math.max(0, this.results.length - visible), + )) + } + + async updateResults() { + const generation = ++this.searchGeneration + if (this.pluginSearch) { + const ranked = await this.pluginSearch(this.query) + if (generation !== this.searchGeneration) return + this.results = ranked.map(result => { + const note = this.notes.find(candidate => candidate.relative === result.id) + return { + note, + terms: result.terms, + matches: result.matches, + score: result.score, + offset: result.offset, + excerpt: highlightMatches( + htmlExcerptToTerminal(result.excerpt), + result.matches, + result.terms, + ), + } + }).filter(result => result.note) + } else { + this.results = search(this.index, this.notesByPath, this.query) + } + this.selected = Math.min(this.selected, Math.max(0, this.results.length - 1)) + this.ensureSelectionVisible() + this.render() + } + + render() { + process.stdout.write(ANSI.clear) + if (this.mode === 'note') return this.renderNote() + process.stdout.write(`${ANSI.bold}${ANSI.cyan}Tannersearch TUI${ANSI.reset} ${ANSI.dim}${this.directory}${ANSI.reset}\n\n`) + process.stdout.write(`${ANSI.bold}Search:${ANSI.reset} ${this.query}${ANSI.cyan}▌${ANSI.reset}\n`) + process.stdout.write(`${ANSI.dim}Type to search · ↑/↓ select · Enter open · Esc clear · Ctrl-C quit${ANSI.reset}\n\n`) + if (!this.query) { + process.stdout.write(`${ANSI.dim}Start typing to search ${this.notes.length} Markdown notes.${ANSI.reset}\n`) + return + } + if (!this.results.length) { + process.stdout.write(`${ANSI.yellow}No matching notes.${ANSI.reset}\n`) + return + } + const visible = this.visibleResultCount() + this.ensureSelectionVisible() + const shown = this.results.slice(this.scrollOffset, this.scrollOffset + visible) + shown.forEach((result, visibleIndex) => { + const index = this.scrollOffset + visibleIndex + const marker = index === this.selected ? `${ANSI.inverse}>${ANSI.reset}` : ' ' + process.stdout.write(`${marker} ${ANSI.bold}${result.note.basename}${ANSI.reset} ${ANSI.dim}(${result.note.relative})${ANSI.reset}\n`) + process.stdout.write(`${result.excerpt || excerpt(result.note, result)}\n\n`) + }) + if (this.results.length > visible) { + process.stdout.write(`${ANSI.dim}Showing ${this.scrollOffset + 1}-${Math.min(this.scrollOffset + visible, this.results.length)} of ${this.results.length} · PageUp/PageDown to scroll${ANSI.reset}\n`) + } + } + + renderNote() { + const result = this.openResult + const note = result.note + process.stdout.write(`${ANSI.bold}${ANSI.green}${note.basename}${ANSI.reset} ${ANSI.dim}— fake cursor at match${ANSI.reset}\n`) + process.stdout.write(`${ANSI.dim}Press q or Esc to return to results.${ANSI.reset}\n\n`) + const lines = note.text.split(/\r?\n/) + const cursor = this.noteCursorPosition() + const visible = lines.slice(this.noteScrollLine, this.noteScrollLine + this.visibleNoteLines()) + visible.forEach((line, index) => { + const lineNumber = this.noteScrollLine + index + if (lineNumber === cursor.line) { + process.stdout.write(`${highlight(line.slice(0, cursor.column), result.terms)}${ANSI.green}▌${ANSI.reset}${highlight(line.slice(cursor.column), result.terms)}\n`) + process.stdout.write(`${' '.repeat(cursor.column)}${ANSI.green}^ cursor${ANSI.reset}\n`) + } else { + process.stdout.write(`${highlight(line, result.terms)}\n`) + } + }) + } +} + +const directory = path.resolve(process.argv[2] || DEFAULT_DIRECTORY) +try { + if (!fs.statSync(directory).isDirectory()) throw new Error(`Not a directory: ${directory}`) + new SearchTUI(directory).init().then(tui => tui.start()) +} catch (error) { + console.error(`search-tui: ${error.message}`) + process.exitCode = 1 +}