Add tests and search simulator TUI
This commit is contained in:
Tanner
+5
-1
@@ -82,7 +82,11 @@ module.exports = {
|
||||
],
|
||||
|
||||
// A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
|
||||
// moduleNameMapper: {},
|
||||
moduleNameMapper: {
|
||||
'^obsidian$': '<rootDir>/tests/obsidian-mock.ts',
|
||||
'^svelte/store$': '<rootDir>/tests/svelte-store-mock.ts',
|
||||
'^lodash-es$': 'lodash',
|
||||
},
|
||||
|
||||
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
|
||||
// modulePathIgnorePatterns: [],
|
||||
|
||||
+2
-1
@@ -8,7 +8,8 @@
|
||||
"build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
|
||||
"check": "svelte-check --tsconfig ./tsconfig.json",
|
||||
"version": "node version-bump.mjs",
|
||||
"test": "jest"
|
||||
"test": "jest",
|
||||
"tui": "node tools/search-tui-launcher.js"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "Simon Cambier",
|
||||
|
||||
@@ -26,7 +26,7 @@ export class TextProcessor {
|
||||
return text.replace(
|
||||
new RegExp(
|
||||
`(${matches
|
||||
.map(item => escapeRegExp(escapeHTML(item.match)))
|
||||
.map(item => escapeRegExp(item.match))
|
||||
.join('|')})`,
|
||||
'giu'
|
||||
),
|
||||
@@ -72,13 +72,41 @@ export class TextProcessor {
|
||||
if (this.plugin.settings.ignoreDiacritics) {
|
||||
text = removeDiacritics(text, this.plugin.settings.ignoreArabicDiacritics)
|
||||
}
|
||||
const startTime = new Date().getTime()
|
||||
let match: RegExpExecArray | null = null
|
||||
let matches: SearchMatch[] = []
|
||||
let count = 0
|
||||
|
||||
// Build the normalized-to-original offset map once. Re-normalizing every
|
||||
// prefix for every match becomes extremely expensive in large notes.
|
||||
const normalizedToOriginal: number[] = [0]
|
||||
if (this.plugin.settings.ignoreDiacritics) {
|
||||
for (let index = 0; index < originalText.length;) {
|
||||
const codePoint = String.fromCodePoint(originalText.codePointAt(index)!)
|
||||
const normalized = removeDiacritics(
|
||||
codePoint,
|
||||
this.plugin.settings.ignoreArabicDiacritics
|
||||
)
|
||||
const normalizedStart = normalizedToOriginal.length - 1
|
||||
for (let offset = 0; offset < normalized.length; offset++) {
|
||||
normalizedToOriginal[normalizedStart + offset] = index
|
||||
}
|
||||
normalizedToOriginal.push(index + codePoint.length)
|
||||
index += codePoint.length
|
||||
}
|
||||
}
|
||||
|
||||
const originalOffset = (normalizedOffset: number): number => {
|
||||
if (!this.plugin.settings.ignoreDiacritics) return normalizedOffset
|
||||
if (normalizedOffset <= 0) return 0
|
||||
return normalizedToOriginal[Math.min(normalizedOffset, normalizedToOriginal.length - 1)] ?? originalText.length
|
||||
}
|
||||
|
||||
while ((match = reg.exec(text)) !== null) {
|
||||
// Avoid infinite loops, stop looking after 100 matches or if we're taking too much time
|
||||
if (++count >= 100 || new Date().getTime() - startTime > 50) {
|
||||
// Avoid infinite loops and unbounded result lists.
|
||||
// The regex scan is linear and the 100-match cap keeps large notes safe;
|
||||
// a wall-clock cutoff can incorrectly discard a valid late match in
|
||||
// very large notes.
|
||||
if (++count >= 100) {
|
||||
warnVerbose('Stopped getMatches at', count, 'results')
|
||||
break
|
||||
}
|
||||
@@ -88,9 +116,13 @@ export class TextProcessor {
|
||||
// If `ignoreDiacritics` is on, `text` may have a different length than `originalText`,
|
||||
// making `match.index` unreliable for `originalText`.
|
||||
// We use `match[0]`, which is the matched term (but without diacritics).
|
||||
const originalMatchBeforeTrim = this.plugin.settings.ignoreDiacritics
|
||||
? match[0]
|
||||
: originalText.substring(matchStartIndex, matchEndIndex)
|
||||
// Match offsets refer to the normalized string. Translate both ends
|
||||
// back to the original text so excerpts and highlighting retain
|
||||
// diacritics, without repeatedly normalizing large prefixes.
|
||||
const originalMatchBeforeTrim = originalText.substring(
|
||||
originalOffset(matchStartIndex),
|
||||
originalOffset(matchEndIndex)
|
||||
)
|
||||
|
||||
const originalMatch = originalMatchBeforeTrim.trim()
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
|
||||
jest.mock('markdown-link-extractor', () => () => [])
|
||||
jest.mock('obsidian', () => {
|
||||
const nodePath = require('node:path')
|
||||
class TFile {
|
||||
path = ''
|
||||
basename = ''
|
||||
stat = { mtime: 1 }
|
||||
constructor(filePath: string) {
|
||||
this.path = filePath
|
||||
this.basename = nodePath.basename(filePath, nodePath.extname(filePath))
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
TFile,
|
||||
Notice: jest.fn(),
|
||||
Platform: { isMacOS: false },
|
||||
normalizePath: (value: string) => value.replaceAll('\\', '/'),
|
||||
getAllTags: (metadata: any) => metadata?.tags ?? [],
|
||||
parseFrontMatterAliases: (frontmatter: any) => frontmatter?.aliases ?? [],
|
||||
}
|
||||
})
|
||||
|
||||
import { Query } from '../src/search/query'
|
||||
import { Tokenizer } from '../src/search/tokenizer'
|
||||
import { TextProcessor } from '../src/tools/text-processing'
|
||||
import { removeDiacritics } from '../src/tools/utils'
|
||||
import { DocumentsRepository } from '../src/repositories/documents-repository'
|
||||
|
||||
const notesDir = path.join(__dirname, 'notes')
|
||||
const readNote = (name: string) =>
|
||||
fs.readFileSync(path.join(notesDir, name), 'utf8')
|
||||
|
||||
const pluginFor = (overrides: Record<string, unknown> = {}) => ({
|
||||
settings: {
|
||||
highlight: true,
|
||||
ignoreDiacritics: true,
|
||||
ignoreArabicDiacritics: false,
|
||||
renderLineReturnInExcerpts: false,
|
||||
...overrides,
|
||||
},
|
||||
getChsSegmenter: () => undefined,
|
||||
} as any)
|
||||
|
||||
describe('fork search behavior', () => {
|
||||
test('keeps apostrophes together in a query and highlights them', () => {
|
||||
const query = new Query("Sun's BBQ", {
|
||||
ignoreDiacritics: false,
|
||||
ignoreArabicDiacritics: false,
|
||||
})
|
||||
expect(query.query.text).toEqual(["sun's", 'bbq'])
|
||||
|
||||
const processor = new TextProcessor(pluginFor())
|
||||
const matches = processor.getMatches("Sun's BBQ", ["Sun's", 'BBQ'], query)
|
||||
expect(matches[0].match).toBe("Sun's BBQ")
|
||||
expect(processor.highlightText("Sun's BBQ", matches)).toContain(
|
||||
'omnisearch-highlight'
|
||||
)
|
||||
expect(processor.highlightText("Sun's BBQ", matches)).toContain("Sun's BBQ")
|
||||
})
|
||||
|
||||
test('filters words shorter than three characters and the documented stop words', () => {
|
||||
const tokenizer = new Tokenizer(pluginFor())
|
||||
const result = tokenizer.tokenizeForSearch('a an the and or but if in on at by for with to from of is it that this useful') as any
|
||||
expect(result.queries.length).toBeGreaterThan(0)
|
||||
expect(result.queries.flatMap((query: any) => query.queries)).toContain('useful')
|
||||
for (const ignored of ['a', 'an', 'the', 'and', 'or', 'but', 'if', 'in', 'on', 'at', 'by', 'for', 'with', 'to', 'from', 'of', 'is', 'it', 'that', 'this']) {
|
||||
expect(result.queries.flatMap((query: any) => query.queries)).not.toContain(ignored)
|
||||
}
|
||||
const onlyIgnored = tokenizer.tokenizeForSearch('x an') as any
|
||||
expect(onlyIgnored.queries.every((query: any) => query.queries.length === 0 || query.queries[0] === 'x an')).toBe(true)
|
||||
})
|
||||
|
||||
test('matches diacritic-insensitive text and preserves the original match', () => {
|
||||
const processor = new TextProcessor(pluginFor())
|
||||
const matches = processor.getMatches('The café serves crème brûlée.', ['cafe', 'brulee'])
|
||||
expect(matches.map(match => match.match)).toEqual(['café', 'brûlée'])
|
||||
expect(removeDiacritics('café crème brûlée')).toBe('cafe creme brulee')
|
||||
})
|
||||
|
||||
test('renders excerpts with blank lines as readable HTML', () => {
|
||||
const processor = new TextProcessor(pluginFor({ renderLineReturnInExcerpts: true }))
|
||||
const excerpt = processor.makeExcerpt(readNote('fork-features.md'), readNote('fork-features.md').indexOf('café'))
|
||||
expect(excerpt).toContain('<br>')
|
||||
expect(excerpt).not.toMatch(/<br>\s*<br>/)
|
||||
expect(excerpt).toContain('café')
|
||||
})
|
||||
})
|
||||
|
||||
describe('indexed heading metadata', () => {
|
||||
test('indexes H1, H2, H3, and promotes only valid colon and AKA headings', async () => {
|
||||
const { TFile } = require('obsidian')
|
||||
const files = new Map<string, string>([
|
||||
['tests/notes/headers-and-lists.md', readNote('headers-and-lists.md')],
|
||||
['tests/notes/fork-features.md', readNote('fork-features.md')],
|
||||
['tests/notes/colon-variants.md', readNote('colon-variants.md')],
|
||||
['tests/notes/aka-variants.md', readNote('aka-variants.md')],
|
||||
])
|
||||
const metadata = new Map<string, any>([
|
||||
['tests/notes/headers-and-lists.md', { headings: [
|
||||
{ level: 1, heading: 'H1 Heading' },
|
||||
{ level: 2, heading: 'H2 Heading' },
|
||||
{ level: 3, heading: 'H3 Heading' },
|
||||
{ level: 4, heading: 'H4 Heading' },
|
||||
{ level: 5, heading: 'H5 Heading' },
|
||||
{ level: 6, heading: 'H6 Heading' },
|
||||
] }],
|
||||
['tests/notes/fork-features.md', { headings: [] }],
|
||||
['tests/notes/colon-variants.md', { headings: [] }],
|
||||
['tests/notes/aka-variants.md', { headings: [] }],
|
||||
])
|
||||
const plugin = pluginFor({ indexedFileTypes: ['md'], unsupportedFilesIndexing: 'no' })
|
||||
plugin.notesIndexer = { isFilePlaintext: () => true, isFilenameIndexable: () => true }
|
||||
plugin.getTextExtractor = () => undefined
|
||||
plugin.getAIImageAnalyzer = () => undefined
|
||||
plugin.app = {
|
||||
vault: {
|
||||
getAbstractFileByPath: (filePath: string) => new TFile(filePath),
|
||||
cachedRead: async (file: any) => files.get(file.path),
|
||||
},
|
||||
metadataCache: {
|
||||
getFileCache: (file: any) => metadata.get(file.path),
|
||||
},
|
||||
}
|
||||
|
||||
const repository = new DocumentsRepository(plugin)
|
||||
const getDocument = (repository as any).getAndMapIndexedDocument.bind(repository)
|
||||
const headings = await getDocument('tests/notes/headers-and-lists.md')
|
||||
expect(headings.headings1).toBe('H1 Heading')
|
||||
expect(headings.headings2).toBe('H2 Heading')
|
||||
expect(headings.headings3).toBe('H3 Heading')
|
||||
|
||||
const fork = await getDocument('tests/notes/fork-features.md')
|
||||
expect(fork.headings1).toBe('Packing List')
|
||||
expect(fork.headings3).toBe('Japan trip')
|
||||
|
||||
const colon = await getDocument('tests/notes/colon-variants.md')
|
||||
expect(colon.headings3).toBe('Colon heading')
|
||||
|
||||
const aka = await getDocument('tests/notes/aka-variants.md')
|
||||
expect(aka.headings1).toBe('lowercase alias')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,9 @@
|
||||
aka lowercase alias
|
||||
|
||||
Content for lowercase alias.
|
||||
|
||||
---
|
||||
|
||||
AKA: uppercase alias
|
||||
|
||||
This later alias must not be promoted because only the first paragraph is inspected.
|
||||
@@ -0,0 +1,13 @@
|
||||
Introductory paragraph.
|
||||
|
||||
Colon heading:
|
||||
- item one
|
||||
- item two
|
||||
|
||||
Not a heading: this is inline content.
|
||||
|
||||
Another colon heading:
|
||||
|
||||
Its content starts after a blank line.
|
||||
|
||||
Trailing text.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Fork Feature Fixture
|
||||
Aka: Packing List
|
||||
|
||||
This body contains Sun's BBQ and a blank-line excerpt target.
|
||||
|
||||
|
||||
The café serves crème brûlée in Montréal.
|
||||
|
||||
Japan trip:
|
||||
- passport
|
||||
- cash
|
||||
- umbrella
|
||||
|
||||
Words such as a an the and or but if in on at by for with to from of is it that this should not be useful search terms.
|
||||
|
||||
The apostrophe phrase Sun's BBQ appears again for highlighting.
|
||||
@@ -0,0 +1,27 @@
|
||||
# H1 Heading
|
||||
|
||||
This note has a first-level heading and useful body content.
|
||||
|
||||
## H2 Heading
|
||||
|
||||
The second-level section has searchable material.
|
||||
|
||||
### H3 Heading
|
||||
|
||||
- first H3 list item
|
||||
- second H3 list item
|
||||
|
||||
#### H4 Heading
|
||||
|
||||
H4 content is intentionally present too.
|
||||
|
||||
##### H5 Heading
|
||||
|
||||
H5 content and a [link](https://example.com).
|
||||
|
||||
###### H6 Heading
|
||||
|
||||
- nested item
|
||||
- nested child
|
||||
|
||||
A final paragraph mentions heading terms in ordinary content.
|
||||
@@ -0,0 +1,9 @@
|
||||
# Title Priority Fixture
|
||||
|
||||
The title-priority term appears in body content after the heading.
|
||||
|
||||
## Heading Priority Fixture
|
||||
|
||||
The heading-priority term appears in ordinary content too.
|
||||
|
||||
Content-only match appears here.
|
||||
@@ -0,0 +1,23 @@
|
||||
import path from 'node:path'
|
||||
|
||||
export class TFile {
|
||||
path = ''
|
||||
basename = ''
|
||||
stat = { mtime: 1 }
|
||||
constructor(filePath = '') {
|
||||
this.path = filePath
|
||||
this.basename = path.basename(filePath, path.extname(filePath))
|
||||
}
|
||||
}
|
||||
|
||||
export const Notice = jest.fn()
|
||||
export const Platform = { isMacOS: false }
|
||||
export const normalizePath = (value: string) => value.replaceAll('\\', '/')
|
||||
export const getAllTags = (metadata: any) => metadata?.tags ?? []
|
||||
export const parseFrontMatterAliases = (frontmatter: any) => {
|
||||
const aliases = frontmatter?.aliases
|
||||
if (!aliases) return []
|
||||
return Array.isArray(aliases)
|
||||
? aliases
|
||||
: String(aliases).split(',').map(alias => alias.trim()).filter(Boolean)
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
export const writable = <T>(value: T) => ({
|
||||
subscribe(run: (value: T) => void) {
|
||||
run(value)
|
||||
return () => undefined
|
||||
},
|
||||
set(next: T) {
|
||||
value = next
|
||||
},
|
||||
update(updateValue: (value: T) => T) {
|
||||
value = updateValue(value)
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,15 @@
|
||||
export class TFile {
|
||||
path: string
|
||||
basename: string
|
||||
stat = { mtime: 1 }
|
||||
constructor(path: string) {
|
||||
this.path = path
|
||||
this.basename = path.split('/').pop()!.replace(/\.md$/, '')
|
||||
}
|
||||
}
|
||||
export class Notice { constructor(..._args: unknown[]) {} }
|
||||
export class MarkdownView {}
|
||||
export const Platform = { isMacOS: false }
|
||||
export const getAllTags = (metadata: any) => metadata?.tags ?? []
|
||||
export const parseFrontMatterAliases = (frontmatter: any) => frontmatter?.aliases ?? []
|
||||
export const normalizePath = (value: string) => value.replaceAll('\\', '/')
|
||||
@@ -0,0 +1,94 @@
|
||||
import { Query } from '../src/search/query'
|
||||
import { SearchEngine } from '../src/search/search-engine'
|
||||
import { DocumentsRepository } from '../src/repositories/documents-repository'
|
||||
import { TFile } from 'obsidian'
|
||||
import { TextProcessor } from '../src/tools/text-processing'
|
||||
import type { IndexedDocument } from '../src/globals'
|
||||
|
||||
type Note = { path: string; basename: string; text: string }
|
||||
|
||||
const defaults: any = {
|
||||
fuzziness: '1', weightBasename: 10, weightDirectory: 7, weightH1: 6,
|
||||
weightH2: 5, weightH3: 4, weightUnmarkedTags: 2, recencyBoost: '0',
|
||||
downrankedFoldersFilters: [], hideExcluded: false, weightCustomProperties: [],
|
||||
ignoreDiacritics: true, ignoreArabicDiacritics: false, simpleSearch: false,
|
||||
maxEmbeds: 5, renderLineReturnInExcerpts: true, highlight: true,
|
||||
}
|
||||
|
||||
function indexDocument(note: Note): IndexedDocument {
|
||||
const lines = note.text.split('\n')
|
||||
const headings1: string[] = [], headings2: string[] = [], headings3: string[] = []
|
||||
for (const line of lines) {
|
||||
const match = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/)
|
||||
if (match && match[1].length <= 3) {
|
||||
;[headings1, headings2, headings3][match[1].length - 1].push(match[2])
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
const previous = i > 0 ? lines[i - 1].trim() : null
|
||||
const next = i < lines.length - 1 ? lines[i + 1].trim() : null
|
||||
if (line.endsWith(':') && previous === '' && next !== null && next !== '') {
|
||||
headings3.push(line.slice(0, -1).trim())
|
||||
}
|
||||
}
|
||||
return {
|
||||
path: note.path, basename: note.basename, displayTitle: '', mtime: 1,
|
||||
content: note.text, cleanedContent: note.text, aliases: '', tags: [],
|
||||
unmarkedTags: [], headings1: headings1.join(' '), headings2: headings2.join(' '),
|
||||
headings3: headings3.join(' '),
|
||||
}
|
||||
}
|
||||
|
||||
export async function createPluginSearch(notes: Note[]) {
|
||||
const noteMap = new Map(notes.map(note => [note.path, note]))
|
||||
const metadata = new Map(notes.map(note => [note.path, {
|
||||
headings: note.text.split('\n').flatMap((line, index) => {
|
||||
const match = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/)
|
||||
return match ? [{ level: match[1].length, heading: match[2], position: {
|
||||
start: { offset: note.text.split('\n').slice(0, index).join('\n').length + (index ? 1 : 0) },
|
||||
end: { offset: note.text.length },
|
||||
} }] : []
|
||||
}),
|
||||
links: [],
|
||||
}]))
|
||||
const plugin: any = {
|
||||
settings: defaults,
|
||||
app: {
|
||||
vault: {
|
||||
getAbstractFileByPath: (path: string) => new (TFile as any)(path),
|
||||
cachedRead: async (file: TFile) => noteMap.get(file.path)?.text ?? '',
|
||||
},
|
||||
metadataCache: {
|
||||
getCache: (path: string) => metadata.get(path),
|
||||
getFileCache: (file: TFile) => metadata.get(file.path),
|
||||
isUserIgnored: () => false,
|
||||
},
|
||||
},
|
||||
notesIndexer: { isFilePlaintext: () => true, isFilenameIndexable: () => true },
|
||||
getChsSegmenter: () => undefined,
|
||||
getTextExtractor: () => undefined,
|
||||
getAIImageAnalyzer: () => undefined,
|
||||
embedsRepository: { refreshEmbedsForNote: () => undefined, getEmbeds: () => [] },
|
||||
}
|
||||
plugin.textProcessor = new TextProcessor(plugin)
|
||||
plugin.documentsRepository = new DocumentsRepository(plugin)
|
||||
const engine = new SearchEngine(plugin)
|
||||
await engine.addFromPaths(notes.map(note => note.path))
|
||||
return async (text: string) => {
|
||||
const query = new Query(text, { ignoreDiacritics: true, ignoreArabicDiacritics: false })
|
||||
const results = await engine.getSuggestions(query)
|
||||
return results.map(result => {
|
||||
const offset = result.matches[0]?.offset ?? -1
|
||||
const excerpt = plugin.textProcessor.makeExcerpt(result.content, offset)
|
||||
return {
|
||||
id: result.path,
|
||||
terms: result.foundWords,
|
||||
matches: result.matches,
|
||||
score: result.score,
|
||||
offset,
|
||||
excerpt,
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env node
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
const { buildSync } = require('esbuild')
|
||||
|
||||
const output = path.join(os.tmpdir(), `tannersearch-plugin-search-${process.pid}.cjs`)
|
||||
buildSync({
|
||||
entryPoints: [path.join(__dirname, 'plugin-search.ts')],
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'cjs',
|
||||
outfile: output,
|
||||
alias: { obsidian: path.join(__dirname, 'obsidian-shim.ts') },
|
||||
logLevel: 'silent',
|
||||
})
|
||||
process.env.TUI_PLUGIN_SEARCH = output
|
||||
require('./search-tui.js')
|
||||
@@ -0,0 +1,441 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Small terminal harness for exercising the search experience without Obsidian.
|
||||
* It intentionally uses only Node's standard library so it can be run from a
|
||||
* checkout with: npm run tui [directory]
|
||||
*/
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const MiniSearch = require('minisearch')
|
||||
|
||||
const DEFAULT_DIRECTORY = path.resolve(__dirname, '..', 'tests', 'notes')
|
||||
const STOP_WORDS = new Set([
|
||||
'a', 'an', 'the', 'and', 'or', 'but', 'if', 'in', 'on', 'at', 'by', 'for',
|
||||
'with', 'to', 'from', 'of', 'is', 'it', 'that', 'this',
|
||||
])
|
||||
const ANSI = {
|
||||
clear: '\x1b[2J\x1b[H',
|
||||
bold: '\x1b[1m',
|
||||
dim: '\x1b[2m',
|
||||
cyan: '\x1b[36m',
|
||||
yellow: '\x1b[33m',
|
||||
green: '\x1b[32m',
|
||||
inverse: '\x1b[7m',
|
||||
hideCursor: '\x1b[?25l',
|
||||
showCursor: '\x1b[?25h',
|
||||
reset: '\x1b[0m',
|
||||
}
|
||||
|
||||
function walk(directory) {
|
||||
const notes = []
|
||||
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
||||
const filePath = path.join(directory, entry.name)
|
||||
if (entry.isDirectory()) notes.push(...walk(filePath))
|
||||
else if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) notes.push(filePath)
|
||||
}
|
||||
return notes
|
||||
}
|
||||
|
||||
function removeDiacritics(value) {
|
||||
return value.normalize('NFD').replace(/\p{Diacritic}/gu, '').normalize('NFC')
|
||||
}
|
||||
|
||||
// Mirrors the plugin's indexing tokenizer. In particular, apostrophes are
|
||||
// preserved as part of a token rather than being treated as separators.
|
||||
function tokenize(text) {
|
||||
return text
|
||||
.split(/[|\[\]()<>{} \t\n\r]+/u)
|
||||
.flatMap(word => [word, ...word.split(/[.,:;!?/\\_#%&*=^@-]+/u)])
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
function processTerm(term) {
|
||||
const processed = removeDiacritics(term).toLowerCase()
|
||||
return processed.length < 3 || STOP_WORDS.has(processed) ? null : processed
|
||||
}
|
||||
|
||||
function headingFields(text) {
|
||||
const lines = text.split(/\r?\n/)
|
||||
const fields = { headings1: [], headings2: [], headings3: [], markdownHeadings: [] }
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
const match = lines[index].match(/^(#{1,6})\s+(.+?)\s*#*\s*$/)
|
||||
if (!match) continue
|
||||
const level = match[1].length
|
||||
if (level <= 3) fields[`headings${level}`].push(match[2])
|
||||
fields.markdownHeadings.push(match[2])
|
||||
}
|
||||
// Match DocumentsRepository's colon-heading promotion rule.
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
const line = lines[index].trim()
|
||||
const previous = index > 0 ? lines[index - 1].trim() : null
|
||||
const next = index < lines.length - 1 ? lines[index + 1].trim() : null
|
||||
if (line.endsWith(':') && previous === '' && next !== null && next !== '') {
|
||||
fields.headings3.push(line.slice(0, -1).trim())
|
||||
}
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
function buildIndex(notes) {
|
||||
const index = new MiniSearch({
|
||||
idField: 'path',
|
||||
fields: ['basename', 'directory', 'aliases', 'content', 'headings1', 'headings2', 'headings3'],
|
||||
tokenize,
|
||||
processTerm,
|
||||
})
|
||||
index.addAll(notes.map(note => {
|
||||
const fields = headingFields(note.text)
|
||||
return {
|
||||
path: note.path,
|
||||
basename: note.basename,
|
||||
directory: path.dirname(note.relative),
|
||||
aliases: '',
|
||||
content: note.text,
|
||||
headings1: fields.headings1.join(' '),
|
||||
headings2: fields.headings2.join(' '),
|
||||
headings3: fields.headings3.join(' '),
|
||||
}
|
||||
}))
|
||||
return index
|
||||
}
|
||||
|
||||
function search(index, notesByPath, query) {
|
||||
const terms = tokenize(query).map(processTerm).filter(Boolean)
|
||||
if (!terms.length) return []
|
||||
const results = index.search(terms.join(' '), {
|
||||
prefix: term => term.length >= 1,
|
||||
bm25: { b: 0.2, d: 0.5, k: 1.2 },
|
||||
fuzzy: term => term.length <= 4 ? 0 : term.length <= 5 ? 0.05 : 0.1,
|
||||
boost: { basename: 10, directory: 7, headings1: 6, headings2: 5, headings3: 4 },
|
||||
})
|
||||
return results.map(result => {
|
||||
const note = notesByPath.get(result.id)
|
||||
const lower = note.text.toLocaleLowerCase()
|
||||
const fields = headingFields(note.text)
|
||||
const titleTerm = terms.find(term => note.basename.toLocaleLowerCase().includes(term))
|
||||
const headingTerm = result.terms.find(term => fields.markdownHeadings.some(heading => heading.toLowerCase().includes(term)))
|
||||
const offset = titleTerm ? 0 : headingTerm
|
||||
? lower.indexOf(headingTerm)
|
||||
: Math.max(0, lower.indexOf(result.terms[0] || terms[0]))
|
||||
return { note, terms: result.terms.length ? result.terms : terms, score: result.score, offset }
|
||||
})
|
||||
}
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
}
|
||||
|
||||
function htmlExcerptToTerminal(html) {
|
||||
const text = html
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<span[^>]*>([\s\S]*?)<\/span>/gi, `${ANSI.yellow}${ANSI.bold}$1${ANSI.reset}${ANSI.dim}`)
|
||||
.replace(/<[^>]+>/g, '')
|
||||
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
||||
.replace(/"/g, '"').replace(/'/g, "'")
|
||||
const unindented = text.split('\n').map(line => line.replace(/^\s+/, ''))
|
||||
return ANSI.dim + unindented.slice(0, 3).join('\n') + ANSI.reset
|
||||
}
|
||||
|
||||
function highlight(text, terms, restore = '') {
|
||||
if (!terms.length) return text
|
||||
const expression = new RegExp(`(${terms.sort((a, b) => b.length - a.length).map(escapeRegExp).join('|')})`, 'giu')
|
||||
return text.replace(expression, `${ANSI.yellow}${ANSI.bold}$1${ANSI.reset}${restore}`)
|
||||
}
|
||||
|
||||
// Use the exact matches returned by TextProcessor rather than MiniSearch's
|
||||
// terms. This preserves complete matches such as "OCD" and "OCD's".
|
||||
function highlightMatches(text, matches, additionalTerms = []) {
|
||||
const terms = [...new Set([
|
||||
...matches.map(match => match.match),
|
||||
...additionalTerms,
|
||||
])].sort((a, b) => b.length - a.length)
|
||||
if (!terms.length) return text
|
||||
const expression = new RegExp(`(${terms.map(escapeRegExp).join('|')})`, 'giu')
|
||||
return text.replace(expression, `${ANSI.yellow}${ANSI.bold}$1${ANSI.reset}${ANSI.dim}`)
|
||||
}
|
||||
|
||||
function excerpt(note, result) {
|
||||
// Obsidian's result preview is short and keeps the note's line breaks.
|
||||
const lines = note.text.split(/\r?\n/)
|
||||
let offset = 0
|
||||
let matchLine = 0
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
if (result.offset <= offset + lines[index].length) {
|
||||
matchLine = index
|
||||
break
|
||||
}
|
||||
offset += lines[index].length + 1
|
||||
}
|
||||
const firstLine = Math.max(0, matchLine - 1)
|
||||
const lastLine = Math.min(lines.length, firstLine + 3)
|
||||
const visible = lines.slice(firstLine, lastLine)
|
||||
const prefix = firstLine ? '…' : ''
|
||||
const suffix = lastLine < lines.length ? '…' : ''
|
||||
return ANSI.dim + prefix + visible.map(line => highlight(line, result.terms, ANSI.dim)).join('\n') + suffix + ANSI.reset
|
||||
}
|
||||
|
||||
function loadNotes(directory) {
|
||||
return walk(directory).map(filePath => ({
|
||||
path: filePath,
|
||||
basename: path.basename(filePath, '.md'),
|
||||
relative: path.relative(directory, filePath),
|
||||
text: fs.readFileSync(filePath, 'utf8'),
|
||||
}))
|
||||
}
|
||||
|
||||
class SearchTUI {
|
||||
constructor(directory) {
|
||||
this.directory = directory
|
||||
this.pluginSearch = null
|
||||
this.searchGeneration = 0
|
||||
this.notes = loadNotes(directory)
|
||||
this.notesByPath = new Map(this.notes.map(note => [note.path, note]))
|
||||
this.index = buildIndex(this.notes)
|
||||
this.query = ''
|
||||
this.results = []
|
||||
this.selected = 0
|
||||
this.scrollOffset = 0
|
||||
this.mode = 'search'
|
||||
this.openResult = null
|
||||
this.noteScrollLine = 0
|
||||
}
|
||||
|
||||
async init() {
|
||||
if (process.env.TUI_PLUGIN_SEARCH) {
|
||||
const { createPluginSearch } = require(process.env.TUI_PLUGIN_SEARCH)
|
||||
this.pluginSearch = await createPluginSearch(this.notes.map(note => ({
|
||||
path: note.relative,
|
||||
basename: note.basename,
|
||||
text: note.text,
|
||||
})))
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
start() {
|
||||
if (!process.stdin.isTTY || !process.stdout.isTTY) {
|
||||
throw new Error('The search TUI requires an interactive terminal (TTY).')
|
||||
}
|
||||
process.stdin.setRawMode(true)
|
||||
process.stdout.write(ANSI.hideCursor)
|
||||
process.stdin.resume()
|
||||
process.stdin.setEncoding('utf8')
|
||||
process.stdin.on('data', key => this.onKey(key))
|
||||
this.render()
|
||||
}
|
||||
|
||||
stop() {
|
||||
process.stdin.setRawMode(false)
|
||||
process.stdin.pause()
|
||||
process.stdin.removeAllListeners('data')
|
||||
// DocumentsRepository owns a repeating cache timer, so merely pausing
|
||||
// stdin leaves the process alive. Restore the terminal and exit explicitly.
|
||||
process.stdout.write(`${ANSI.showCursor}\n`)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
onKey(key) {
|
||||
if (key === '\u0003') return this.stop() // Ctrl-C
|
||||
if (this.mode === 'note') {
|
||||
if (key === 'q' || key === '\u001b' || key === '\u001b\u001b') {
|
||||
this.mode = 'search'
|
||||
this.render()
|
||||
} else if (key === '\u001b[A') {
|
||||
this.noteScrollLine = Math.max(0, this.noteScrollLine - 1)
|
||||
this.render()
|
||||
} else if (key === '\u001b[B') {
|
||||
this.noteScrollLine += 1
|
||||
this.ensureNoteScrollBounds()
|
||||
this.render()
|
||||
} else if (key === '\u001b[5~') {
|
||||
this.noteScrollLine = Math.max(0, this.noteScrollLine - this.visibleNoteLines())
|
||||
this.render()
|
||||
} else if (key === '\u001b[6~') {
|
||||
this.noteScrollLine += this.visibleNoteLines()
|
||||
this.ensureNoteScrollBounds()
|
||||
this.render()
|
||||
} else if (key === '\u001b[H' || key === '\u001b[1~') {
|
||||
this.noteScrollLine = 0
|
||||
this.render()
|
||||
} else if (key === '\u001b[F' || key === '\u001b[4~') {
|
||||
this.noteScrollLine = Number.MAX_SAFE_INTEGER
|
||||
this.ensureNoteScrollBounds()
|
||||
this.render()
|
||||
}
|
||||
return
|
||||
}
|
||||
if (key === '\u0008' || key === '\u007f') {
|
||||
this.query = this.query.slice(0, -1)
|
||||
this.updateResults()
|
||||
} else if (key === '\r' || key === '\n') {
|
||||
if (this.results.length) {
|
||||
this.openResult = this.results[this.selected]
|
||||
this.mode = 'note'
|
||||
this.noteScrollLine = 0
|
||||
this.ensureNoteCursorVisible()
|
||||
this.render()
|
||||
}
|
||||
} else if (key === '\u001b[A') {
|
||||
this.selected = Math.max(0, this.selected - 1)
|
||||
this.ensureSelectionVisible()
|
||||
this.render()
|
||||
} else if (key === '\u001b[B') {
|
||||
this.selected = Math.min(Math.max(0, this.results.length - 1), this.selected + 1)
|
||||
this.ensureSelectionVisible()
|
||||
this.render()
|
||||
} else if (key === '\u001b[5~') {
|
||||
this.selected = Math.max(0, this.selected - this.visibleResultCount())
|
||||
this.ensureSelectionVisible()
|
||||
this.render()
|
||||
} else if (key === '\u001b[6~') {
|
||||
this.selected = Math.min(Math.max(0, this.results.length - 1), this.selected + this.visibleResultCount())
|
||||
this.ensureSelectionVisible()
|
||||
this.render()
|
||||
} else if (key === '\u001b[H' || key === '\u001b[1~') {
|
||||
this.selected = 0
|
||||
this.ensureSelectionVisible()
|
||||
this.render()
|
||||
} else if (key === '\u001b[F' || key === '\u001b[4~') {
|
||||
this.selected = Math.max(0, this.results.length - 1)
|
||||
this.ensureSelectionVisible()
|
||||
this.render()
|
||||
} else if (key === '\u001b') {
|
||||
this.query = ''
|
||||
this.updateResults()
|
||||
} else if (key >= ' ' && key !== '\u007f') {
|
||||
this.query += key
|
||||
this.updateResults()
|
||||
}
|
||||
}
|
||||
|
||||
visibleNoteLines() {
|
||||
return Math.max(1, (process.stdout.rows || 24) - 6)
|
||||
}
|
||||
|
||||
noteCursorPosition() {
|
||||
if (!this.openResult) return { line: 0, column: 0 }
|
||||
let remaining = this.openResult.offset
|
||||
const lines = this.openResult.note.text.split(/\r?\n/)
|
||||
for (let line = 0; line < lines.length; line++) {
|
||||
if (remaining <= lines[line].length) return { line, column: Math.max(0, remaining) }
|
||||
remaining -= lines[line].length + 1
|
||||
}
|
||||
return { line: lines.length - 1, column: lines.at(-1)?.length ?? 0 }
|
||||
}
|
||||
|
||||
ensureNoteScrollBounds() {
|
||||
if (!this.openResult) return
|
||||
const lineCount = this.openResult.note.text.split(/\r?\n/).length
|
||||
this.noteScrollLine = Math.max(0, Math.min(this.noteScrollLine, Math.max(0, lineCount - this.visibleNoteLines())))
|
||||
}
|
||||
|
||||
ensureNoteCursorVisible() {
|
||||
const cursor = this.noteCursorPosition()
|
||||
const visible = this.visibleNoteLines()
|
||||
this.noteScrollLine = cursor.line - Math.floor(visible / 2)
|
||||
this.ensureNoteScrollBounds()
|
||||
}
|
||||
|
||||
visibleResultCount() {
|
||||
// Each result occupies a title line, three excerpt lines, and a spacer.
|
||||
// Keep the header and help text visible while scrolling.
|
||||
return Math.max(1, Math.floor(((process.stdout.rows || 24) - 6) / 5))
|
||||
}
|
||||
|
||||
ensureSelectionVisible() {
|
||||
const visible = this.visibleResultCount()
|
||||
if (this.selected < this.scrollOffset) this.scrollOffset = this.selected
|
||||
if (this.selected >= this.scrollOffset + visible) {
|
||||
this.scrollOffset = this.selected - visible + 1
|
||||
}
|
||||
this.scrollOffset = Math.max(0, Math.min(
|
||||
this.scrollOffset,
|
||||
Math.max(0, this.results.length - visible),
|
||||
))
|
||||
}
|
||||
|
||||
async updateResults() {
|
||||
const generation = ++this.searchGeneration
|
||||
if (this.pluginSearch) {
|
||||
const ranked = await this.pluginSearch(this.query)
|
||||
if (generation !== this.searchGeneration) return
|
||||
this.results = ranked.map(result => {
|
||||
const note = this.notes.find(candidate => candidate.relative === result.id)
|
||||
return {
|
||||
note,
|
||||
terms: result.terms,
|
||||
matches: result.matches,
|
||||
score: result.score,
|
||||
offset: result.offset,
|
||||
excerpt: highlightMatches(
|
||||
htmlExcerptToTerminal(result.excerpt),
|
||||
result.matches,
|
||||
result.terms,
|
||||
),
|
||||
}
|
||||
}).filter(result => result.note)
|
||||
} else {
|
||||
this.results = search(this.index, this.notesByPath, this.query)
|
||||
}
|
||||
this.selected = Math.min(this.selected, Math.max(0, this.results.length - 1))
|
||||
this.ensureSelectionVisible()
|
||||
this.render()
|
||||
}
|
||||
|
||||
render() {
|
||||
process.stdout.write(ANSI.clear)
|
||||
if (this.mode === 'note') return this.renderNote()
|
||||
process.stdout.write(`${ANSI.bold}${ANSI.cyan}Tannersearch TUI${ANSI.reset} ${ANSI.dim}${this.directory}${ANSI.reset}\n\n`)
|
||||
process.stdout.write(`${ANSI.bold}Search:${ANSI.reset} ${this.query}${ANSI.cyan}▌${ANSI.reset}\n`)
|
||||
process.stdout.write(`${ANSI.dim}Type to search · ↑/↓ select · Enter open · Esc clear · Ctrl-C quit${ANSI.reset}\n\n`)
|
||||
if (!this.query) {
|
||||
process.stdout.write(`${ANSI.dim}Start typing to search ${this.notes.length} Markdown notes.${ANSI.reset}\n`)
|
||||
return
|
||||
}
|
||||
if (!this.results.length) {
|
||||
process.stdout.write(`${ANSI.yellow}No matching notes.${ANSI.reset}\n`)
|
||||
return
|
||||
}
|
||||
const visible = this.visibleResultCount()
|
||||
this.ensureSelectionVisible()
|
||||
const shown = this.results.slice(this.scrollOffset, this.scrollOffset + visible)
|
||||
shown.forEach((result, visibleIndex) => {
|
||||
const index = this.scrollOffset + visibleIndex
|
||||
const marker = index === this.selected ? `${ANSI.inverse}>${ANSI.reset}` : ' '
|
||||
process.stdout.write(`${marker} ${ANSI.bold}${result.note.basename}${ANSI.reset} ${ANSI.dim}(${result.note.relative})${ANSI.reset}\n`)
|
||||
process.stdout.write(`${result.excerpt || excerpt(result.note, result)}\n\n`)
|
||||
})
|
||||
if (this.results.length > visible) {
|
||||
process.stdout.write(`${ANSI.dim}Showing ${this.scrollOffset + 1}-${Math.min(this.scrollOffset + visible, this.results.length)} of ${this.results.length} · PageUp/PageDown to scroll${ANSI.reset}\n`)
|
||||
}
|
||||
}
|
||||
|
||||
renderNote() {
|
||||
const result = this.openResult
|
||||
const note = result.note
|
||||
process.stdout.write(`${ANSI.bold}${ANSI.green}${note.basename}${ANSI.reset} ${ANSI.dim}— fake cursor at match${ANSI.reset}\n`)
|
||||
process.stdout.write(`${ANSI.dim}Press q or Esc to return to results.${ANSI.reset}\n\n`)
|
||||
const lines = note.text.split(/\r?\n/)
|
||||
const cursor = this.noteCursorPosition()
|
||||
const visible = lines.slice(this.noteScrollLine, this.noteScrollLine + this.visibleNoteLines())
|
||||
visible.forEach((line, index) => {
|
||||
const lineNumber = this.noteScrollLine + index
|
||||
if (lineNumber === cursor.line) {
|
||||
process.stdout.write(`${highlight(line.slice(0, cursor.column), result.terms)}${ANSI.green}▌${ANSI.reset}${highlight(line.slice(cursor.column), result.terms)}\n`)
|
||||
process.stdout.write(`${' '.repeat(cursor.column)}${ANSI.green}^ cursor${ANSI.reset}\n`)
|
||||
} else {
|
||||
process.stdout.write(`${highlight(line, result.terms)}\n`)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const directory = path.resolve(process.argv[2] || DEFAULT_DIRECTORY)
|
||||
try {
|
||||
if (!fs.statSync(directory).isDirectory()) throw new Error(`Not a directory: ${directory}`)
|
||||
new SearchTUI(directory).init().then(tui => tui.start())
|
||||
} catch (error) {
|
||||
console.error(`search-tui: ${error.message}`)
|
||||
process.exitCode = 1
|
||||
}
|
||||
Reference in New Issue
Block a user