diff --git a/src/globals.ts b/src/globals.ts index 7d5531a..572729b 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -23,6 +23,8 @@ export type SearchNote = { export type IndexedNote = { path: string basename: string + mtime: number + content: string aliases: string headings1: string diff --git a/src/notes.ts b/src/notes.ts index e37466d..6713730 100644 --- a/src/notes.ts +++ b/src/notes.ts @@ -6,6 +6,7 @@ import { } from 'obsidian' import type { IndexedNote, ResultNote } from './globals' import { stringsToRegex } from './utils' +import { settings } from './settings' /** * This is an in-memory cache of the notes, with all their computed fields @@ -14,9 +15,29 @@ import { stringsToRegex } from './utils' */ export let notesCache: Record = {} +const notesCacheFilePath = `${app.vault.configDir}/plugins/omnisearch/notesCache.json` + export function resetNotesCache(): void { notesCache = {} } + +export async function loadNotesCache(): Promise { + if (settings.storeIndexInFile && await app.vault.adapter.exists(notesCacheFilePath)) { + try { + const json = await app.vault.adapter.read(notesCacheFilePath) + notesCache = JSON.parse(json) + console.log('Notes cache loaded from the file') + } + catch(e) { + console.trace('Could not load Notes cache from the file') + console.error(e) + } + } + + if (!notesCache) { + notesCache = {} + } +} export function getNoteFromCache(key: string): IndexedNote | undefined { return notesCache[key] } @@ -117,3 +138,14 @@ export function getNonExistingNotes( export function removeAnchors(name: string): string { return name.split(/[\^#]+/)[0] } + +export async function saveNotesCacheToFile(): Promise { + const json = JSON.stringify(notesCache) + await app.vault.adapter.write(notesCacheFilePath, json) + console.log('Notes cache saved to the file') +} + +export function isCacheOutdated(file: TFile): boolean { + const indexedNote = getNoteFromCache(file.path) + return !indexedNote || indexedNote.mtime !== file.stat.mtime +} diff --git a/src/search.ts b/src/search.ts index 637a3d7..0b2c8ab 100644 --- a/src/search.ts +++ b/src/search.ts @@ -25,9 +25,14 @@ import { addNoteToCache, removeAnchors, getNonExistingNotesFromCache, + loadNotesCache, + saveNotesCacheToFile, + isCacheOutdated } from './notes' let minisearchInstance: MiniSearch +let isIndexChanged: boolean +const searchIndexFilePath = `${app.vault.configDir}/plugins/omnisearch/searchIndex.json` const tokenize = (text: string): string[] => { const tokens = text.split(SPACE_OR_PUNCTUATION) @@ -46,8 +51,7 @@ const tokenize = (text: string): string[] => { * and adds all the notes to the index */ export async function initGlobalSearchIndex(): Promise { - resetNotesCache() - minisearchInstance = new MiniSearch({ + const options = { tokenize, processTerm: term => settings.ignoreDiacritics ? removeDiacritics(term) : term, @@ -60,29 +64,68 @@ export async function initGlobalSearchIndex(): Promise { 'headings2', 'headings3', ], - }) + } + + if (settings.storeIndexInFile && await app.vault.adapter.exists(searchIndexFilePath)) { + try { + const json = await app.vault.adapter.read(searchIndexFilePath) + minisearchInstance = MiniSearch.loadJSON(json, options) + console.log('MiniSearch index loaded from the file') + await loadNotesCache() + } + catch(e) { + console.trace('Could not load MiniSearch index from the file') + console.error(e) + } + } + + if (!minisearchInstance) { + minisearchInstance = new MiniSearch(options) + resetNotesCache() + } // Index files that are already present const start = new Date().getTime() - const files = app.vault.getMarkdownFiles() + + const allFiles = app.vault.getMarkdownFiles() + + let files + let notesSuffix + if (settings.storeIndexInFile) { + files = allFiles.filter(file => isCacheOutdated(file)) + notesSuffix = 'modified notes' + } else { + files = allFiles + notesSuffix = 'notes' + } + + console.log(`Omnisearch - indexing ${files.length} ${notesSuffix}`) // This is basically the same behavior as MiniSearch's `addAllAsync()`. // We index files by batches of 10 - if (files.length) { - console.log('Omnisearch - indexing ' + files.length + ' files') - } for (let i = 0; i < files.length; ++i) { if (i % 10 === 0) await wait(0) const file = files[i] - if (file) await addToIndex(file) + if (file) { + if (getNoteFromCache(file.path)) { + removeFromIndex(file.path) + } + await addToIndex(file) + } } - if (files.length > 0 && settings.showIndexingNotices) { - new Notice( - `Omnisearch - Indexed ${files.length} notes in ${ + if (files.length > 0) { + const message = `Omnisearch - Indexed ${files.length} ${notesSuffix} in ${ new Date().getTime() - start - }ms`, - ) + }ms` + + console.log(message) + + if (settings.showIndexingNotices) { + new Notice(message) + } + + await saveIndexToFile() } } @@ -258,6 +301,8 @@ export async function addToIndex(file: TAbstractFile): Promise { basename: file.basename, content, path: file.path, + mtime: file.stat.mtime, + aliases: getAliasesFromMetadata(metadata).join(''), headings1: metadata ? extractHeadingsFromCache(metadata, 1).join(' ') @@ -271,6 +316,7 @@ export async function addToIndex(file: TAbstractFile): Promise { } minisearchInstance.add(note) + isIndexChanged = true addNoteToCache(note.path, note) } catch (e) { @@ -292,6 +338,8 @@ export function addNonExistingToIndex(name: string, parent: string): void { const note = { path: filename, basename: name, + mtime: 0, + content: '', aliases: '', headings1: '', @@ -302,6 +350,7 @@ export function addNonExistingToIndex(name: string, parent: string): void { parent, } as IndexedNote minisearchInstance.add(note) + isIndexChanged = true addNoteToCache(filename, note) } @@ -317,6 +366,7 @@ export function removeFromIndex(path: string): void { const note = getNoteFromCache(path) if (note) { minisearchInstance.remove(note) + isIndexChanged = true removeNoteFromCache(path) getNonExistingNotesFromCache() .filter(n => n.parent === path) @@ -339,4 +389,17 @@ export async function reindexNotes(): Promise { await addToIndex(note) } notesToReindex.clear() + + await saveIndexToFile() +} + +async function saveIndexToFile(): Promise { + if (settings.storeIndexInFile && minisearchInstance && isIndexChanged) { + const json = JSON.stringify(minisearchInstance) + await app.vault.adapter.write(searchIndexFilePath, json) + console.log('MiniSearch index saved to the file') + + await saveNotesCacheToFile() + isIndexChanged = false + } } diff --git a/src/settings.ts b/src/settings.ts index c284629..5e3d369 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -16,6 +16,7 @@ export interface OmnisearchSettings extends WeightingSettings { showShortName: boolean CtrlJK: boolean CtrlNP: boolean + storeIndexInFile: boolean } export class SettingsTab extends PluginSettingTab { @@ -82,6 +83,18 @@ export class SettingsTab extends PluginSettingTab { // }), // ) + new Setting(containerEl) + .setName('Store index in file') + .setDesc( + 'EXPERIMENTAL - index is store on disk, instead of being rebuilt on every startup.', + ) + .addToggle(toggle => + toggle.setValue(settings.storeIndexInFile).onChange(async v => { + settings.storeIndexInFile = v + await saveSettings(this.plugin) + }), + ) + // #endregion Behavior // #region User Interface @@ -193,6 +206,8 @@ export const DEFAULT_SETTINGS: OmnisearchSettings = { CtrlJK: false, CtrlNP: false, + + storeIndexInFile: false, } as const export let settings: OmnisearchSettings = Object.assign({}, DEFAULT_SETTINGS)