From 7cd4f6c502319e31b8e54447ea9716b0b9d03f24 Mon Sep 17 00:00:00 2001 From: Michael Naumov Date: Mon, 6 Jun 2022 00:42:29 -0600 Subject: [PATCH] Save/load index to files, improve logging --- src/globals.ts | 4 ++- src/notes.ts | 31 +++++++++++++++++++ src/search.ts | 81 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 101 insertions(+), 15 deletions(-) diff --git a/src/globals.ts b/src/globals.ts index 7d5531a..b51ad3e 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -22,7 +22,9 @@ export type SearchNote = { export type IndexedNote = { path: string - basename: string + basename: string, + mtime: number, + content: string aliases: string headings1: string diff --git a/src/notes.ts b/src/notes.ts index e37466d..4bebb2c 100644 --- a/src/notes.ts +++ b/src/notes.ts @@ -14,9 +14,29 @@ import { stringsToRegex } from './utils' */ export let notesCache: Record = {} +const notesCacheFilePath = `${app.vault.configDir}/plugins/omnisearch/notesCache.json` + export function resetNotesCache(): void { notesCache = {} } + +export async function loadNotesCache(): Promise { + if (await app.vault.adapter.exists(notesCacheFilePath)) { + try { + const json = await app.vault.adapter.read(notesCacheFilePath) + notesCache = JSON.parse(json) + console.log("Notes cache loaded from the file") + } + catch(e) { + console.trace("Could not load Notes cache from the file") + console.error(e) + } + } + + if (!notesCache) { + notesCache = {} + } +} export function getNoteFromCache(key: string): IndexedNote | undefined { return notesCache[key] } @@ -117,3 +137,14 @@ export function getNonExistingNotes( export function removeAnchors(name: string): string { return name.split(/[\^#]+/)[0] } + +export async function saveNotesCacheToFile(): Promise { + const json = JSON.stringify(notesCache) + await app.vault.adapter.write(notesCacheFilePath, json) + console.log("Notes cache saved to the file") +} + +export function isCacheOutdated(file: TFile): boolean { + const indexedNote = getNoteFromCache(file.path) + return !indexedNote || indexedNote.mtime !== file.stat.mtime +} \ No newline at end of file diff --git a/src/search.ts b/src/search.ts index 74bcd37..f9ee16f 100644 --- a/src/search.ts +++ b/src/search.ts @@ -24,9 +24,14 @@ import { addNoteToCache, removeAnchors, getNonExistingNotesFromCache, + loadNotesCache, + saveNotesCacheToFile, + isCacheOutdated } from './notes' let minisearchInstance: MiniSearch +let isIndexChanged: boolean +const searchIndexFilePath = `${app.vault.configDir}/plugins/omnisearch/searchIndex.json` const tokenize = (text: string): string[] => { const tokens = text.split(SPACE_OR_PUNCTUATION) @@ -45,8 +50,7 @@ const tokenize = (text: string): string[] => { * and adds all the notes to the index */ export async function initGlobalSearchIndex(): Promise { - resetNotesCache() - minisearchInstance = new MiniSearch({ + const options = { tokenize, idField: 'path', fields: [ @@ -57,29 +61,58 @@ export async function initGlobalSearchIndex(): Promise { 'headings2', 'headings3', ], - }) + } + + if (await app.vault.adapter.exists(searchIndexFilePath)) { + try { + const json = await app.vault.adapter.read(searchIndexFilePath) + minisearchInstance = MiniSearch.loadJSON(json, options) + console.log("MiniSearch index loaded from the file") + } + catch(e) { + console.trace("Could not load MiniSearch index from the file") + console.error(e) + } + } + + if (!minisearchInstance) { + minisearchInstance = new MiniSearch(options) + resetNotesCache() + } + + await loadNotesCache() // Index files that are already present const start = new Date().getTime() - const files = app.vault.getMarkdownFiles() + const files = app.vault.getMarkdownFiles().filter(file => isCacheOutdated(file)) // This is basically the same behavior as MiniSearch's `addAllAsync()`. // We index files by batches of 10 - if (files.length) { - console.log('Omnisearch - indexing ' + files.length + ' files') - } + console.log('Omnisearch - indexing ' + files.length + ' modified notes') + for (let i = 0; i < files.length; ++i) { if (i % 10 === 0) await wait(0) const file = files[i] - if (file) await addToIndex(file) + if (file) { + if (getNoteFromCache(file.path)) { + removeFromIndex(file.path) + } + await addToIndex(file) + } } - if (files.length > 0 && settings.showIndexingNotices) { - new Notice( - `Omnisearch - Indexed ${files.length} notes in ${ - new Date().getTime() - start - }ms`, - ) + if (files.length > 0) { + const message = `Omnisearch - Indexed ${files.length} modified notes in ${ + new Date().getTime() - start + }ms` + + console.log(message) + + if (settings.showIndexingNotices) { + new Notice(message) + } + + await saveIndexToFile() } } @@ -255,6 +288,8 @@ export async function addToIndex(file: TAbstractFile): Promise { basename: file.basename, content, path: file.path, + mtime: file.stat.mtime, + aliases: getAliasesFromMetadata(metadata).join(''), headings1: metadata ? extractHeadingsFromCache(metadata, 1).join(' ') @@ -268,6 +303,7 @@ export async function addToIndex(file: TAbstractFile): Promise { } minisearchInstance.add(note) + isIndexChanged = true addNoteToCache(note.path, note) } catch (e) { @@ -289,6 +325,8 @@ export function addNonExistingToIndex(name: string, parent: string): void { const note = { path: filename, basename: name, + mtime: 0, + content: '', aliases: '', headings1: '', @@ -299,6 +337,7 @@ export function addNonExistingToIndex(name: string, parent: string): void { parent, } as IndexedNote minisearchInstance.add(note) + isIndexChanged = true addNoteToCache(filename, note) } @@ -314,6 +353,7 @@ export function removeFromIndex(path: string): void { const note = getNoteFromCache(path) if (note) { minisearchInstance.remove(note) + isIndexChanged = true removeNoteFromCache(path) getNonExistingNotesFromCache() .filter(n => n.parent === path) @@ -336,4 +376,17 @@ export async function reindexNotes(): Promise { await addToIndex(note) } notesToReindex.clear() + + await saveIndexToFile() } + +async function saveIndexToFile(): Promise { + if (minisearchInstance && isIndexChanged) { + const json = JSON.stringify(minisearchInstance) + await app.vault.adapter.write(searchIndexFilePath, json) + console.log("MiniSearch index saved to the file") + + await saveNotesCacheToFile() + isIndexChanged = false + } +} \ No newline at end of file