From 7cd4f6c502319e31b8e54447ea9716b0b9d03f24 Mon Sep 17 00:00:00 2001 From: Michael Naumov Date: Mon, 6 Jun 2022 00:42:29 -0600 Subject: [PATCH 1/5] Save/load index to files, improve logging --- src/globals.ts | 4 ++- src/notes.ts | 31 +++++++++++++++++++ src/search.ts | 81 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 101 insertions(+), 15 deletions(-) diff --git a/src/globals.ts b/src/globals.ts index 7d5531a..b51ad3e 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -22,7 +22,9 @@ export type SearchNote = { export type IndexedNote = { path: string - basename: string + basename: string, + mtime: number, + content: string aliases: string headings1: string diff --git a/src/notes.ts b/src/notes.ts index e37466d..4bebb2c 100644 --- a/src/notes.ts +++ b/src/notes.ts @@ -14,9 +14,29 @@ import { stringsToRegex } from './utils' */ export let notesCache: Record = {} +const notesCacheFilePath = `${app.vault.configDir}/plugins/omnisearch/notesCache.json` + export function resetNotesCache(): void { notesCache = {} } + +export async function loadNotesCache(): Promise { + if (await app.vault.adapter.exists(notesCacheFilePath)) { + try { + const json = await app.vault.adapter.read(notesCacheFilePath) + notesCache = JSON.parse(json) + console.log("Notes cache loaded from the file") + } + catch(e) { + console.trace("Could not load Notes cache from the file") + console.error(e) + } + } + + if (!notesCache) { + notesCache = {} + } +} export function getNoteFromCache(key: string): IndexedNote | undefined { return notesCache[key] } @@ -117,3 +137,14 @@ export function getNonExistingNotes( export function removeAnchors(name: string): string { return name.split(/[\^#]+/)[0] } + +export async function saveNotesCacheToFile(): Promise { + const json = JSON.stringify(notesCache) + await app.vault.adapter.write(notesCacheFilePath, json) + console.log("Notes cache saved to the file") +} + +export function isCacheOutdated(file: TFile): boolean { + const indexedNote = getNoteFromCache(file.path) + return !indexedNote || indexedNote.mtime !== file.stat.mtime +} \ No newline at end of file diff --git a/src/search.ts b/src/search.ts index 74bcd37..f9ee16f 100644 --- a/src/search.ts +++ b/src/search.ts @@ -24,9 +24,14 @@ import { addNoteToCache, removeAnchors, getNonExistingNotesFromCache, + loadNotesCache, + saveNotesCacheToFile, + isCacheOutdated } from './notes' let minisearchInstance: MiniSearch +let isIndexChanged: boolean +const searchIndexFilePath = `${app.vault.configDir}/plugins/omnisearch/searchIndex.json` const tokenize = (text: string): string[] => { const tokens = text.split(SPACE_OR_PUNCTUATION) @@ -45,8 +50,7 @@ const tokenize = (text: string): string[] => { * and adds all the notes to the index */ export async function initGlobalSearchIndex(): Promise { - resetNotesCache() - minisearchInstance = new MiniSearch({ + const options = { tokenize, idField: 'path', fields: [ @@ -57,29 +61,58 @@ export async function initGlobalSearchIndex(): Promise { 'headings2', 'headings3', ], - }) + } + + if (await app.vault.adapter.exists(searchIndexFilePath)) { + try { + const json = await app.vault.adapter.read(searchIndexFilePath) + minisearchInstance = MiniSearch.loadJSON(json, options) + console.log("MiniSearch index loaded from the file") + } + catch(e) { + console.trace("Could not load MiniSearch index from the file") + console.error(e) + } + } + + if (!minisearchInstance) { + minisearchInstance = new MiniSearch(options) + resetNotesCache() + } + + await loadNotesCache() // Index files that are already present const start = new Date().getTime() - const files = app.vault.getMarkdownFiles() + const files = app.vault.getMarkdownFiles().filter(file => isCacheOutdated(file)) // This is basically the same behavior as MiniSearch's `addAllAsync()`. // We index files by batches of 10 - if (files.length) { - console.log('Omnisearch - indexing ' + files.length + ' files') - } + console.log('Omnisearch - indexing ' + files.length + ' modified notes') + for (let i = 0; i < files.length; ++i) { if (i % 10 === 0) await wait(0) const file = files[i] - if (file) await addToIndex(file) + if (file) { + if (getNoteFromCache(file.path)) { + removeFromIndex(file.path) + } + await addToIndex(file) + } } - if (files.length > 0 && settings.showIndexingNotices) { - new Notice( - `Omnisearch - Indexed ${files.length} notes in ${ - new Date().getTime() - start - }ms`, - ) + if (files.length > 0) { + const message = `Omnisearch - Indexed ${files.length} modified notes in ${ + new Date().getTime() - start + }ms` + + console.log(message) + + if (settings.showIndexingNotices) { + new Notice(message) + } + + await saveIndexToFile() } } @@ -255,6 +288,8 @@ export async function addToIndex(file: TAbstractFile): Promise { basename: file.basename, content, path: file.path, + mtime: file.stat.mtime, + aliases: getAliasesFromMetadata(metadata).join(''), headings1: metadata ? extractHeadingsFromCache(metadata, 1).join(' ') @@ -268,6 +303,7 @@ export async function addToIndex(file: TAbstractFile): Promise { } minisearchInstance.add(note) + isIndexChanged = true addNoteToCache(note.path, note) } catch (e) { @@ -289,6 +325,8 @@ export function addNonExistingToIndex(name: string, parent: string): void { const note = { path: filename, basename: name, + mtime: 0, + content: '', aliases: '', headings1: '', @@ -299,6 +337,7 @@ export function addNonExistingToIndex(name: string, parent: string): void { parent, } as IndexedNote minisearchInstance.add(note) + isIndexChanged = true addNoteToCache(filename, note) } @@ -314,6 +353,7 @@ export function removeFromIndex(path: string): void { const note = getNoteFromCache(path) if (note) { minisearchInstance.remove(note) + isIndexChanged = true removeNoteFromCache(path) getNonExistingNotesFromCache() .filter(n => n.parent === path) @@ -336,4 +376,17 @@ export async function reindexNotes(): Promise { await addToIndex(note) } notesToReindex.clear() + + await saveIndexToFile() } + +async function saveIndexToFile(): Promise { + if (minisearchInstance && isIndexChanged) { + const json = JSON.stringify(minisearchInstance) + await app.vault.adapter.write(searchIndexFilePath, json) + console.log("MiniSearch index saved to the file") + + await saveNotesCacheToFile() + isIndexChanged = false + } +} \ No newline at end of file From 9b1123dee8a84147dadf7bd9aa5afa5a4359b5cf Mon Sep 17 00:00:00 2001 From: Michael Naumov Date: Mon, 6 Jun 2022 08:12:51 -0600 Subject: [PATCH 2/5] Add storeIndexInFile setting --- src/notes.ts | 3 ++- src/search.ts | 30 ++++++++++++++++++++---------- src/settings.ts | 12 ++++++++++++ 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/notes.ts b/src/notes.ts index 4bebb2c..9bceaff 100644 --- a/src/notes.ts +++ b/src/notes.ts @@ -6,6 +6,7 @@ import { } from 'obsidian' import type { IndexedNote, ResultNote } from './globals' import { stringsToRegex } from './utils' +import { settings } from './settings' /** * This is an in-memory cache of the notes, with all their computed fields @@ -21,7 +22,7 @@ export function resetNotesCache(): void { } export async function loadNotesCache(): Promise { - if (await app.vault.adapter.exists(notesCacheFilePath)) { + if (settings.storeIndexInFile && await app.vault.adapter.exists(notesCacheFilePath)) { try { const json = await app.vault.adapter.read(notesCacheFilePath) notesCache = JSON.parse(json) diff --git a/src/search.ts b/src/search.ts index f9ee16f..8653a1c 100644 --- a/src/search.ts +++ b/src/search.ts @@ -63,11 +63,12 @@ export async function initGlobalSearchIndex(): Promise { ], } - if (await app.vault.adapter.exists(searchIndexFilePath)) { + if (settings.storeIndexInFile && await app.vault.adapter.exists(searchIndexFilePath)) { try { const json = await app.vault.adapter.read(searchIndexFilePath) minisearchInstance = MiniSearch.loadJSON(json, options) console.log("MiniSearch index loaded from the file") + await loadNotesCache() } catch(e) { console.trace("Could not load MiniSearch index from the file") @@ -80,16 +81,25 @@ export async function initGlobalSearchIndex(): Promise { resetNotesCache() } - await loadNotesCache() - // Index files that are already present const start = new Date().getTime() - const files = app.vault.getMarkdownFiles().filter(file => isCacheOutdated(file)) + + const allFiles = app.vault.getMarkdownFiles() + + let files + let notesSuffix + if (settings.storeIndexInFile) { + files = allFiles.filter(file => isCacheOutdated(file)) + notesSuffix = 'modified notes' + } else { + files = allFiles + notesSuffix = 'notes' + } + + console.log(`Omnisearch - indexing ${files.length} ${notesSuffix}`) // This is basically the same behavior as MiniSearch's `addAllAsync()`. // We index files by batches of 10 - console.log('Omnisearch - indexing ' + files.length + ' modified notes') - for (let i = 0; i < files.length; ++i) { if (i % 10 === 0) await wait(0) const file = files[i] @@ -102,9 +112,9 @@ export async function initGlobalSearchIndex(): Promise { } if (files.length > 0) { - const message = `Omnisearch - Indexed ${files.length} modified notes in ${ - new Date().getTime() - start - }ms` + const message = `Omnisearch - Indexed ${files.length} ${notesSuffix} in ${ + new Date().getTime() - start + }ms` console.log(message) @@ -381,7 +391,7 @@ export async function reindexNotes(): Promise { } async function saveIndexToFile(): Promise { - if (minisearchInstance && isIndexChanged) { + if (settings.storeIndexInFile && minisearchInstance && isIndexChanged) { const json = JSON.stringify(minisearchInstance) await app.vault.adapter.write(searchIndexFilePath, json) console.log("MiniSearch index saved to the file") diff --git a/src/settings.ts b/src/settings.ts index 5c5a836..45ce43a 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -15,6 +15,7 @@ export interface OmnisearchSettings extends WeightingSettings { showShortName: boolean CtrlJK: boolean CtrlNP: boolean + storeIndexInFile: boolean } export class SettingsTab extends PluginSettingTab { @@ -68,6 +69,15 @@ export class SettingsTab extends PluginSettingTab { }), ) + new Setting(containerEl) + .setName('Store index in file') + .addToggle(toggle => + toggle.setValue(settings.storeIndexInFile).onChange(async v => { + settings.storeIndexInFile = v + await saveSettings(this.plugin) + }), + ) + // #endregion Behavior // #region User Interface @@ -174,6 +184,8 @@ export const DEFAULT_SETTINGS: OmnisearchSettings = { CtrlJK: false, CtrlNP: false, + + storeIndexInFile: false } as const export let settings: OmnisearchSettings = Object.assign({}, DEFAULT_SETTINGS) From ff38f8258bf2a2988049ba7dbaf286c21f66eb44 Mon Sep 17 00:00:00 2001 From: Michael Naumov Date: Mon, 6 Jun 2022 08:16:24 -0600 Subject: [PATCH 3/5] Use singe quotes consistently --- src/notes.ts | 6 +++--- src/search.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/notes.ts b/src/notes.ts index 9bceaff..a21bbab 100644 --- a/src/notes.ts +++ b/src/notes.ts @@ -26,10 +26,10 @@ export async function loadNotesCache(): Promise { try { const json = await app.vault.adapter.read(notesCacheFilePath) notesCache = JSON.parse(json) - console.log("Notes cache loaded from the file") + console.log('Notes cache loaded from the file') } catch(e) { - console.trace("Could not load Notes cache from the file") + console.trace('Could not load Notes cache from the file') console.error(e) } } @@ -142,7 +142,7 @@ export function removeAnchors(name: string): string { export async function saveNotesCacheToFile(): Promise { const json = JSON.stringify(notesCache) await app.vault.adapter.write(notesCacheFilePath, json) - console.log("Notes cache saved to the file") + console.log('Notes cache saved to the file') } export function isCacheOutdated(file: TFile): boolean { diff --git a/src/search.ts b/src/search.ts index 8653a1c..5e9acd2 100644 --- a/src/search.ts +++ b/src/search.ts @@ -67,11 +67,11 @@ export async function initGlobalSearchIndex(): Promise { try { const json = await app.vault.adapter.read(searchIndexFilePath) minisearchInstance = MiniSearch.loadJSON(json, options) - console.log("MiniSearch index loaded from the file") + console.log('MiniSearch index loaded from the file') await loadNotesCache() } catch(e) { - console.trace("Could not load MiniSearch index from the file") + console.trace('Could not load MiniSearch index from the file') console.error(e) } } @@ -394,7 +394,7 @@ async function saveIndexToFile(): Promise { if (settings.storeIndexInFile && minisearchInstance && isIndexChanged) { const json = JSON.stringify(minisearchInstance) await app.vault.adapter.write(searchIndexFilePath, json) - console.log("MiniSearch index saved to the file") + console.log('MiniSearch index saved to the file') await saveNotesCacheToFile() isIndexChanged = false From 0c031c6defa7b8312b76b7416477031084a80e7b Mon Sep 17 00:00:00 2001 From: Michael Naumov Date: Mon, 6 Jun 2022 08:19:49 -0600 Subject: [PATCH 4/5] Remove trailing commas --- src/globals.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/globals.ts b/src/globals.ts index b51ad3e..572729b 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -22,8 +22,8 @@ export type SearchNote = { export type IndexedNote = { path: string - basename: string, - mtime: number, + basename: string + mtime: number content: string aliases: string From f0e4eeaf054ddbd4407c24b7f2f93f49d522d8a9 Mon Sep 17 00:00:00 2001 From: Michael Naumov Date: Mon, 6 Jun 2022 11:02:32 -0600 Subject: [PATCH 5/5] Add trailing newline --- src/notes.ts | 2 +- src/search.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/notes.ts b/src/notes.ts index a21bbab..6713730 100644 --- a/src/notes.ts +++ b/src/notes.ts @@ -148,4 +148,4 @@ export async function saveNotesCacheToFile(): Promise { export function isCacheOutdated(file: TFile): boolean { const indexedNote = getNoteFromCache(file.path) return !indexedNote || indexedNote.mtime !== file.stat.mtime -} \ No newline at end of file +} diff --git a/src/search.ts b/src/search.ts index 5e9acd2..8111e94 100644 --- a/src/search.ts +++ b/src/search.ts @@ -399,4 +399,4 @@ async function saveIndexToFile(): Promise { await saveNotesCacheToFile() isIndexChanged = false } -} \ No newline at end of file +}