Save/load index to files, improve logging

src/globals.ts

@@ -22,7 +22,9 @@ export type SearchNote = {
 
 export type IndexedNote = {
   path: string
-  basename: string
+  basename: string,
+  mtime: number,
+
   content: string
   aliases: string
   headings1: string

src/notes.ts

@@ -14,9 +14,29 @@ import { stringsToRegex } from './utils'
  */
 export let notesCache: Record<string, IndexedNote> = {}
 
+const notesCacheFilePath = `${app.vault.configDir}/plugins/omnisearch/notesCache.json`
+
 export function resetNotesCache(): void {
   notesCache = {}
 }
+
+export async function loadNotesCache(): Promise<void> {
+  if (await app.vault.adapter.exists(notesCacheFilePath)) {
+    try {
+      const json = await app.vault.adapter.read(notesCacheFilePath)
+      notesCache = JSON.parse(json)
+      console.log("Notes cache loaded from the file")
+    }
+    catch(e) {
+      console.trace("Could not load Notes cache from the file")
+      console.error(e)
+    } 
+  }
+
+  if (!notesCache) {
+    notesCache = {}
+  }
+}
 export function getNoteFromCache(key: string): IndexedNote | undefined {
   return notesCache[key]
 }
@@ -117,3 +137,14 @@ export function getNonExistingNotes(
 export function removeAnchors(name: string): string {
   return name.split(/[\^#]+/)[0]
 }
+
+export async function saveNotesCacheToFile(): Promise<void> {
+  const json = JSON.stringify(notesCache)
+  await app.vault.adapter.write(notesCacheFilePath, json)
+  console.log("Notes cache saved to the file")
+}
+
+export function isCacheOutdated(file: TFile): boolean {
+  const indexedNote = getNoteFromCache(file.path)
+  return !indexedNote || indexedNote.mtime !== file.stat.mtime
+}

src/search.ts

@@ -24,9 +24,14 @@ import {
   addNoteToCache,
   removeAnchors,
   getNonExistingNotesFromCache,
+  loadNotesCache,
+  saveNotesCacheToFile,
+  isCacheOutdated
 } from './notes'
 
 let minisearchInstance: MiniSearch<IndexedNote>
+let isIndexChanged: boolean
+const searchIndexFilePath = `${app.vault.configDir}/plugins/omnisearch/searchIndex.json`
 
 const tokenize = (text: string): string[] => {
   const tokens = text.split(SPACE_OR_PUNCTUATION)
@@ -45,8 +50,7 @@ const tokenize = (text: string): string[] => {
  * and adds all the notes to the index
  */
 export async function initGlobalSearchIndex(): Promise<void> {
-  resetNotesCache()
+  const options = {
-  minisearchInstance = new MiniSearch({
     tokenize,
     idField: 'path',
     fields: [
@@ -57,29 +61,58 @@ export async function initGlobalSearchIndex(): Promise<void> {
       'headings2',
       'headings3',
     ],
-  })
+  }
+
+  if (await app.vault.adapter.exists(searchIndexFilePath)) {
+    try {
+      const json = await app.vault.adapter.read(searchIndexFilePath)
+      minisearchInstance = MiniSearch.loadJSON(json, options)
+      console.log("MiniSearch index loaded from the file")
+    }
+    catch(e) {
+      console.trace("Could not load MiniSearch index from the file")
+      console.error(e)
+    }
+  }
+  
+  if (!minisearchInstance) {
+    minisearchInstance = new MiniSearch(options)
+    resetNotesCache()
+  }
+
+  await loadNotesCache()
 
   // Index files that are already present
   const start = new Date().getTime()
-  const files = app.vault.getMarkdownFiles()
+  const files = app.vault.getMarkdownFiles().filter(file => isCacheOutdated(file))
 
   // This is basically the same behavior as MiniSearch's `addAllAsync()`.
   // We index files by batches of 10
-  if (files.length) {
+  console.log('Omnisearch - indexing ' + files.length + ' modified notes')
-    console.log('Omnisearch - indexing ' + files.length + ' files')
+
-  }
   for (let i = 0; i < files.length; ++i) {
     if (i % 10 === 0) await wait(0)
     const file = files[i]
-    if (file) await addToIndex(file)
+    if (file) {
+      if (getNoteFromCache(file.path)) {
+        removeFromIndex(file.path)
+      }
+      await addToIndex(file)
+    }
   }
 
-  if (files.length > 0 && settings.showIndexingNotices) {
+  if (files.length > 0) {
-    new Notice(
+    const message = `Omnisearch - Indexed ${files.length} modified notes in ${
-      `Omnisearch - Indexed ${files.length} notes in ${
       new Date().getTime() - start
-      }ms`,
+    }ms`
-    )
+
+    console.log(message)
+
+    if (settings.showIndexingNotices) {
+      new Notice(message)
+    }
+
+    await saveIndexToFile()
   }
 }
 
@@ -255,6 +288,8 @@ export async function addToIndex(file: TAbstractFile): Promise<void> {
       basename: file.basename,
       content,
       path: file.path,
+      mtime: file.stat.mtime,
+
       aliases: getAliasesFromMetadata(metadata).join(''),
       headings1: metadata
         ? extractHeadingsFromCache(metadata, 1).join(' ')
@@ -268,6 +303,7 @@ export async function addToIndex(file: TAbstractFile): Promise<void> {
     }
 
     minisearchInstance.add(note)
+    isIndexChanged = true
     addNoteToCache(note.path, note)
   }
   catch (e) {
@@ -289,6 +325,8 @@ export function addNonExistingToIndex(name: string, parent: string): void {
   const note = {
     path: filename,
     basename: name,
+    mtime: 0,
+
     content: '',
     aliases: '',
     headings1: '',
@@ -299,6 +337,7 @@ export function addNonExistingToIndex(name: string, parent: string): void {
     parent,
   } as IndexedNote
   minisearchInstance.add(note)
+  isIndexChanged = true
   addNoteToCache(filename, note)
 }
 
@@ -314,6 +353,7 @@ export function removeFromIndex(path: string): void {
   const note = getNoteFromCache(path)
   if (note) {
     minisearchInstance.remove(note)
+    isIndexChanged = true
     removeNoteFromCache(path)
     getNonExistingNotesFromCache()
       .filter(n => n.parent === path)
@@ -336,4 +376,17 @@ export async function reindexNotes(): Promise<void> {
     await addToIndex(note)
   }
   notesToReindex.clear()
+
+  await saveIndexToFile()
+}
+
+async function saveIndexToFile(): Promise<void> {
+  if (minisearchInstance && isIndexChanged) {
+    const json = JSON.stringify(minisearchInstance)
+    await app.vault.adapter.write(searchIndexFilePath, json)
+    console.log("MiniSearch index saved to the file")
+
+    await saveNotesCacheToFile()
+    isIndexChanged = false
+  }
 }