#14 - non-existing notes are indexed

src/components/GlyphAddNote.svelte

@@ -0,0 +1,2 @@
+<script lang="ts"></script>
+<span class="suggestion-flair" aria-label="Not created yet, select to create"><svg viewBox="0 0 100 100" class="add-note-glyph" width="16" height="16"><path fill="currentColor" stroke="currentColor" d="M23.3,6.7c-3.7,0-6.7,3-6.7,6.7v73.3c0,3.7,3,6.7,6.7,6.7h28.4c-3.2-4.8-5.1-10.5-5.1-16.7c0-16.6,13.4-30,30-30 c2.3,0,4.5,0.3,6.7,0.8V31.7c0-0.9-0.3-1.7-1-2.4L60.7,7.6c-0.6-0.6-1.5-1-2.4-1L23.3,6.7z M56.7,13L77,33.3H60 c-1.8,0-3.3-1.5-3.3-3.3L56.7,13z M76.7,53.3c-12.9,0-23.3,10.4-23.3,23.3S63.8,100,76.7,100S100,89.6,100,76.7 S89.6,53.3,76.7,53.3z M76.7,63.3c1.8,0,3.3,1.5,3.3,3.3v6.7h6.7c1.8,0,3.3,1.5,3.3,3.3c0,1.8-1.5,3.3-3.3,3.3H80v6.7 c0,1.8-1.5,3.3-3.3,3.3c-1.8,0-3.3-1.5-3.3-3.3V80h-6.7c-1.8,0-3.3-1.5-3.3-3.3s1.5-3.3,3.3-3.3h6.7v-6.7 C73.3,64.8,74.8,63.3,76.7,63.3L76.7,63.3z"></path></svg></span>

src/components/ResultItemContainer.svelte

@@ -1,8 +1,9 @@
 <script lang="ts">
-import { createEventDispatcher } from "svelte"
+import GlyphAddNote from "./GlyphAddNote.svelte"
 
 export let id: string
 export let selected = false
+export let glyph = false
 </script>
 
 <div
@@ -13,5 +14,8 @@ export let selected = false
   on:click
   on:auxclick
 >
+  {#if glyph}
+    <GlyphAddNote />
+  {/if}
   <slot />
 </div>

src/components/ResultItemVault.svelte

@@ -1,4 +1,5 @@
 <script lang="ts">
+import { getNoteFromCache } from "src/notes"
 import type { ResultNote } from "../globals"
 import { getMatches } from "../search"
 import { highlighter, makeExcerpt, stringsToRegex } from "../utils"
@@ -10,9 +11,10 @@ export let note: ResultNote
 $: reg = stringsToRegex(note.foundWords)
 $: matches = getMatches(note.content, reg)
 $: cleanedContent = makeExcerpt(note.content, note.matches[0]?.offset ?? -1)
+$: glyph = getNoteFromCache(note.path)?.doesNotExist
 </script>
 
-<ResultItemContainer id={note.path} {selected} on:mousemove on:click>
+<ResultItemContainer id={note.path} {selected} on:mousemove on:click {glyph}>
   <span class="omnisearch-result__title">
     {@html note.basename.replace(reg, highlighter)}
   </span>

src/globals.ts

@@ -28,6 +28,8 @@ export type IndexedNote = {
   headings1: string
   headings2: string
   headings3: string
+
+  doesNotExist?: boolean
 }
 
 export type SearchMatch = {

src/main.ts

@@ -1,9 +1,9 @@
 import { Plugin, TFile } from 'obsidian'
 import {
+  addNonExistingToIndex,
   addToIndex,
   initGlobalSearchIndex,
   removeFromIndex,
-  removeFromIndexByPath,
 } from './search'
 import { OmnisearchInFileModal, OmnisearchVaultModal } from './modals'
 import { loadSettings, SettingsTab } from './settings'
@@ -39,19 +39,21 @@ export default class OmnisearchPlugin extends Plugin {
       )
       this.registerEvent(
         this.app.vault.on('delete', file => {
-          removeFromIndex(file)
+          removeFromIndex(file.path)
+          // Re-index the note as non-existing file
+          addNonExistingToIndex(file.name)
         }),
       )
       this.registerEvent(
         this.app.vault.on('modify', async file => {
-          removeFromIndex(file)
+          removeFromIndex(file.path)
           await addToIndex(file)
         }),
       )
       this.registerEvent(
         this.app.vault.on('rename', async (file, oldPath) => {
           if (file instanceof TFile && file.path.endsWith('.md')) {
-            removeFromIndexByPath(oldPath)
+            removeFromIndex(oldPath)
             await addToIndex(file)
           }
         }),

src/notes.ts

@@ -1,7 +1,27 @@
-import { MarkdownView } from 'obsidian'
+import { MarkdownView, TFile, type CachedMetadata } from 'obsidian'
-import type { ResultNote } from './globals'
+import type { IndexedNote, ResultNote } from './globals'
 import { stringsToRegex } from './utils'
 
+/**
+ * This is an in-memory cache of the notes, with all their computed fields
+ * used by the search engine.
+ * This cache allows us to quickly de-index notes when they are deleted or updated.
+ */
+export let notesCache: Record<string, IndexedNote> = {}
+
+export function resetNotesCache(): void {
+  notesCache = {}
+}
+export function getNoteFromCache(key: string): IndexedNote | undefined {
+  return notesCache[key]
+}
+export function addNoteToCache(key: string, note: IndexedNote): void {
+  notesCache[key] = note
+}
+export function removeNoteFromCache(key: string): void {
+  delete notesCache[key]
+}
+
 export async function openNote(
   item: ResultNote,
   newPane = false,
@@ -40,3 +60,23 @@ export async function createNote(name: string): Promise<void> {
     console.error(e)
   }
 }
+
+/**
+ * For a given file, returns a list of links leading to notes that don't exist
+ * @param file
+ * @param metadata
+ * @returns
+ */
+export function getNonExistingNotes(
+  file: TFile,
+  metadata: CachedMetadata,
+): string[] {
+  return (metadata.links ?? [])
+    .map(l => {
+      const path = l.link.split(/[\^#]+/)[0] // Remove anchors and headings
+      return app.metadataCache.getFirstLinkpathDest(path, file.path)
+        ? ''
+        : l.link
+    })
+    .filter(l => !!l)
+}

src/search.ts

@@ -16,9 +16,15 @@ import {
 } from './utils'
 import type { Query } from './query'
 import { settings } from './settings'
+import {
+  removeNoteFromCache,
+  getNoteFromCache,
+  getNonExistingNotes,
+  resetNotesCache,
+  addNoteToCache,
+} from './notes'
 
 let minisearchInstance: MiniSearch<IndexedNote>
-let indexedNotes: Record<string, IndexedNote> = {}
 
 const tokenize = (text: string): string[] => {
   const tokens = text.split(SPACE_OR_PUNCTUATION)
@@ -37,7 +43,7 @@ const tokenize = (text: string): string[] => {
  * and adds all the notes to the index
  */
 export async function initGlobalSearchIndex(): Promise<void> {
-  indexedNotes = {}
+  resetNotesCache()
   minisearchInstance = new MiniSearch({
     tokenize,
     idField: 'path',
@@ -73,9 +79,6 @@ export async function initGlobalSearchIndex(): Promise<void> {
       }ms`,
     )
   }
-
-  // Listen to the query input to trigger a search
-  // subscribeToQuery()
 }
 
 /**
@@ -115,20 +118,20 @@ async function search(query: Query): Promise<SearchResult[]> {
   const exactTerms = query.getExactTerms()
   if (exactTerms.length) {
     results = results.filter(r => {
-      const title = indexedNotes[r.id]?.path.toLowerCase() ?? ''
+      const title = getNoteFromCache(r.id)?.path.toLowerCase() ?? ''
       const content = stripMarkdownCharacters(
-        indexedNotes[r.id]?.content ?? '',
+        getNoteFromCache(r.id)?.content ?? '',
       ).toLowerCase()
       return exactTerms.every(q => content.includes(q) || title.includes(q))
     })
   }
 
-  // // If the search query contains exclude terms, filter out results that have them
+  // If the search query contains exclude terms, filter out results that have them
   const exclusions = query.exclusions
   if (exclusions.length) {
     results = results.filter(r => {
       const content = stripMarkdownCharacters(
-        indexedNotes[r.id]?.content ?? '',
+        getNoteFromCache(r.id)?.content ?? '',
       ).toLowerCase()
       return exclusions.every(q => !content.includes(q.value))
     })
@@ -145,7 +148,9 @@ async function search(query: Query): Promise<SearchResult[]> {
 export function getMatches(text: string, reg: RegExp): SearchMatch[] {
   let match: RegExpExecArray | null = null
   const matches: SearchMatch[] = []
+  let count = 0 // TODO: FIXME: this is a hack to avoid infinite loops
   while ((match = reg.exec(text)) !== null) {
+    if (++count > 100) break
     const m = match[0]
     if (m) matches.push({ match: m, offset: match.index })
   }
@@ -181,7 +186,7 @@ export async function getSuggestions(
 
   // Map the raw results to get usable suggestions
   const suggestions = results.map(result => {
-    const note = indexedNotes[result.id]
+    const note = getNoteFromCache(result.id)
     if (!note) {
       throw new Error(`Note "${result.id}" not indexed`)
     }
@@ -216,11 +221,27 @@ export async function addToIndex(file: TAbstractFile): Promise<void> {
   if (!(file instanceof TFile) || file.extension !== 'md') {
     return
   }
+
+  // Check if the file was already indexed as non-existent,
+  // and if so, remove it from the index (before adding it again)
+  if (getNoteFromCache(file.path)?.doesNotExist) {
+    removeFromIndex(file.path)
+  }
+
   try {
     // console.log(`Omnisearch - adding ${file.path} to index`)
-    const metadata = app.metadataCache.getFileCache(file)
 
-    if (indexedNotes[file.path]) {
+    // Look for links that lead to non-existing files,
+    // and index them as well
+    const metadata = app.metadataCache.getFileCache(file)
+    if (metadata) {
+      const nonExisting = getNonExistingNotes(file, metadata)
+      for (const name of nonExisting.filter(o => !getNoteFromCache(o))) {
+        addNonExistingToIndex(name)
+      }
+    }
+
+    if (getNoteFromCache(file.path)) {
       throw new Error(`${file.basename} is already indexed`)
     }
 
@@ -245,7 +266,7 @@ export async function addToIndex(file: TAbstractFile): Promise<void> {
     }
 
     minisearchInstance.add(note)
-    indexedNotes[note.path] = note
+    addNoteToCache(note.path, note)
   }
   catch (e) {
     console.trace('Error while indexing ' + file.basename)
@@ -254,25 +275,42 @@ export async function addToIndex(file: TAbstractFile): Promise<void> {
 }
 
 /**
- * Removes a file from the index
+ * Index a non-existing note.
- * @param file
+ * Useful to find internal links that lead (yet) to nowhere
- * @returns
+ * @param name
  */
-export function removeFromIndex(file: TAbstractFile): void {
+export function addNonExistingToIndex(name: string): void {
-  if (file instanceof TFile && file.path.endsWith('.md')) {
+  const filename = name + (name.endsWith('.md') ? '' : '.md')
-    // console.log(`Omnisearch - removing ${file.path} from index`)
+  const note = {
-    return removeFromIndexByPath(file.path)
+    path: filename,
-  }
+    basename: name,
+    content: '',
+    aliases: '',
+    headings1: '',
+    headings2: '',
+    headings3: '',
+    doesNotExist: true,
+  } as IndexedNote
+  minisearchInstance.add(note)
+  addNoteToCache(filename, note)
 }
 
 /**
  * Removes a file from the index, by its path
  * @param path
  */
-export function removeFromIndexByPath(path: string): void {
+export function removeFromIndex(path: string): void {
-  const note = indexedNotes[path]
+  if (!path.endsWith('.md')) {
+    console.info(`"${path}" is not a .md file`)
+    return
+  }
+  const note = getNoteFromCache(path)
   if (note) {
+    // Delete the original
     minisearchInstance.remove(note)
-    delete indexedNotes[path]
+    removeNoteFromCache(path)
+  }
+  else {
+    console.warn(`not not found under path ${path}`)
   }
 }

src/vendor/parse-query.ts

@@ -75,7 +75,9 @@ export function parseQuery(
   const regex =
     /(\S+:'(?:[^'\\]|\\.)*')|(\S+:"(?:[^"\\]|\\.)*")|(-?"(?:[^"\\]|\\.)*")|(-?'(?:[^'\\]|\\.)*')|\S+|\S+:\S+/g
   let match
+  let count = 0 // TODO: FIXME: this is a hack to avoid infinite loops
   while ((match = regex.exec(string)) !== null) {
+    if (++count > 100) break
     let term = match[0]
     const sepIndex = term.indexOf(':')