tanner/obsidian-tannersearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Minisearch 6.0 mostly ok

Browse Source
This commit is contained in:
Simon Cambier
2022-11-25 22:40:59 +01:00
parent dcef2d3719
commit e3ac5a4bac
16 changed files with 196 additions and 1058 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
src/cache-manager.ts
View File

@@ -1,16 +1,11 @@
import { Notice, type TFile } from 'obsidian'
import { Notice } from 'obsidian'
import type { IndexedDocument } from './globals'
import { database } from './database'
import MiniSearch from 'minisearch'
import { minisearchOptions } from './search/search-engine'
import type { AsPlainObject } from 'minisearch'
import type MiniSearch from 'minisearch'
import { makeMD5 } from './tools/utils'
class CacheManager {
/**
* @deprecated
* @private
*/
private liveDocuments: Map<string, IndexedDocument> = new Map()
/**
* Show an empty input field next time the user opens Omnisearch modal
*/
@@ -40,36 +35,6 @@ class CacheManager {
return data
}
/**
* Important: keep this method async for the day it _really_ becomes async.
* This will avoid a refactor.
* @deprecated
* @param path
* @param note
*/
public async updateLiveDocument(
path: string,
note: IndexedDocument
): Promise<void> {
this.liveDocuments.set(path, note)
}
/**
* @deprecated
* @param key
*/
public deleteLiveDocument(key: string): void {
this.liveDocuments.delete(key)
}
/**
* @deprecated
* @param key
*/
public getLiveDocument(key: string): IndexedDocument | undefined {
return this.liveDocuments.get(key)
}
//#region Minisearch
public getDocumentsChecksum(documents: IndexedDocument[]): string {
@@ -87,20 +52,13 @@ class CacheManager {
)
}
public async getMinisearchCache(): Promise<MiniSearch | null> {
// Retrieve documents and make their checksum
const cachedDocs = await database.documents.toArray()
// Add those documents in the live cache
cachedDocs.forEach(doc =>
cacheManager.updateLiveDocument(doc.path, doc.document)
)
// Retrieve the search cache, and verify the checksum
const cachedIndex = (await database.minisearch.toArray())[0]
public async getMinisearchCache(): Promise<{
paths: { path: string; mtime: number }[]
data: AsPlainObject
} | null> {
try {
return MiniSearch.loadJS(cachedIndex.data, minisearchOptions)
const cachedIndex = (await database.minisearch.toArray())[0]
return cachedIndex
} catch (e) {
new Notice(
'Omnisearch - Cache missing or invalid. Some freezes may occur while Omnisearch indexes your vault.'
@@ -111,75 +69,15 @@ class CacheManager {
}
}
/**
* Get a dict listing the deleted/added documents since last cache
* @param documents
*/
public async getDiffDocuments(documents: IndexedDocument[]): Promise<{
toDelete: string[]
toAdd: IndexedDocument[]
toUpdate: { oldDoc: IndexedDocument; newDoc: IndexedDocument }[]
}> {
let cachedDocs = await database.documents.toArray()
// present in `documents` but not in `cachedDocs`
const toAdd = documents.filter(
d => !cachedDocs.find(c => c.path === d.path)
)
// present in `cachedDocs` but not in `documents`
const toDelete = cachedDocs
.filter(c => !documents.find(d => d.path === c.path))
.map(d => d.path)
// toUpdate: same path, but different mtime
const toUpdate = cachedDocs
.filter(({ mtime: cMtime, path: cPath }) =>
documents.some(
({ mtime: dMtime, path: dPath }) =>
cPath === dPath && dMtime !== cMtime
)
)
.map(c => ({
oldDoc: c.document,
newDoc: documents.find(d => d.path === c.path)!,
}))
return {
toAdd,
toDelete,
toUpdate,
}
}
public async writeMinisearchCache(
minisearch: MiniSearch,
documents: IndexedDocument[]
indexed: Map<string, number>
): Promise<void> {
const { toDelete, toAdd, toUpdate } = await this.getDiffDocuments(documents)
// Delete
// console.log(`Omnisearch - Cache - Will delete ${toDelete.length} documents`)
await database.documents.bulkDelete(toDelete)
// Add
// console.log(`Omnisearch - Cache - Will add ${toAdd.length} documents`)
await database.documents.bulkAdd(
toAdd.map(o => ({ document: o, mtime: o.mtime, path: o.path }))
)
// Update
// console.log(`Omnisearch - Cache - Will update ${toUpdate.length} documents`)
await database.documents.bulkPut(
toUpdate.map(o => ({
document: o.newDoc,
mtime: o.newDoc.mtime,
path: o.newDoc.path,
}))
)
const paths = Array.from(indexed).map(([k, v]) => ({ path: k, mtime: v }))
await database.minisearch.clear()
await database.minisearch.add({
date: new Date().toISOString(),
checksum: this.getDocumentsChecksum(documents),
paths,
data: minisearch.toJSON(),
})
console.log('Omnisearch - Search cache written')

4
src/components/ModalInFile.svelte
View File

@@ -9,7 +9,6 @@
import { loopIndex } from 'src/tools/utils'
import { onDestroy, onMount, tick } from 'svelte'
import { MarkdownView } from 'obsidian'
import { SearchEngine } from 'src/search/search-engine'
import ModalContainer from './ModalContainer.svelte'
import {
OmnisearchInFileModal,
@@ -18,6 +17,7 @@
import ResultItemInFile from './ResultItemInFile.svelte'
import { Query } from 'src/search/query'
import { openNote } from 'src/tools/notes'
import { searchEngine } from 'src/search/omnisearch'
export let modal: OmnisearchInFileModal
export let parent: OmnisearchVaultModal | null = null
@@ -50,7 +50,7 @@
query = new Query(searchQuery)
note =
(
await SearchEngine.getEngine().getSuggestions(query, {
await searchEngine.getSuggestions(query, {
singleFilePath,
})
)[0] ?? null

19
src/components/ModalVault.svelte
View File

@@ -3,9 +3,8 @@
import { onDestroy, onMount, tick } from 'svelte'
import InputSearch from './InputSearch.svelte'
import ModalContainer from './ModalContainer.svelte'
import { eventBus, IndexingStep, type ResultNote } from 'src/globals'
import { eventBus, indexingStep, IndexingStepType, type ResultNote } from 'src/globals'
import { createNote, openNote } from 'src/tools/notes'
import { SearchEngine } from 'src/search/search-engine'
import { getCtrlKeyLabel, getExtension, isFilePDF, loopIndex } from 'src/tools/utils'
import {
OmnisearchInFileModal,
@@ -16,6 +15,7 @@
import { settings } from '../settings'
import * as NotesIndex from '../notes-index'
import { cacheManager } from '../cache-manager'
import { searchEngine } from 'src/search/omnisearch'
export let modal: OmnisearchVaultModal
export let previousQuery: string | undefined
@@ -24,7 +24,6 @@
let searchQuery: string | undefined
let resultNotes: ResultNote[] = []
let query: Query
let { indexingStep } = SearchEngine
let indexingStepDesc = ''
$: selectedNote = resultNotes[selectedIndex]
@@ -36,20 +35,20 @@
}
$: {
switch ($indexingStep) {
case IndexingStep.LoadingCache:
case IndexingStepType.LoadingCache:
indexingStepDesc = 'Loading cache...'
break
case IndexingStep.ReadingNotes:
case IndexingStepType.ReadingNotes:
updateResults()
indexingStepDesc = 'Reading notes...'
break
case IndexingStep.ReadingPDFs:
case IndexingStepType.ReadingPDFs:
indexingStepDesc = 'Reading PDFs...'
break
case IndexingStep.ReadingImages:
case IndexingStepType.ReadingImages:
indexingStepDesc = 'Reading images...'
break
case IndexingStep.UpdatingCache:
case IndexingStepType.UpdatingCache:
indexingStepDesc = 'Updating cache...'
break
default:
@@ -99,7 +98,7 @@
async function updateResults() {
query = new Query(searchQuery)
resultNotes = (await SearchEngine.getEngine().getSuggestions(query)).sort(
resultNotes = (await searchEngine.getSuggestions(query)).sort(
(a, b) => b.score - a.score
)
selectedIndex = 0
@@ -139,7 +138,7 @@
openNote(note, newPane)
}
async function onClickCreateNote(e: MouseEvent) {
async function onClickCreateNote(_e: MouseEvent) {
await createNoteAndCloseModal()
}

3
src/components/ResultItemVault.svelte
View File

@@ -1,5 +1,4 @@
<script lang="ts">
import { cacheManager } from 'src/cache-manager'
import { settings, showExcerpt } from 'src/settings'
import type { ResultNote } from '../globals'
import {
@@ -28,7 +27,7 @@
}
$: reg = stringsToRegex(note.foundWords)
$: cleanedContent = makeExcerpt(note.content, note.matches[0]?.offset ?? -1)
$: glyph = cacheManager.getLiveDocument(note.path)?.doesNotExist
$: glyph = false //cacheManager.getLiveDocument(note.path)?.doesNotExist
$: title = settings.showShortName ? note.basename : note.path
</script>

6
src/components/modals.ts
View File

@@ -35,7 +35,7 @@ abstract class OmnisearchModal extends Modal {
{ k: 'K', dir: 'up' },
] as const) {
for (const modifier of ['Ctrl', 'Mod'] as const) {
this.scope.register([modifier], key.k, e => {
this.scope.register([modifier], key.k, _e => {
if (this.app.vault.getConfig('vimMode')) {
// e.preventDefault()
eventBus.emit('arrow-' + key.dir)
@@ -50,7 +50,7 @@ abstract class OmnisearchModal extends Modal {
{ k: 'P', dir: 'up' },
] as const) {
for (const modifier of ['Ctrl', 'Mod'] as const) {
this.scope.register([modifier], key.k, e => {
this.scope.register([modifier], key.k, _e => {
if (this.app.vault.getConfig('vimMode')) {
// e.preventDefault()
eventBus.emit('arrow-' + key.dir)
@@ -108,7 +108,7 @@ abstract class OmnisearchModal extends Modal {
})
// Context
this.scope.register(['Ctrl'], 'H', e => {
this.scope.register(['Ctrl'], 'H', _e => {
eventBus.emit(EventNames.ToggleExcerpts)
})
}

77
src/database.ts
View File

@@ -3,11 +3,47 @@ import type { AsPlainObject } from 'minisearch'
import type { IndexedDocument } from './globals'
export class OmnisearchCache extends Dexie {
public static readonly dbVersion = 7
public static readonly dbVersion = 8
public static readonly dbName = 'omnisearch/cache/' + app.appId
private static instance: OmnisearchCache
//#region Table declarations
/**
* @deprecated
*/
documents!: Dexie.Table<
{
path: string
mtime: number
document: IndexedDocument
},
string
>
searchHistory!: Dexie.Table<{ id?: number; query: string }, number>
minisearch!: Dexie.Table<
{
date: string
paths: Array<{ path: string; mtime: number }>
data: AsPlainObject
},
string
>
private constructor() {
super(OmnisearchCache.dbName)
// Database structure
this.version(OmnisearchCache.dbVersion).stores({
searchHistory: '++id',
documents: 'path',
minisearch: 'date',
})
}
//#endregion Table declarations
/**
* Deletes Omnisearch databases that have an older version than the current one
*/
@@ -29,34 +65,6 @@ export class OmnisearchCache extends Dexie {
}
}
//#region Table declarations
documents!: Dexie.Table<
{
path: string
mtime: number
/**
* @deprecated
*/
document: IndexedDocument
},
string
>
searchHistory!: Dexie.Table<{ id?: number; query: string }, number>
minisearch!: Dexie.Table<
{
date: string
/**
* @deprecated
*/
checksum: string
data: AsPlainObject
},
string
>
//#endregion Table declarations
public static getInstance() {
if (!OmnisearchCache.instance) {
OmnisearchCache.instance = new OmnisearchCache()
@@ -64,19 +72,8 @@ export class OmnisearchCache extends Dexie {
return OmnisearchCache.instance
}
private constructor() {
super(OmnisearchCache.dbName)
// Database structure
this.version(OmnisearchCache.dbVersion).stores({
searchHistory: '++id',
documents: 'path',
minisearch: 'date',
})
}
public async clearCache() {
await this.minisearch.clear()
await this.documents.clear()
}
}

4
src/file-loader.ts
View File

@@ -1,4 +1,3 @@
import { cacheManager } from './cache-manager'
import {
extractHeadingsFromCache,
getAliasesFromMetadata,
@@ -47,9 +46,8 @@ async function getBinaryFiles(files: TFile[]): Promise<IndexedDocument[]> {
const input = []
for (const file of files) {
input.push(
new Promise(async (resolve, reject) => {
new Promise(async (resolve, _reject) => {
const doc = await getIndexedDocument(file.path)
// await cacheManager.updateLiveDocument(file.path, doc)
data.push(doc)
return resolve(null)
})

5
src/globals.ts
View File

@@ -1,4 +1,5 @@
import { EventBus } from './tools/event-bus'
import { writable } from 'svelte/store'
export const regexLineSplit = /\r?\n|\r|((\.|\?|!)( |\r?\n|\r))/g
export const regexYaml = /^---\s*\n(.*?)\n?^---\s?/ms
@@ -16,7 +17,7 @@ export const EventNames = {
ToggleExcerpts: 'toggle-excerpts',
} as const
export const enum IndexingStep {
export const enum IndexingStepType {
Done,
LoadingCache,
ReadingNotes,
@@ -50,6 +51,8 @@ export const isSearchMatch = (o: { offset?: number }): o is SearchMatch => {
return o.offset !== undefined
}
export const indexingStep = writable(IndexingStepType.LoadingCache)
export type ResultNote = {
score: number
path: string

119
src/main.ts
View File

@@ -1,18 +1,15 @@
import { Notice, Platform, Plugin, TFile } from 'obsidian'
import { SearchEngine } from './search/search-engine'
import {
OmnisearchInFileModal,
OmnisearchVaultModal,
} from './components/modals'
import { loadSettings, settings, SettingsTab, showExcerpt } from './settings'
import { eventBus, EventNames, IndexingStep } from './globals'
import { eventBus, EventNames, indexingStep, IndexingStepType } from './globals'
import api from './tools/api'
import { isFilePlaintext, wait } from './tools/utils'
import * as FileLoader from './file-loader'
import { isFileImage, isFilePDF, isFilePlaintext } from './tools/utils'
import { OmnisearchCache } from './database'
import { cacheManager } from './cache-manager'
import * as NotesIndex from './notes-index'
import { addToIndexAndMemCache } from "./notes-index";
import { searchEngine } from './search/omnisearch'
export default class OmnisearchPlugin extends Plugin {
private ribbonButton?: HTMLElement
@@ -56,12 +53,12 @@ export default class OmnisearchPlugin extends Plugin {
// Listeners to keep the search index up-to-date
this.registerEvent(
this.app.vault.on('create', file => {
NotesIndex.addToIndexAndMemCache(file)
searchEngine.addFromPaths([file.path])
})
)
this.registerEvent(
this.app.vault.on('delete', file => {
NotesIndex.removeFromIndex(file.path)
searchEngine.removeFromPaths([file.path])
})
)
this.registerEvent(
@@ -72,8 +69,8 @@ export default class OmnisearchPlugin extends Plugin {
this.registerEvent(
this.app.vault.on('rename', async (file, oldPath) => {
if (file instanceof TFile && isFilePlaintext(file.path)) {
NotesIndex.removeFromIndex(oldPath)
await NotesIndex.addToIndexAndMemCache(file)
searchEngine.removeFromPaths([oldPath])
await searchEngine.addFromPaths([file.path])
}
})
)
@@ -108,105 +105,57 @@ export default class OmnisearchPlugin extends Plugin {
async function populateIndex(): Promise<void> {
console.time('Omnisearch - Indexing total time')
// Initialize minisearch
let engine = SearchEngine.getEngine()
// if not iOS, load data from cache
if (!Platform.isIosApp) {
engine = await SearchEngine.initFromCache()
}
// // if not iOS, load data from cache
// if (!Platform.isIosApp) {
// engine = await SearchEngine.initFromCache()
// }
// Load plaintext files
SearchEngine.indexingStep.set(IndexingStep.ReadingNotes)
indexingStep.set(IndexingStepType.ReadingNotes)
console.log('Omnisearch - Reading notes')
const plainTextFiles = await FileLoader.getPlainTextFiles()
let allFiles = [...plainTextFiles]
// iOS: since there's no cache, directly index the documents
if (Platform.isIosApp) {
await wait(1000)
await engine.addAllToMinisearch(plainTextFiles)
}
const plainTextFiles = app.vault
.getFiles()
.filter(f => isFilePlaintext(f.path))
.map(p => p.path)
await searchEngine.addFromPaths(plainTextFiles)
let allFiles: string[] = [...plainTextFiles]
// Load PDFs
if (settings.PDFIndexing) {
SearchEngine.indexingStep.set(IndexingStep.ReadingPDFs)
indexingStep.set(IndexingStepType.ReadingPDFs)
console.log('Omnisearch - Reading PDFs')
const pdfDocuments = await FileLoader.getPDFAsDocuments()
// iOS: since there's no cache, just index the documents
if (Platform.isIosApp) {
await wait(1000)
await engine.addAllToMinisearch(pdfDocuments)
}
const pdfDocuments = app.vault
.getFiles()
.filter(f => isFilePDF(f.path))
.map(p => p.path)
await searchEngine.addFromPaths(pdfDocuments)
// Add PDFs to the files list
allFiles = [...allFiles, ...pdfDocuments]
}
// Load Images
if (settings.imagesIndexing) {
SearchEngine.indexingStep.set(IndexingStep.ReadingImages)
indexingStep.set(IndexingStepType.ReadingImages)
console.log('Omnisearch - Reading Images')
const imagesDocuments = await FileLoader.getImagesAsDocuments()
// iOS: since there's no cache, just index the documents
if (Platform.isIosApp) {
await wait(1000)
await engine.addAllToMinisearch(imagesDocuments)
}
const imagesDocuments = app.vault
.getFiles()
.filter(f => isFileImage(f.path))
.map(p => p.path)
await searchEngine.addFromPaths(imagesDocuments)
// Add Images to the files list
allFiles = [...allFiles, ...imagesDocuments]
}
console.log('Omnisearch - Total number of files: ' + allFiles.length)
let needToUpdateCache = false
// Other platforms: make a diff of what's to add/update/delete
if (!Platform.isIosApp) {
SearchEngine.indexingStep.set(IndexingStep.UpdatingCache)
console.log('Omnisearch - Checking index cache diff...')
// Check which documents need to be removed/added/updated
const diffDocs = await cacheManager.getDiffDocuments(allFiles)
console.log(
`Omnisearch - Files to add/remove/update: ${diffDocs.toAdd.length}/${diffDocs.toDelete.length}/${diffDocs.toUpdate.length}`
)
if (
diffDocs.toAdd.length +
diffDocs.toDelete.length +
diffDocs.toUpdate.length >
100
) {
new Notice(
`Omnisearch - A great number of files need to be added/updated/cleaned. This process may make cause slowdowns.`
)
}
needToUpdateCache = !!(
diffDocs.toAdd.length ||
diffDocs.toDelete.length ||
diffDocs.toUpdate.length
)
// Add
await engine.addAllToMinisearch(diffDocs.toAdd)
// Delete
for (const pathToDel of diffDocs.toDelete) {
NotesIndex.removeFromIndex(pathToDel)
}
// Update (delete + add)
diffDocs.toUpdate.forEach(({ oldDoc, newDoc }) => {
NotesIndex.removeFromIndex(oldDoc.path)
})
await engine.addAllToMinisearch(diffDocs.toUpdate.map(d => d.newDoc))
}
// Load PDFs into the main search engine, and write cache
// SearchEngine.loadTmpDataIntoMain()
SearchEngine.indexingStep.set(IndexingStep.Done)
indexingStep.set(IndexingStepType.Done)
if (!Platform.isIosApp && needToUpdateCache) {
if (!Platform.isIosApp) {
console.log('Omnisearch - Writing cache...')
await SearchEngine.getEngine().writeToCache(allFiles)
await searchEngine.writeToCache()
}
console.timeEnd('Omnisearch - Indexing total time')

74
src/notes-index.ts
View File

@@ -1,38 +1,7 @@
import { Notice, TAbstractFile, TFile } from 'obsidian'
import { isFileIndexable, wait } from './tools/utils'
import type { TAbstractFile } from 'obsidian'
import { removeAnchors } from './tools/notes'
import { SearchEngine } from './search/search-engine'
import { cacheManager } from './cache-manager'
import type { IndexedDocument } from './globals'
import { getIndexedDocument } from "./file-loader";
const indexedList: Set<string> = new Set()
/**
* Adds a file to the search index
* @param file
* @returns
*/
export async function addToIndexAndMemCache(
file: TAbstractFile
): Promise<void> {
if (!(file instanceof TFile) || !isFileIndexable(file.path)) {
return
}
try {
if (indexedList.has(file.path)) {
throw new Error(`${file.basename} is already indexed`)
}
// Make the document and index it
SearchEngine.getEngine().addSingleToMinisearch(file.path)
indexedList.add(file.path)
} catch (e) {
// console.trace('Error while indexing ' + file.basename)
console.error(e)
}
}
import { searchEngine } from './search/omnisearch'
/**
* Index a non-existing note.
@@ -43,7 +12,6 @@ export async function addToIndexAndMemCache(
export function addNonExistingToIndex(name: string, parent: string): void {
name = removeAnchors(name)
const filename = name + (name.endsWith('.md') ? '' : '.md')
if (cacheManager.getLiveDocument(filename)) return
const note: IndexedDocument = {
path: filename,
@@ -60,30 +28,7 @@ export function addNonExistingToIndex(name: string, parent: string): void {
doesNotExist: true,
parent,
}
SearchEngine.getEngine().addSingleToMinisearch(note.path)
}
/**
* Removes a file from the index, by its path.
*/
export function removeFromIndex(path: string): void {
if (!isFileIndexable(path)) {
console.info(`"${path}" is not an indexable file`)
return
}
if (indexedList.has(path)) {
SearchEngine.getEngine().removeFromMinisearch(path)
// FIXME: only remove non-existing notes if they don't have another parent
// cacheManager
// .getNonExistingNotesFromMemCache()
// .filter(n => n.parent === path)
// .forEach(n => {
// removeFromIndex(n.path)
// })
} else {
console.warn(`Omnisearch - Note not found under path ${path}`)
}
// searchEngine.addDocuments([note])
}
const notesToReindex = new Set<TAbstractFile>()
@@ -97,13 +42,8 @@ export function markNoteForReindex(note: TAbstractFile): void {
}
export async function refreshIndex(): Promise<void> {
if (notesToReindex.size > 0) {
console.info(`Omnisearch - Reindexing ${notesToReindex.size} notes`)
for (const note of notesToReindex) {
removeFromIndex(note.path)
await addToIndexAndMemCache(note)
await wait(0)
}
notesToReindex.clear()
}
const paths = [...notesToReindex].map(n => n.path)
searchEngine.removeFromPaths(paths)
searchEngine.addFromPaths(paths)
notesToReindex.clear()
}

179
src/search/search-engine.ts → src/search/omnisearch.ts
View File

@@ -1,25 +1,20 @@
import MiniSearch, { type Options, type SearchResult } from 'minisearch'
import {
chsRegex,
type IndexedDocument,
IndexingStep,
type ResultNote,
type SearchMatch,
SPACE_OR_PUNCTUATION,
} from '../globals'
import MiniSearch, {
type AsPlainObject,
type Options,
type SearchResult,
} from 'minisearch'
import type { IndexedDocument, ResultNote, SearchMatch } from '../globals'
import { chsRegex, SPACE_OR_PUNCTUATION } from '../globals'
import { settings } from '../settings'
import {
removeDiacritics,
stringsToRegex,
stripMarkdownCharacters,
} from '../tools/utils'
import type { Query } from './query'
import { settings } from '../settings'
import { cacheManager } from '../cache-manager'
import { writable } from 'svelte/store'
import { Notice } from 'obsidian'
import { getIndexedDocument } from '../file-loader'
let previousResults: SearchResult[] = []
import type { Query } from './query'
import { cacheManager } from '../cache-manager'
const tokenize = (text: string): string[] => {
const tokens = text.split(SPACE_OR_PUNCTUATION)
@@ -32,67 +27,70 @@ const tokenize = (text: string): string[] => {
} else return tokens
}
export const minisearchOptions: Options<IndexedDocument> = {
tokenize,
processTerm: (term: string) =>
(settings.ignoreDiacritics ? removeDiacritics(term) : term).toLowerCase(),
idField: 'path',
fields: [
'basename',
'aliases',
'content',
'headings1',
'headings2',
'headings3',
],
storeFields: ['tags'],
logger(level, message, code) {
if (code === 'version_conflict') {
new Notice(
'Omnisearch - Your index cache may be incorrect or corrupted. If this message keeps appearing, go to Settings to clear the cache.'
)
}
},
}
export class SearchEngine {
private static engine?: SearchEngine
public static indexingStep = writable(IndexingStep.LoadingCache)
/**
* The main singleton SearchEngine instance.
* Should be used for all queries
*/
public static getEngine(): SearchEngine {
if (!this.engine) {
this.engine = new SearchEngine()
}
return this.engine
}
/**
* Instantiates the main instance with cache data (if it exists)
*/
public static async initFromCache(): Promise<SearchEngine> {
try {
const cache = await cacheManager.getMinisearchCache()
if (cache) {
this.getEngine().minisearch = cache
export class Omnisearch {
public static readonly options: Options<IndexedDocument> = {
tokenize,
processTerm: (term: string) =>
(settings.ignoreDiacritics ? removeDiacritics(term) : term).toLowerCase(),
idField: 'path',
fields: [
'basename',
'aliases',
'content',
'headings1',
'headings2',
'headings3',
],
storeFields: ['tags'],
logger(_level, _message, code) {
if (code === 'version_conflict') {
new Notice(
'Omnisearch - Your index cache may be incorrect or corrupted. If this message keeps appearing, go to Settings to clear the cache.'
)
}
} catch (e) {
new Notice(
'Omnisearch - Cache missing or invalid. Some freezes may occur while Omnisearch indexes your vault.'
)
console.error('Omnisearch - Could not init engine from cache')
console.error(e)
}
return this.getEngine()
},
}
private minisearch: MiniSearch
private indexedDocuments: Map<string, number> = new Map()
private previousResults: SearchResult[] = []
constructor() {
this.minisearch = new MiniSearch(Omnisearch.options)
}
private minisearch: MiniSearch
async loadCache(): Promise<void> {
const cache = await cacheManager.getMinisearchCache()
if (cache) {
this.minisearch = MiniSearch.loadJS(cache.data, Omnisearch.options)
this.indexedDocuments = new Map(cache.paths.map(o => [o.path, o.mtime]))
}
}
private constructor() {
this.minisearch = new MiniSearch(minisearchOptions)
/**
* Add notes/PDFs/images to the search index
* @param paths
*/
public async addFromPaths(paths: string[]): Promise<void> {
let documents = await Promise.all(
paths.map(async path => await getIndexedDocument(path))
)
// If a document is already added, discard it
this.removeFromPaths(
documents.filter(d => this.indexedDocuments.has(d.path)).map(d => d.path)
)
documents.forEach(doc => this.indexedDocuments.set(doc.path, doc.mtime))
await this.minisearch.addAllAsync(documents)
}
/**
* Discard a document from minisearch
* @param paths
*/
public removeFromPaths(paths: string[]): void {
paths.forEach(p => this.indexedDocuments.delete(p))
this.minisearch.discardAll(paths)
}
/**
@@ -104,7 +102,7 @@ export class SearchEngine {
options: { prefixLength: number }
): Promise<SearchResult[]> {
if (query.isEmpty()) {
previousResults = []
this.previousResults = []
return []
}
@@ -120,7 +118,7 @@ export class SearchEngine {
headings3: settings.weightH3,
},
})
if (!results.length) return previousResults
if (!results.length) return this.previousResults
// Downrank files that are in Obsidian's excluded list
if (settings.respectExcluded) {
@@ -169,14 +167,11 @@ export class SearchEngine {
)
.slice(0, 50)
previousResults = results
this.previousResults = results
return results
}
/**
* Parses a text against a regex, and returns the { string, offset } matches
*/
public getMatches(text: string, reg: RegExp, query: Query): SearchMatch[] {
let match: RegExpExecArray | null = null
const matches: SearchMatch[] = []
@@ -296,26 +291,12 @@ export class SearchEngine {
return resultNotes
}
// #region Read/write minisearch index
public async addAllToMinisearch(
documents: IndexedDocument[],
chunkSize = 10
): Promise<void> {
await this.minisearch.addAllAsync(documents, { chunkSize })
}
public addSingleToMinisearch(path: string): void {
getIndexedDocument(path).then(doc => this.minisearch.add(doc))
}
public removeFromMinisearch(path: string): void {
this.minisearch.discard(path)
}
// #endregion
public async writeToCache(documents: IndexedDocument[]): Promise<void> {
await cacheManager.writeMinisearchCache(this.minisearch, documents)
public async writeToCache(): Promise<void> {
await cacheManager.writeMinisearchCache(
this.minisearch,
this.indexedDocuments
)
}
}
export const searchEngine = new Omnisearch()

6
src/tools/api.ts
View File

@@ -1,6 +1,6 @@
import type { ResultNote } from '../globals'
import { Query } from '../search/query'
import { SearchEngine } from '../search/search-engine'
import { searchEngine } from '../search/omnisearch'
type ResultNoteApi = {
score: number
@@ -35,8 +35,8 @@ function mapResults(results: ResultNote[]): ResultNoteApi[] {
async function search(q: string): Promise<ResultNoteApi[]> {
const query = new Query(q)
const raw = await SearchEngine.getEngine().getSuggestions(query)
const raw = await searchEngine.getSuggestions(query)
return mapResults(raw)
}
export default { search }
export default {search}

2
src/tools/utils.ts
View File

@@ -170,7 +170,7 @@ export async function filterAsync<T>(
callbackfn: (value: T, index: number, array: T[]) => Promise<boolean>
): Promise<T[]> {
const filterMap = await mapAsync(array, callbackfn)
return array.filter((value, index) => filterMap[index])
return array.filter((_value, index) => filterMap[index])
}
/**

4
src/vendor/parse-query.ts vendored
View File

@@ -86,7 +86,7 @@ export function parseQuery(
let val = term.slice(sepIndex + 1)
// Strip backslashes respecting escapes
val = (val + '').replace(/\\(.?)/g, function (s, n1) {
val = (val + '').replace(/\\(.?)/g, function (_s, n1) {
switch (n1) {
case '\\':
return '\\'
@@ -115,7 +115,7 @@ export function parseQuery(
}
// Strip backslashes respecting escapes
term = (term + '').replace(/\\(.?)/g, function (s, n1) {
term = (term + '').replace(/\\(.?)/g, function (_s, n1) {
switch (n1) {
case '\\':
return '\\'