tanner/obsidian-tannersearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Squashed commit of the following:

Browse Source 
commit 3b229cad538ad88ef2d366964c4261bc0e02fb7c
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Sat Nov 5 14:30:08 2022 +0100

    1.8.0-beta.1

commit f43c369b2dd0a1083b171724e3f7466429505629
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Sat Nov 5 13:39:45 2022 +0100

    Squashed commit of the following:

    commit 93508ee95046385baf62475e5bd835ed9fafe6d3
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Sat Nov 5 13:35:56 2022 +0100

        Cleaning

    commit 205e6a7cce4c1939338820f366f7ae8a067ec7fb
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Fri Nov 4 08:53:46 2022 +0100

        Added logs

    commit ea19b94e164581829908ac71d09a60e230925a7f
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:27:24 2022 +0100

        Notices

    commit 53ff4e822b3c292a56da150b94a1cfe43e199d44
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:27:09 2022 +0100

        Custom minisearch build + Notice when the cache could be corrupted

    commit 498408afd1c350dd68969318c3533fff8aa6c172
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:26:22 2022 +0100

        Added a button to manually clear the cache

    commit 90afe5d3868989626ba4613b064e24ac7efa88be
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:03:41 2022 +0100

        Optimized loading minisearch from cache

    commit 719dcb9c82f09f56dabb828ac13c9c1db7f795bb
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 21:43:49 2022 +0100

        #92 - Refactored cache to make it behave like pre-indexedDb

    commit 2164ccfa39d83eef23231d01e8aa35ac30e0d31c
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Wed Nov 2 23:13:59 2022 +0100

        Removed cache & tmp engine

    commit 50eb33bbd4d074be9a9952eaf871cd8f58b327e6
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Wed Nov 2 22:56:04 2022 +0100

        More efficient loading of PDFs

commit a6342a675f
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Wed Nov 2 10:34:02 2022 +0100

    #120 - Cleaning of old cache databases

commit b6890567f3
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Mon Oct 31 17:28:17 2022 +0100

    Updated Readme
This commit is contained in:
Simon Cambier
2022-11-05 14:58:25 +01:00
parent 777b172904
commit 087ec5cc99
12 changed files with 367 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
src/search/search-engine.ts
View File

@@ -1,8 +1,4 @@
import MiniSearch, {
type AsPlainObject,
type Options,
type SearchResult,
} from 'minisearch'
import MiniSearch, { type Options, type SearchResult } from 'minisearch'
import {
chsRegex,
type IndexedDocument,
@@ -19,6 +15,7 @@ import type { Query } from './query'
import { settings } from '../settings'
import { cacheManager } from '../cache-manager'
import { writable } from 'svelte/store'
import { Notice } from 'obsidian'
const tokenize = (text: string): string[] => {
const tokens = text.split(SPACE_OR_PUNCTUATION)
@@ -45,11 +42,15 @@ export const minisearchOptions: Options<IndexedDocument> = {
'headings3',
],
storeFields: ['tags'],
callbackWhenDesync() {
new Notice(
'Omnisearch - Your index cache may be incorrect or corrupted. If this message keeps appearing, go to Settings to clear the cache.'
)
},
}
export class SearchEngine {
private static engine?: SearchEngine
private static tmpEngine?: SearchEngine
public static isIndexing = writable(true)
/**
@@ -63,41 +64,23 @@ export class SearchEngine {
return this.engine
}
/**
* The secondary instance. This one is indexed in the background,
* while the main instance is quickly filled with cache data
*/
public static getTmpEngine(): SearchEngine {
if (!this.tmpEngine) {
this.tmpEngine = new SearchEngine()
}
return this.tmpEngine
}
/**
* Instantiates the main instance with cache data (if it exists)
*/
public static async initFromCache(): Promise<void> {
public static async initFromCache(): Promise<SearchEngine> {
try {
const cache = await cacheManager.getMinisearchCache()
if (cache) {
this.getEngine().minisearch = cache
}
} catch (e) {
new Notice(
'Omnisearch - Cache missing or invalid. Some freezes may occur while Omnisearch indexes your vault.'
)
console.error('Omnisearch - Could not init engine from cache')
console.error(e)
}
}
/**
* Loads the freshest indexed data into the main instance.
*/
public static loadTmpDataIntoMain(): void {
const tmpData = this.getTmpEngine().minisearch.toJSON()
this.getEngine().minisearch = MiniSearch.loadJS(tmpData, minisearchOptions)
}
public static clearTmp(): void {
this.getTmpEngine().minisearch = new MiniSearch(minisearchOptions)
return this.getEngine()
}
private minisearch: MiniSearch
@@ -147,9 +130,10 @@ export class SearchEngine {
const exactTerms = query.getExactTerms()
if (exactTerms.length) {
results = results.filter(r => {
const title = cacheManager.getDocument(r.id)?.path.toLowerCase() ?? ''
const title =
cacheManager.getLiveDocument(r.id)?.path.toLowerCase() ?? ''
const content = stripMarkdownCharacters(
cacheManager.getDocument(r.id)?.content ?? ''
cacheManager.getLiveDocument(r.id)?.content ?? ''
).toLowerCase()
return exactTerms.every(q => content.includes(q) || title.includes(q))
})
@@ -160,7 +144,7 @@ export class SearchEngine {
if (exclusions.length) {
results = results.filter(r => {
const content = stripMarkdownCharacters(
cacheManager.getDocument(r.id)?.content ?? ''
cacheManager.getLiveDocument(r.id)?.content ?? ''
).toLowerCase()
return exclusions.every(q => !content.includes(q.value))
})
@@ -240,9 +224,10 @@ export class SearchEngine {
// Map the raw results to get usable suggestions
return results.map(result => {
let note = cacheManager.getDocument(result.id)
let note = cacheManager.getLiveDocument(result.id)
if (!note) {
// throw new Error(`Omnisearch - Note "${result.id}" not indexed`)
console.warn(`Omnisearch - Note "${result.id}" not in the live cache`)
note = {
content: '',
basename: result.id,
@@ -286,8 +271,11 @@ export class SearchEngine {
// #region Read/write minisearch index
public async addAllToMinisearch(documents: IndexedDocument[]): Promise<void> {
await this.minisearch.addAllAsync(documents)
public async addAllToMinisearch(
documents: IndexedDocument[],
chunkSize = 10
): Promise<void> {
await this.minisearch.addAllAsync(documents, { chunkSize })
}
public addSingleToMinisearch(document: IndexedDocument): void {
@@ -300,7 +288,7 @@ export class SearchEngine {
// #endregion
public async writeToCache(): Promise<void> {
await cacheManager.writeMinisearchCache(this.minisearch)
public async writeToCache(documents: IndexedDocument[]): Promise<void> {
await cacheManager.writeMinisearchCache(this.minisearch, documents)
}
}