tanner/obsidian-tannersearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Squashed commit of the following:

Browse Source 
commit 3b229cad538ad88ef2d366964c4261bc0e02fb7c
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Sat Nov 5 14:30:08 2022 +0100

    1.8.0-beta.1

commit f43c369b2dd0a1083b171724e3f7466429505629
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Sat Nov 5 13:39:45 2022 +0100

    Squashed commit of the following:

    commit 93508ee95046385baf62475e5bd835ed9fafe6d3
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Sat Nov 5 13:35:56 2022 +0100

        Cleaning

    commit 205e6a7cce4c1939338820f366f7ae8a067ec7fb
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Fri Nov 4 08:53:46 2022 +0100

        Added logs

    commit ea19b94e164581829908ac71d09a60e230925a7f
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:27:24 2022 +0100

        Notices

    commit 53ff4e822b3c292a56da150b94a1cfe43e199d44
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:27:09 2022 +0100

        Custom minisearch build + Notice when the cache could be corrupted

    commit 498408afd1c350dd68969318c3533fff8aa6c172
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:26:22 2022 +0100

        Added a button to manually clear the cache

    commit 90afe5d3868989626ba4613b064e24ac7efa88be
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:03:41 2022 +0100

        Optimized loading minisearch from cache

    commit 719dcb9c82f09f56dabb828ac13c9c1db7f795bb
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 21:43:49 2022 +0100

        #92 - Refactored cache to make it behave like pre-indexedDb

    commit 2164ccfa39d83eef23231d01e8aa35ac30e0d31c
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Wed Nov 2 23:13:59 2022 +0100

        Removed cache & tmp engine

    commit 50eb33bbd4d074be9a9952eaf871cd8f58b327e6
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Wed Nov 2 22:56:04 2022 +0100

        More efficient loading of PDFs

commit a6342a675f
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Wed Nov 2 10:34:02 2022 +0100

    #120 - Cleaning of old cache databases

commit b6890567f3
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Mon Oct 31 17:28:17 2022 +0100

    Updated Readme
This commit is contained in:
Simon Cambier
2022-11-05 14:58:25 +01:00
parent 777b172904
commit 087ec5cc99
12 changed files with 367 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
src/cache-manager.ts
View File

@@ -1,11 +1,13 @@
import type { TFile } from 'obsidian'
import { Notice, type TFile } from 'obsidian'
import type { IndexedDocument } from './globals'
import { database } from './database'
import MiniSearch from 'minisearch'
import { minisearchOptions } from './search/search-engine'
import { makeMD5, wait } from './tools/utils'
import { settings } from './settings'
class CacheManager {
private documentsCache: Map<string, IndexedDocument> = new Map()
private liveDocuments: Map<string, IndexedDocument> = new Map()
/**
* Show an empty input field next time the user opens Omnisearch modal
*/
@@ -35,48 +37,147 @@ class CacheManager {
return data
}
public async updateDocument(path: string, note: IndexedDocument) {
this.documentsCache.set(path, note)
/**
* Important: keep this method async for the day it _really_ becomes async.
* This will avoid a refactor.
* @param path
* @param note
*/
public async updateLiveDocument(
path: string,
note: IndexedDocument
): Promise<void> {
this.liveDocuments.set(path, note)
}
public deleteDocument(key: string): void {
this.documentsCache.delete(key)
public deleteLiveDocument(key: string): void {
this.liveDocuments.delete(key)
}
public getDocument(key: string): IndexedDocument | undefined {
return this.documentsCache.get(key)
}
public getNonExistingNotesFromMemCache(): IndexedDocument[] {
return Object.values(this.documentsCache).filter(note => note.doesNotExist)
public getLiveDocument(key: string): IndexedDocument | undefined {
return this.liveDocuments.get(key)
}
public isDocumentOutdated(file: TFile): boolean {
const indexedNote = this.getDocument(file.path)
const indexedNote = this.getLiveDocument(file.path)
return !indexedNote || indexedNote.mtime !== file.stat.mtime
}
//#region Minisearch
public getDocumentsChecksum(documents: IndexedDocument[]): string {
return makeMD5(
JSON.stringify(
documents.sort((a, b) => {
if (a.path < b.path) {
return -1
} else if (a.path > b.path) {
return 1
}
return 0
})
)
)
}
public async getMinisearchCache(): Promise<MiniSearch | null> {
const cache = (await database.minisearch.toArray())[0]
if (!cache) {
// Retrieve documents and make their checksum
const cachedDocs = await database.documents.toArray()
const checksum = this.getDocumentsChecksum(cachedDocs.map(d => d.document))
// Add those documents in the live cache
cachedDocs.forEach(doc =>
cacheManager.updateLiveDocument(doc.path, doc.document)
)
// Retrieve the search cache, and verify the checksum
const cachedIndex = (await database.minisearch.toArray())[0]
if (cachedIndex?.checksum !== checksum) {
console.warn("Omnisearch - Cache - Checksums don't match, clearing cache")
// Invalid (or null) cache, clear everything
await database.minisearch.clear()
await database.documents.clear()
return null
}
try {
return MiniSearch.loadJSON(cache.data, minisearchOptions)
return MiniSearch.loadJS(cachedIndex.data, minisearchOptions)
} catch (e) {
if (settings.showIndexingNotices) {
new Notice(
'Omnisearch - Cache missing or invalid. Some freezes may occur while Omnisearch indexes your vault.'
)
}
console.error('Omnisearch - Error while loading Minisearch cache')
console.error(e)
return null
}
}
public async writeMinisearchCache(minisearch: MiniSearch): Promise<void> {
/**
* Get a dict listing the deleted/added documents since last cache
* @param documents
*/
public async getDiffDocuments(documents: IndexedDocument[]): Promise<{
toDelete: IndexedDocument[]
toAdd: IndexedDocument[]
toUpdate: { old: IndexedDocument; new: IndexedDocument }[]
}> {
let cachedDocs = await database.documents.toArray()
const toAdd = documents.filter(
d => !cachedDocs.find(c => c.path === d.path)
)
const toDelete = cachedDocs
.filter(c => !documents.find(d => d.path === c.path))
.map(d => d.document)
const toUpdate = cachedDocs
.filter(c =>
documents.find(d => d.path === c.path && d.mtime !== c.mtime)
)
.map(c => ({
old: c.document,
new: documents.find(d => d.path === c.path)!,
}))
return {
toDelete,
toAdd,
toUpdate,
}
}
public async writeMinisearchCache(
minisearch: MiniSearch,
documents: IndexedDocument[]
): Promise<void> {
const { toDelete, toAdd, toUpdate } = await this.getDiffDocuments(documents)
// Delete
// console.log(`Omnisearch - Cache - Will delete ${toDelete.length} documents`)
await database.documents.bulkDelete(toDelete.map(o => o.path))
// Add
// console.log(`Omnisearch - Cache - Will add ${toAdd.length} documents`)
await database.documents.bulkAdd(
toAdd.map(o => ({ document: o, mtime: o.mtime, path: o.path }))
)
// Update
// console.log(`Omnisearch - Cache - Will update ${toUpdate.length} documents`)
await database.documents.bulkPut(
toUpdate.map(o => ({
document: o.new,
mtime: o.new.mtime,
path: o.new.path,
}))
)
await database.minisearch.clear()
await database.minisearch.add({
date: new Date().toISOString(),
data: JSON.stringify(minisearch.toJSON()),
checksum: this.getDocumentsChecksum(documents),
data: minisearch.toJSON(),
})
console.log('Omnisearch - Search cache written')
}