From 85b7810d070833a86791bb0cca157ae35dbc0f9d Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sun, 26 May 2024 17:39:31 +0200 Subject: [PATCH] Renamed and reorganized some objets & methods --- src/cache-manager.ts | 117 +++--------------------------------- src/database.ts | 45 ++++++++++++-- src/main.ts | 18 +++--- src/notes-index.ts | 55 ----------------- src/notes-indexer.ts | 105 ++++++++++++++++++++++++++++++++ src/search/search-engine.ts | 4 +- src/settings.ts | 3 +- 7 files changed, 165 insertions(+), 182 deletions(-) delete mode 100644 src/notes-index.ts create mode 100644 src/notes-indexer.ts diff --git a/src/cache-manager.ts b/src/cache-manager.ts index 5189e81..105ed6e 100644 --- a/src/cache-manager.ts +++ b/src/cache-manager.ts @@ -1,5 +1,5 @@ -import { Notice, TFile } from 'obsidian' -import type { DocumentRef, IndexedDocument } from './globals' +import { TFile } from 'obsidian' +import type { IndexedDocument } from './globals' import { extractHeadingsFromCache, getAliasesFromMetadata, @@ -10,14 +10,12 @@ import { isFileOffice, isFilePDF, logDebug, - makeMD5, removeDiacritics, stripMarkdownCharacters, } from './tools/utils' import type { CanvasData } from 'obsidian/canvas' -import type MiniSearch from 'minisearch' -import type { AsPlainObject } from 'minisearch' import type OmnisearchPlugin from './main' +import { getNonExistingNotes } from './tools/notes' export class CacheManager { /** @@ -73,8 +71,7 @@ export class CacheManager { return } this.nextQueryIsEmpty = false - // TODO: rename - const database = this.plugin.cache + const database = this.plugin.database let history = await database.searchHistory.toArray() history = history.filter(s => s.query !== query).reverse() history.unshift({ query }) @@ -87,7 +84,7 @@ export class CacheManager { * @returns The search history, in reverse chronological order */ public async getSearchHistory(): Promise> { - const data = (await this.plugin.cache.searchHistory.toArray()) + const data = (await this.plugin.database.searchHistory.toArray()) .reverse() .map(o => o.query) if (this.nextQueryIsEmpty) { @@ -96,101 +93,6 @@ export class CacheManager { return data } - public getDocumentsChecksum(documents: IndexedDocument[]): string { - return makeMD5( - JSON.stringify( - documents.sort((a, b) => { - if (a.path < b.path) { - return -1 - } else if (a.path > b.path) { - return 1 - } - return 0 - }) - ) - ) - } - - //#region Minisearch - - public async getMinisearchCache(): Promise<{ - paths: DocumentRef[] - data: AsPlainObject - } | null> { - try { - const cachedIndex = ( - await this.plugin.cache.minisearch.toArray() - )[0] - return cachedIndex - } catch (e) { - new Notice( - 'Omnisearch - Cache missing or invalid. Some freezes may occur while Omnisearch indexes your vault.' - ) - console.error('Omnisearch - Error while loading Minisearch cache') - console.error(e) - return null - } - } - - public async writeMinisearchCache( - minisearch: MiniSearch, - indexed: Map - ): Promise { - const paths = Array.from(indexed).map(([k, v]) => ({ path: k, mtime: v })) - // TODO: rename - const database = this.plugin.cache - await database.minisearch.clear() - await database.minisearch.add({ - date: new Date().toISOString(), - paths, - data: minisearch.toJSON(), - }) - console.log('Omnisearch - Search cache written') - } - - public isFileIndexable(path: string): boolean { - return this.isFilenameIndexable(path) || this.isContentIndexable(path) - } - - //#endregion Minisearch - - public isContentIndexable(path: string): boolean { - const settings = this.plugin.settings - const hasTextExtractor = !!this.plugin.getTextExtractor() - const canIndexPDF = hasTextExtractor && settings.PDFIndexing - const canIndexImages = hasTextExtractor && settings.imagesIndexing - return ( - this.isFilePlaintext(path) || - isFileCanvas(path) || - isFileFromDataloomPlugin(path) || - (canIndexPDF && isFilePDF(path)) || - (canIndexImages && isFileImage(path)) - ) - } - - public isFilenameIndexable(path: string): boolean { - return ( - this.canIndexUnsupportedFiles() || - this.isFilePlaintext(path) || - isFileCanvas(path) || - isFileFromDataloomPlugin(path) - ) - } - - public canIndexUnsupportedFiles(): boolean { - return ( - this.plugin.settings.unsupportedFilesIndexing === 'yes' || - (this.plugin.settings.unsupportedFilesIndexing === 'default' && - !!this.plugin.app.vault.getConfig('showUnsupportedFiles')) - ) - } - - private isFilePlaintext(path: string): boolean { - return [...this.plugin.settings.indexedFileTypes, 'md'].some(t => - path.endsWith(`.${t}`) - ) - } - /** * This function is responsible for extracting the text from a file and * returning it as an `IndexedDocument` object. @@ -209,7 +111,7 @@ export class CacheManager { // ** Plain text ** // Just read the file content - if (this.isFilePlaintext(path)) { + if (this.plugin.notesIndexer.isFilePlaintext(path)) { content = await app.vault.cachedRead(file) } @@ -283,7 +185,7 @@ export class CacheManager { } // ** Unsupported files ** - else if (this.isFilenameIndexable(path)) { + else if (this.plugin.notesIndexer.isFilenameIndexable(path)) { content = file.path } @@ -297,10 +199,9 @@ export class CacheManager { // Look for links that lead to non-existing files, // and add them to the index. if (metadata) { - // // FIXME: https://github.com/scambier/obsidian-omnisearch/issues/129 - // const nonExisting = getNonExistingNotes(file, metadata) + // const nonExisting = getNonExistingNotes(this.plugin.app, file, metadata) // for (const name of nonExisting.filter( - // o => !cacheManager.getLiveDocument(o) + // o => !this.getLiveDocument(o) // )) { // NotesIndex.addNonExistingToIndex(name, file.path) // } diff --git a/src/database.ts b/src/database.ts index 7464398..e3e9d32 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,10 +1,11 @@ import Dexie from 'dexie' +import type MiniSearch from 'minisearch' import type { AsPlainObject } from 'minisearch' import type { DocumentRef } from './globals' import { Notice } from 'obsidian' import type OmnisearchPlugin from './main' -export class OmnisearchCache extends Dexie { +export class Database extends Dexie { public static readonly dbVersion = 8 searchHistory!: Dexie.Table<{ id?: number; query: string }, number> minisearch!: Dexie.Table< @@ -17,29 +18,61 @@ export class OmnisearchCache extends Dexie { > constructor(private plugin: OmnisearchPlugin) { - super(OmnisearchCache.getDbName(plugin.app.appId)) + super(Database.getDbName(plugin.app.appId)) // Database structure - this.version(OmnisearchCache.dbVersion).stores({ + this.version(Database.dbVersion).stores({ searchHistory: '++id', minisearch: 'date', }) } - public static getDbName(appId: string) { + private static getDbName(appId: string) { return 'omnisearch/cache/' + appId } //#endregion Table declarations + public async getMinisearchCache(): Promise<{ + paths: DocumentRef[] + data: AsPlainObject + } | null> { + try { + const cachedIndex = (await this.plugin.database.minisearch.toArray())[0] + return cachedIndex + } catch (e) { + new Notice( + 'Omnisearch - Cache missing or invalid. Some freezes may occur while Omnisearch indexes your vault.' + ) + console.error('Omnisearch - Error while loading Minisearch cache') + console.error(e) + return null + } + } + + public async writeMinisearchCache( + minisearch: MiniSearch, + indexed: Map + ): Promise { + const paths = Array.from(indexed).map(([k, v]) => ({ path: k, mtime: v })) + const database = this.plugin.database + await database.minisearch.clear() + await database.minisearch.add({ + date: new Date().toISOString(), + paths, + data: minisearch.toJSON(), + }) + console.log('Omnisearch - Search cache written') + } + /** * Deletes Omnisearch databases that have an older version than the current one */ public async clearOldDatabases(): Promise { const toDelete = (await indexedDB.databases()).filter( db => - db.name === OmnisearchCache.getDbName(this.plugin.app.appId) && + db.name === Database.getDbName(this.plugin.app.appId) && // version multiplied by 10 https://github.com/dexie/Dexie.js/issues/59 - db.version !== OmnisearchCache.dbVersion * 10 + db.version !== Database.dbVersion * 10 ) if (toDelete.length) { console.log('Omnisearch - Those IndexedDb databases will be deleted:') diff --git a/src/main.ts b/src/main.ts index ea9ba04..3f1610d 100644 --- a/src/main.ts +++ b/src/main.ts @@ -15,11 +15,11 @@ import { } from './settings' import { eventBus, EventNames, indexingStep, IndexingStepType, type TextExtractorApi } from './globals' import { notifyOnIndexed, registerAPI } from './tools/api' -import { OmnisearchCache } from './database' +import { Database } from './database' import { SearchEngine } from './search/search-engine' import { CacheManager } from './cache-manager' import { logDebug } from './tools/utils' -import { NotesIndexer } from './notes-index' +import { NotesIndexer } from './notes-indexer' import { TextProcessor } from "./tools/text-processing"; export default class OmnisearchPlugin extends Plugin { @@ -29,7 +29,7 @@ export default class OmnisearchPlugin extends Plugin { // FIXME: merge cache and cacheManager, or find other names public readonly cacheManager: CacheManager - public readonly cache = new OmnisearchCache(this) + public readonly database = new Database(this) public readonly notesIndexer = new NotesIndexer(this) public readonly textProcessor = new TextProcessor(this) @@ -58,7 +58,7 @@ export default class OmnisearchPlugin extends Plugin { } await cleanOldCacheFiles(this.app) - await this.cache.clearOldDatabases() + await this.database.clearOldDatabases() registerAPI(this) @@ -98,7 +98,7 @@ export default class OmnisearchPlugin extends Plugin { // Listeners to keep the search index up-to-date this.registerEvent( this.app.vault.on('create', file => { - if (this.cacheManager.isFileIndexable(file.path)) { + if (this.notesIndexer.isFileIndexable(file.path)) { logDebug('Indexing new file', file.path) // await cacheManager.addToLiveCache(file.path) searchEngine.addFromPaths([file.path]) @@ -114,7 +114,7 @@ export default class OmnisearchPlugin extends Plugin { ) this.registerEvent( this.app.vault.on('modify', async file => { - if (this.cacheManager.isFileIndexable(file.path)) { + if (this.notesIndexer.isFileIndexable(file.path)) { logDebug('Updating file', file.path) await this.cacheManager.addToLiveCache(file.path) this.notesIndexer.markNoteForReindex(file) @@ -123,7 +123,7 @@ export default class OmnisearchPlugin extends Plugin { ) this.registerEvent( this.app.vault.on('rename', async (file, oldPath) => { - if (this.cacheManager.isFileIndexable(file.path)) { + if (this.notesIndexer.isFileIndexable(file.path)) { logDebug('Renaming file', file.path) this.cacheManager.removeFromLiveCache(oldPath) await this.cacheManager.addToLiveCache(file.path) @@ -161,7 +161,7 @@ export default class OmnisearchPlugin extends Plugin { // Clear cache when disabling Omnisearch if (process.env.NODE_ENV === 'production') { - await this.cache.clearCache() + await this.database.clearCache() } this.apiHttpServer.close() } @@ -199,7 +199,7 @@ export default class OmnisearchPlugin extends Plugin { indexingStep.set(IndexingStepType.ReadingFiles) const files = this.app.vault .getFiles() - .filter(f => this.cacheManager.isFileIndexable(f.path)) + .filter(f => this.notesIndexer.isFileIndexable(f.path)) console.log(`Omnisearch - ${files.length} files total`) console.log( `Omnisearch - Cache is ${isCacheEnabled() ? 'enabled' : 'disabled'}` diff --git a/src/notes-index.ts b/src/notes-index.ts deleted file mode 100644 index 81a3483..0000000 --- a/src/notes-index.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { TAbstractFile } from 'obsidian' -import type OmnisearchPlugin from './main' - -export class NotesIndexer { - private notesToReindex = new Set() - - constructor(private plugin: OmnisearchPlugin) {} - - /** - * Updated notes are not reindexed immediately for performance reasons. - * They're added to a list, and reindex is done the next time we open Omnisearch. - */ - public markNoteForReindex(note: TAbstractFile): void { - this.notesToReindex.add(note) - } - - public async refreshIndex(): Promise { - const paths = [...this.notesToReindex].map(n => n.path) - if (paths.length) { - const searchEngine = this.plugin.searchEngine - searchEngine.removeFromPaths(paths) - await searchEngine.addFromPaths(paths) - this.notesToReindex.clear() - // console.log(`Omnisearch - Reindexed ${paths.length} file(s)`) - } - } -} - -// /** -// * Index a non-existing note. -// * Useful to find internal links that lead (yet) to nowhere -// * @param name -// * @param parent The note referencing the -// */ -// export function addNonExistingToIndex(name: string, parent: string): void { -// name = removeAnchors(name) -// const filename = name + (name.endsWith('.md') ? '' : '.md') -// -// const note: IndexedDocument = { -// path: filename, -// basename: name, -// mtime: 0, -// -// content: '', -// tags: [], -// aliases: '', -// headings1: '', -// headings2: '', -// headings3: '', -// -// doesNotExist: true, -// parent, -// } -// // searchEngine.addDocuments([note]) -// } diff --git a/src/notes-indexer.ts b/src/notes-indexer.ts new file mode 100644 index 0000000..cffaf6e --- /dev/null +++ b/src/notes-indexer.ts @@ -0,0 +1,105 @@ +import type { TAbstractFile } from 'obsidian' +import type OmnisearchPlugin from './main' +import { removeAnchors } from './tools/notes' +import type { IndexedDocument } from './globals' +import { + isFileCanvas, + isFileFromDataloomPlugin, + isFileImage, + isFilePDF, +} from './tools/utils' + +export class NotesIndexer { + private notesToReindex = new Set() + + constructor(private plugin: OmnisearchPlugin) {} + + /** + * Updated notes are not reindexed immediately for performance reasons. + * They're added to a list, and reindex is done the next time we open Omnisearch. + */ + public markNoteForReindex(note: TAbstractFile): void { + this.notesToReindex.add(note) + } + + public async refreshIndex(): Promise { + const paths = [...this.notesToReindex].map(n => n.path) + if (paths.length) { + const searchEngine = this.plugin.searchEngine + searchEngine.removeFromPaths(paths) + await searchEngine.addFromPaths(paths) + this.notesToReindex.clear() + } + } + + public isFileIndexable(path: string): boolean { + return this.isFilenameIndexable(path) || this.isContentIndexable(path) + } + + public isContentIndexable(path: string): boolean { + const settings = this.plugin.settings + const hasTextExtractor = !!this.plugin.getTextExtractor() + const canIndexPDF = hasTextExtractor && settings.PDFIndexing + const canIndexImages = hasTextExtractor && settings.imagesIndexing + return ( + this.isFilePlaintext(path) || + isFileCanvas(path) || + isFileFromDataloomPlugin(path) || + (canIndexPDF && isFilePDF(path)) || + (canIndexImages && isFileImage(path)) + ) + } + + public isFilenameIndexable(path: string): boolean { + return ( + this.canIndexUnsupportedFiles() || + this.isFilePlaintext(path) || + isFileCanvas(path) || + isFileFromDataloomPlugin(path) + ) + } + + public canIndexUnsupportedFiles(): boolean { + return ( + this.plugin.settings.unsupportedFilesIndexing === 'yes' || + (this.plugin.settings.unsupportedFilesIndexing === 'default' && + !!this.plugin.app.vault.getConfig('showUnsupportedFiles')) + ) + } + + /** + * Index a non-existing note. + * Useful to find internal links that lead (yet) to nowhere + * @param name + * @param parent The note referencing the + */ + public addNonExistingToIndex(name: string, parent: string): void { + name = removeAnchors(name) + const filename = name + (name.endsWith('.md') ? '' : '.md') + + const note: IndexedDocument = { + path: filename, + basename: name, + mtime: 0, + + content: '', + cleanedContent: '', + tags: [], + unmarkedTags: [], + aliases: '', + headings1: '', + headings2: '', + headings3: '', + + doesNotExist: true, + parent, + } + // searchEngine.addDocuments([note]) + } + + public isFilePlaintext(path: string): boolean { + return [...this.plugin.settings.indexedFileTypes, 'md'].some(t => + path.endsWith(`.${t}`) + ) + } +} diff --git a/src/search/search-engine.ts b/src/search/search-engine.ts index bd7d74d..dba9a80 100644 --- a/src/search/search-engine.ts +++ b/src/search/search-engine.ts @@ -25,7 +25,7 @@ export class SearchEngine { * Return true if the cache is valid */ async loadCache(): Promise { - const cache = await this.plugin.cacheManager.getMinisearchCache() + const cache = await this.plugin.database.getMinisearchCache() if (cache) { this.minisearch = await MiniSearch.loadJSAsync( cache.data, @@ -414,7 +414,7 @@ export class SearchEngine { } public async writeToCache(): Promise { - await this.plugin.cacheManager.writeMinisearchCache( + await this.plugin.database.writeMinisearchCache( this.minisearch, this.indexedDocuments ) diff --git a/src/settings.ts b/src/settings.ts index 177ffb4..e286ba9 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -93,8 +93,7 @@ export class SettingsTab extends PluginSettingTab { display(): void { const { containerEl } = this - // TODO: rename - const database = this.plugin.cache + const database = this.plugin.database const textExtractor = this.plugin.getTextExtractor() containerEl.empty()