tanner/obsidian-tannersearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Renamed and reorganized some objets & methods

Browse Source
This commit is contained in:
Simon Cambier
2024-05-26 17:39:31 +02:00
parent 0e9f7d9c20
commit 85b7810d07
7 changed files with 165 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
src/cache-manager.ts
View File

@@ -1,5 +1,5 @@
import { Notice, TFile } from 'obsidian'
import type { DocumentRef, IndexedDocument } from './globals'
import { TFile } from 'obsidian'
import type { IndexedDocument } from './globals'
import {
extractHeadingsFromCache,
getAliasesFromMetadata,
@@ -10,14 +10,12 @@ import {
isFileOffice,
isFilePDF,
logDebug,
makeMD5,
removeDiacritics,
stripMarkdownCharacters,
} from './tools/utils'
import type { CanvasData } from 'obsidian/canvas'
import type MiniSearch from 'minisearch'
import type { AsPlainObject } from 'minisearch'
import type OmnisearchPlugin from './main'
import { getNonExistingNotes } from './tools/notes'
export class CacheManager {
/**
@@ -73,8 +71,7 @@ export class CacheManager {
return
}
this.nextQueryIsEmpty = false
// TODO: rename
const database = this.plugin.cache
const database = this.plugin.database
let history = await database.searchHistory.toArray()
history = history.filter(s => s.query !== query).reverse()
history.unshift({ query })
@@ -87,7 +84,7 @@ export class CacheManager {
* @returns The search history, in reverse chronological order
*/
public async getSearchHistory(): Promise<ReadonlyArray<string>> {
const data = (await this.plugin.cache.searchHistory.toArray())
const data = (await this.plugin.database.searchHistory.toArray())
.reverse()
.map(o => o.query)
if (this.nextQueryIsEmpty) {
@@ -96,101 +93,6 @@ export class CacheManager {
return data
}
public getDocumentsChecksum(documents: IndexedDocument[]): string {
return makeMD5(
JSON.stringify(
documents.sort((a, b) => {
if (a.path < b.path) {
return -1
} else if (a.path > b.path) {
return 1
}
return 0
})
)
)
}
//#region Minisearch
public async getMinisearchCache(): Promise<{
paths: DocumentRef[]
data: AsPlainObject
} | null> {
try {
const cachedIndex = (
await this.plugin.cache.minisearch.toArray()
)[0]
return cachedIndex
} catch (e) {
new Notice(
'Omnisearch - Cache missing or invalid. Some freezes may occur while Omnisearch indexes your vault.'
)
console.error('Omnisearch - Error while loading Minisearch cache')
console.error(e)
return null
}
}
public async writeMinisearchCache(
minisearch: MiniSearch,
indexed: Map<string, number>
): Promise<void> {
const paths = Array.from(indexed).map(([k, v]) => ({ path: k, mtime: v }))
// TODO: rename
const database = this.plugin.cache
await database.minisearch.clear()
await database.minisearch.add({
date: new Date().toISOString(),
paths,
data: minisearch.toJSON(),
})
console.log('Omnisearch - Search cache written')
}
public isFileIndexable(path: string): boolean {
return this.isFilenameIndexable(path) || this.isContentIndexable(path)
}
//#endregion Minisearch
public isContentIndexable(path: string): boolean {
const settings = this.plugin.settings
const hasTextExtractor = !!this.plugin.getTextExtractor()
const canIndexPDF = hasTextExtractor && settings.PDFIndexing
const canIndexImages = hasTextExtractor && settings.imagesIndexing
return (
this.isFilePlaintext(path) ||
isFileCanvas(path) ||
isFileFromDataloomPlugin(path) ||
(canIndexPDF && isFilePDF(path)) ||
(canIndexImages && isFileImage(path))
)
}
public isFilenameIndexable(path: string): boolean {
return (
this.canIndexUnsupportedFiles() ||
this.isFilePlaintext(path) ||
isFileCanvas(path) ||
isFileFromDataloomPlugin(path)
)
}
public canIndexUnsupportedFiles(): boolean {
return (
this.plugin.settings.unsupportedFilesIndexing === 'yes' ||
(this.plugin.settings.unsupportedFilesIndexing === 'default' &&
!!this.plugin.app.vault.getConfig('showUnsupportedFiles'))
)
}
private isFilePlaintext(path: string): boolean {
return [...this.plugin.settings.indexedFileTypes, 'md'].some(t =>
path.endsWith(`.${t}`)
)
}
/**
* This function is responsible for extracting the text from a file and
* returning it as an `IndexedDocument` object.
@@ -209,7 +111,7 @@ export class CacheManager {
// ** Plain text **
// Just read the file content
if (this.isFilePlaintext(path)) {
if (this.plugin.notesIndexer.isFilePlaintext(path)) {
content = await app.vault.cachedRead(file)
}
@@ -283,7 +185,7 @@ export class CacheManager {
}
// ** Unsupported files **
else if (this.isFilenameIndexable(path)) {
else if (this.plugin.notesIndexer.isFilenameIndexable(path)) {
content = file.path
}
@@ -297,10 +199,9 @@ export class CacheManager {
// Look for links that lead to non-existing files,
// and add them to the index.
if (metadata) {
// // FIXME: https://github.com/scambier/obsidian-omnisearch/issues/129
// const nonExisting = getNonExistingNotes(file, metadata)
// const nonExisting = getNonExistingNotes(this.plugin.app, file, metadata)
// for (const name of nonExisting.filter(
// o => !cacheManager.getLiveDocument(o)
// o => !this.getLiveDocument(o)
// )) {
// NotesIndex.addNonExistingToIndex(name, file.path)
// }

45
src/database.ts
View File

@@ -1,10 +1,11 @@
import Dexie from 'dexie'
import type MiniSearch from 'minisearch'
import type { AsPlainObject } from 'minisearch'
import type { DocumentRef } from './globals'
import { Notice } from 'obsidian'
import type OmnisearchPlugin from './main'
export class OmnisearchCache extends Dexie {
export class Database extends Dexie {
public static readonly dbVersion = 8
searchHistory!: Dexie.Table<{ id?: number; query: string }, number>
minisearch!: Dexie.Table<
@@ -17,29 +18,61 @@ export class OmnisearchCache extends Dexie {
>
constructor(private plugin: OmnisearchPlugin) {
super(OmnisearchCache.getDbName(plugin.app.appId))
super(Database.getDbName(plugin.app.appId))
// Database structure
this.version(OmnisearchCache.dbVersion).stores({
this.version(Database.dbVersion).stores({
searchHistory: '++id',
minisearch: 'date',
})
}
public static getDbName(appId: string) {
private static getDbName(appId: string) {
return 'omnisearch/cache/' + appId
}
//#endregion Table declarations
public async getMinisearchCache(): Promise<{
paths: DocumentRef[]
data: AsPlainObject
} | null> {
try {
const cachedIndex = (await this.plugin.database.minisearch.toArray())[0]
return cachedIndex
} catch (e) {
new Notice(
'Omnisearch - Cache missing or invalid. Some freezes may occur while Omnisearch indexes your vault.'
)
console.error('Omnisearch - Error while loading Minisearch cache')
console.error(e)
return null
}
}
public async writeMinisearchCache(
minisearch: MiniSearch,
indexed: Map<string, number>
): Promise<void> {
const paths = Array.from(indexed).map(([k, v]) => ({ path: k, mtime: v }))
const database = this.plugin.database
await database.minisearch.clear()
await database.minisearch.add({
date: new Date().toISOString(),
paths,
data: minisearch.toJSON(),
})
console.log('Omnisearch - Search cache written')
}
/**
* Deletes Omnisearch databases that have an older version than the current one
*/
public async clearOldDatabases(): Promise<void> {
const toDelete = (await indexedDB.databases()).filter(
db =>
db.name === OmnisearchCache.getDbName(this.plugin.app.appId) &&
db.name === Database.getDbName(this.plugin.app.appId) &&
// version multiplied by 10 https://github.com/dexie/Dexie.js/issues/59
db.version !== OmnisearchCache.dbVersion * 10
db.version !== Database.dbVersion * 10
)
if (toDelete.length) {
console.log('Omnisearch - Those IndexedDb databases will be deleted:')

18
src/main.ts
View File

@@ -15,11 +15,11 @@ import {
} from './settings'
import { eventBus, EventNames, indexingStep, IndexingStepType, type TextExtractorApi } from './globals'
import { notifyOnIndexed, registerAPI } from './tools/api'
import { OmnisearchCache } from './database'
import { Database } from './database'
import { SearchEngine } from './search/search-engine'
import { CacheManager } from './cache-manager'
import { logDebug } from './tools/utils'
import { NotesIndexer } from './notes-index'
import { NotesIndexer } from './notes-indexer'
import { TextProcessor } from "./tools/text-processing";
export default class OmnisearchPlugin extends Plugin {
@@ -29,7 +29,7 @@ export default class OmnisearchPlugin extends Plugin {
// FIXME: merge cache and cacheManager, or find other names
public readonly cacheManager: CacheManager
public readonly cache = new OmnisearchCache(this)
public readonly database = new Database(this)
public readonly notesIndexer = new NotesIndexer(this)
public readonly textProcessor = new TextProcessor(this)
@@ -58,7 +58,7 @@ export default class OmnisearchPlugin extends Plugin {
}
await cleanOldCacheFiles(this.app)
await this.cache.clearOldDatabases()
await this.database.clearOldDatabases()
registerAPI(this)
@@ -98,7 +98,7 @@ export default class OmnisearchPlugin extends Plugin {
// Listeners to keep the search index up-to-date
this.registerEvent(
this.app.vault.on('create', file => {
if (this.cacheManager.isFileIndexable(file.path)) {
if (this.notesIndexer.isFileIndexable(file.path)) {
logDebug('Indexing new file', file.path)
// await cacheManager.addToLiveCache(file.path)
searchEngine.addFromPaths([file.path])
@@ -114,7 +114,7 @@ export default class OmnisearchPlugin extends Plugin {
)
this.registerEvent(
this.app.vault.on('modify', async file => {
if (this.cacheManager.isFileIndexable(file.path)) {
if (this.notesIndexer.isFileIndexable(file.path)) {
logDebug('Updating file', file.path)
await this.cacheManager.addToLiveCache(file.path)
this.notesIndexer.markNoteForReindex(file)
@@ -123,7 +123,7 @@ export default class OmnisearchPlugin extends Plugin {
)
this.registerEvent(
this.app.vault.on('rename', async (file, oldPath) => {
if (this.cacheManager.isFileIndexable(file.path)) {
if (this.notesIndexer.isFileIndexable(file.path)) {
logDebug('Renaming file', file.path)
this.cacheManager.removeFromLiveCache(oldPath)
await this.cacheManager.addToLiveCache(file.path)
@@ -161,7 +161,7 @@ export default class OmnisearchPlugin extends Plugin {
// Clear cache when disabling Omnisearch
if (process.env.NODE_ENV === 'production') {
await this.cache.clearCache()
await this.database.clearCache()
}
this.apiHttpServer.close()
}
@@ -199,7 +199,7 @@ export default class OmnisearchPlugin extends Plugin {
indexingStep.set(IndexingStepType.ReadingFiles)
const files = this.app.vault
.getFiles()
.filter(f => this.cacheManager.isFileIndexable(f.path))
.filter(f => this.notesIndexer.isFileIndexable(f.path))
console.log(`Omnisearch - ${files.length} files total`)
console.log(
`Omnisearch - Cache is ${isCacheEnabled() ? 'enabled' : 'disabled'}`

55
src/notes-index.ts
View File

@@ -1,55 +0,0 @@
import type { TAbstractFile } from 'obsidian'
import type OmnisearchPlugin from './main'
export class NotesIndexer {
private notesToReindex = new Set<TAbstractFile>()
constructor(private plugin: OmnisearchPlugin) {}
/**
* Updated notes are not reindexed immediately for performance reasons.
* They're added to a list, and reindex is done the next time we open Omnisearch.
*/
public markNoteForReindex(note: TAbstractFile): void {
this.notesToReindex.add(note)
}
public async refreshIndex(): Promise<void> {
const paths = [...this.notesToReindex].map(n => n.path)
if (paths.length) {
const searchEngine = this.plugin.searchEngine
searchEngine.removeFromPaths(paths)
await searchEngine.addFromPaths(paths)
this.notesToReindex.clear()
// console.log(`Omnisearch - Reindexed ${paths.length} file(s)`)
}
}
}
// /**
// * Index a non-existing note.
// * Useful to find internal links that lead (yet) to nowhere
// * @param name
// * @param parent The note referencing the
// */
// export function addNonExistingToIndex(name: string, parent: string): void {
// name = removeAnchors(name)
// const filename = name + (name.endsWith('.md') ? '' : '.md')
//
// const note: IndexedDocument = {
// path: filename,
// basename: name,
// mtime: 0,
//
// content: '',
// tags: [],
// aliases: '',
// headings1: '',
// headings2: '',
// headings3: '',
//
// doesNotExist: true,
// parent,
// }
// // searchEngine.addDocuments([note])
// }

105
src/notes-indexer.ts Normal file
View File

@@ -0,0 +1,105 @@
import type { TAbstractFile } from 'obsidian'
import type OmnisearchPlugin from './main'
import { removeAnchors } from './tools/notes'
import type { IndexedDocument } from './globals'
import {
isFileCanvas,
isFileFromDataloomPlugin,
isFileImage,
isFilePDF,
} from './tools/utils'
export class NotesIndexer {
private notesToReindex = new Set<TAbstractFile>()
constructor(private plugin: OmnisearchPlugin) {}
/**
* Updated notes are not reindexed immediately for performance reasons.
* They're added to a list, and reindex is done the next time we open Omnisearch.
*/
public markNoteForReindex(note: TAbstractFile): void {
this.notesToReindex.add(note)
}
public async refreshIndex(): Promise<void> {
const paths = [...this.notesToReindex].map(n => n.path)
if (paths.length) {
const searchEngine = this.plugin.searchEngine
searchEngine.removeFromPaths(paths)
await searchEngine.addFromPaths(paths)
this.notesToReindex.clear()
}
}
public isFileIndexable(path: string): boolean {
return this.isFilenameIndexable(path) || this.isContentIndexable(path)
}
public isContentIndexable(path: string): boolean {
const settings = this.plugin.settings
const hasTextExtractor = !!this.plugin.getTextExtractor()
const canIndexPDF = hasTextExtractor && settings.PDFIndexing
const canIndexImages = hasTextExtractor && settings.imagesIndexing
return (
this.isFilePlaintext(path) ||
isFileCanvas(path) ||
isFileFromDataloomPlugin(path) ||
(canIndexPDF && isFilePDF(path)) ||
(canIndexImages && isFileImage(path))
)
}
public isFilenameIndexable(path: string): boolean {
return (
this.canIndexUnsupportedFiles() ||
this.isFilePlaintext(path) ||
isFileCanvas(path) ||
isFileFromDataloomPlugin(path)
)
}
public canIndexUnsupportedFiles(): boolean {
return (
this.plugin.settings.unsupportedFilesIndexing === 'yes' ||
(this.plugin.settings.unsupportedFilesIndexing === 'default' &&
!!this.plugin.app.vault.getConfig('showUnsupportedFiles'))
)
}
/**
* Index a non-existing note.
* Useful to find internal links that lead (yet) to nowhere
* @param name
* @param parent The note referencing the
*/
public addNonExistingToIndex(name: string, parent: string): void {
name = removeAnchors(name)
const filename = name + (name.endsWith('.md') ? '' : '.md')
const note: IndexedDocument = {
path: filename,
basename: name,
mtime: 0,
content: '',
cleanedContent: '',
tags: [],
unmarkedTags: [],
aliases: '',
headings1: '',
headings2: '',
headings3: '',
doesNotExist: true,
parent,
}
// searchEngine.addDocuments([note])
}
public isFilePlaintext(path: string): boolean {
return [...this.plugin.settings.indexedFileTypes, 'md'].some(t =>
path.endsWith(`.${t}`)
)
}
}

4
src/search/search-engine.ts
View File

@@ -25,7 +25,7 @@ export class SearchEngine {
* Return true if the cache is valid
*/
async loadCache(): Promise<boolean> {
const cache = await this.plugin.cacheManager.getMinisearchCache()
const cache = await this.plugin.database.getMinisearchCache()
if (cache) {
this.minisearch = await MiniSearch.loadJSAsync(
cache.data,
@@ -414,7 +414,7 @@ export class SearchEngine {
}
public async writeToCache(): Promise<void> {
await this.plugin.cacheManager.writeMinisearchCache(
await this.plugin.database.writeMinisearchCache(
this.minisearch,
this.indexedDocuments
)

3
src/settings.ts
View File

@@ -93,8 +93,7 @@ export class SettingsTab extends PluginSettingTab {
display(): void {
const { containerEl } = this
// TODO: rename
const database = this.plugin.cache
const database = this.plugin.database
const textExtractor = this.plugin.getTextExtractor()
containerEl.empty()