From 2f6e25ce4731dd2d77373c2dfea1f7e8b1491248 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Fri, 7 Jun 2024 12:30:00 +0200 Subject: [PATCH 01/18] Don't needlessly refresh the live cache --- src/main.ts | 12 +++++++++--- src/notes-indexer.ts | 6 ++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main.ts b/src/main.ts index 4924dd0..4817945 100644 --- a/src/main.ts +++ b/src/main.ts @@ -42,6 +42,7 @@ export default class OmnisearchPlugin extends Plugin { public readonly searchEngine = new SearchEngine(this) private ribbonButton?: HTMLElement + private refreshIndexCallback?: () => void constructor(app: App, manifest: PluginManifest) { super(app, manifest) @@ -106,7 +107,6 @@ export default class OmnisearchPlugin extends Plugin { this.app.vault.on('create', file => { if (this.notesIndexer.isFileIndexable(file.path)) { logDebug('Indexing new file', file.path) - // await cacheManager.addToLiveCache(file.path) searchEngine.addFromPaths([file.path]) } }) @@ -121,8 +121,6 @@ export default class OmnisearchPlugin extends Plugin { this.registerEvent( this.app.vault.on('modify', async file => { if (this.notesIndexer.isFileIndexable(file.path)) { - logDebug('Updating file', file.path) - await this.cacheManager.addToLiveCache(file.path) this.notesIndexer.flagNoteForReindex(file) } }) @@ -139,6 +137,10 @@ export default class OmnisearchPlugin extends Plugin { }) ) + this.refreshIndexCallback = this.notesIndexer.refreshIndex.bind(this.notesIndexer) + addEventListener('blur', this.refreshIndexCallback) + removeEventListener + await this.executeFirstLaunchTasks() await this.populateIndex() @@ -165,6 +167,10 @@ export default class OmnisearchPlugin extends Plugin { // @ts-ignore delete globalThis['omnisearch'] + if (this.refreshIndexCallback) { + removeEventListener('blur', this.refreshIndexCallback) + } + // Clear cache when disabling Omnisearch if (process.env.NODE_ENV === 'production') { await this.database.clearCache() diff --git a/src/notes-indexer.ts b/src/notes-indexer.ts index 611051b..89f51f7 100644 --- a/src/notes-indexer.ts +++ b/src/notes-indexer.ts @@ -7,6 +7,7 @@ import { isFileFromDataloomPlugin, isFileImage, isFilePDF, + logDebug, } from './tools/utils' export class NotesIndexer { @@ -23,6 +24,11 @@ export class NotesIndexer { } public async refreshIndex(): Promise { + for (const file of this.notesToReindex) { + logDebug('Updating file', file.path) + await this.plugin.cacheManager.addToLiveCache(file.path) + } + const paths = [...this.notesToReindex].map(n => n.path) if (paths.length) { this.plugin.searchEngine.removeFromPaths(paths) From 5d95817f0bcdcee13fa825d4f6787fb0ed5849e5 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Fri, 7 Jun 2024 12:31:24 +0200 Subject: [PATCH 02/18] 1.24.0-beta.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 73e3e86..89cd06b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "scambier.obsidian-search", - "version": "1.23.0", + "version": "1.24.0-beta.1", "description": "A search engine for Obsidian", "main": "dist/main.js", "scripts": { From 6cd777ea73e0e438d8e0984a4e49eb0727e90d05 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sun, 16 Jun 2024 20:22:48 +0200 Subject: [PATCH 03/18] Fixed notice --- src/settings.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/settings.ts b/src/settings.ts index 8512252..a02503e 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -674,10 +674,11 @@ export class SettingsTab extends PluginSettingTab { toggle.setValue(isPluginDisabled(this.app)).onChange(async v => { if (v) { this.app.saveLocalStorage(K_DISABLE_OMNISEARCH, '1') + new Notice('Omnisearch - Disabled. Please restart Obsidian.') } else { this.app.saveLocalStorage(K_DISABLE_OMNISEARCH) // No value = unset + new Notice('Omnisearch - Enabled. Please restart Obsidian.') } - new Notice('Omnisearch - Disabled. Please restart Obsidian.') }) ) From 074a96fcfff5c090d85abe2099b207e794ca106f Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Thu, 20 Jun 2024 18:34:54 +0200 Subject: [PATCH 04/18] Show "dashboard" icon for excalidraw files --- src/cache-manager.ts | 4 ++-- src/components/ResultItemVault.svelte | 17 ++++++++++++----- src/notes-indexer.ts | 6 +++--- src/tools/utils.ts | 6 +++++- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/cache-manager.ts b/src/cache-manager.ts index 6b8ebaf..8b94e9e 100644 --- a/src/cache-manager.ts +++ b/src/cache-manager.ts @@ -5,7 +5,7 @@ import { getAliasesFromMetadata, getTagsFromMetadata, isFileCanvas, - isFileFromDataloomPlugin, + isFileFromDataloom, isFileImage, isFileOffice, isFilePDF, @@ -135,7 +135,7 @@ export class CacheManager { } // ** Dataloom plugin ** - else if (isFileFromDataloomPlugin(path)) { + else if (isFileFromDataloom(path)) { try { const data = JSON.parse(await app.vault.cachedRead(file)) // data is a json object, we recursively iterate the keys diff --git a/src/components/ResultItemVault.svelte b/src/components/ResultItemVault.svelte index d5a6387..22f9ea6 100644 --- a/src/components/ResultItemVault.svelte +++ b/src/components/ResultItemVault.svelte @@ -3,7 +3,7 @@ import type { ResultNote } from '../globals' import { getExtension, - isFileCanvas, + isFileCanvas, isFileExcalidraw, isFileImage, isFilePDF, pathWithoutFilename, @@ -44,11 +44,18 @@ setIcon(elFolderPathIcon, 'folder-open') } if (elFilePathIcon) { - if (isFileImage(note.path)) setIcon(elFilePathIcon, 'image') - else if (isFilePDF(note.path)) setIcon(elFilePathIcon, 'file-text') - else if (isFileCanvas(note.path)) + if (isFileImage(note.path)) { + setIcon(elFilePathIcon, 'image') + } + else if (isFilePDF(note.path)) { + setIcon(elFilePathIcon, 'file-text') + } + else if (isFileCanvas(note.path) || isFileExcalidraw(note.path)) { setIcon(elFilePathIcon, 'layout-dashboard') - else setIcon(elFilePathIcon, 'file') + } + else { + setIcon(elFilePathIcon, 'file') + } } } diff --git a/src/notes-indexer.ts b/src/notes-indexer.ts index 89f51f7..c5e56f6 100644 --- a/src/notes-indexer.ts +++ b/src/notes-indexer.ts @@ -4,7 +4,7 @@ import { removeAnchors } from './tools/notes' import type { IndexedDocument } from './globals' import { isFileCanvas, - isFileFromDataloomPlugin, + isFileFromDataloom, isFileImage, isFilePDF, logDebug, @@ -49,7 +49,7 @@ export class NotesIndexer { return ( this.isFilePlaintext(path) || isFileCanvas(path) || - isFileFromDataloomPlugin(path) || + isFileFromDataloom(path) || (canIndexPDF && isFilePDF(path)) || (canIndexImages && isFileImage(path)) ) @@ -60,7 +60,7 @@ export class NotesIndexer { this.canIndexUnsupportedFiles() || this.isFilePlaintext(path) || isFileCanvas(path) || - isFileFromDataloomPlugin(path) + isFileFromDataloom(path) ) } diff --git a/src/tools/utils.ts b/src/tools/utils.ts index 664d1de..1c32558 100644 --- a/src/tools/utils.ts +++ b/src/tools/utils.ts @@ -152,7 +152,11 @@ export function isFileCanvas(path: string): boolean { return path.endsWith('.canvas') } -export function isFileFromDataloomPlugin(path: string): boolean { +export function isFileExcalidraw(path: string): boolean { + return path.endsWith('.excalidraw') +} + +export function isFileFromDataloom(path: string): boolean { return path.endsWith('.loom') } From 85a23d8352d5f23e5214d5d7bee54fbf0cb1375c Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Thu, 27 Jun 2024 20:33:49 +0200 Subject: [PATCH 05/18] #380 - Replaced alt+o with ctrl+o for "open in background" --- src/components/modals.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/modals.ts b/src/components/modals.ts index e439866..a4ecd41 100644 --- a/src/components/modals.ts +++ b/src/components/modals.ts @@ -119,7 +119,7 @@ abstract class OmnisearchModal extends Modal { }) // Open in background - this.scope.register(['Alt'], 'O', e => { + this.scope.register(['Ctrl'], 'O', e => { if (!isInputComposition()) { // Check if the user is still typing e.preventDefault() From 87d2085fda7fcc23a624441696bd8acdc7e1a684 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Thu, 27 Jun 2024 21:00:18 +0200 Subject: [PATCH 06/18] #373 - Arabic diacritics --- src/search/query.ts | 4 ++-- src/search/search-engine.ts | 9 +++++++-- src/settings.ts | 14 ++++++++++++++ src/tools/text-processing.ts | 2 +- src/tools/utils.ts | 29 ++++++++++++++++++++++------- 5 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/search/query.ts b/src/search/query.ts index 2264832..f822d46 100644 --- a/src/search/query.ts +++ b/src/search/query.ts @@ -13,9 +13,9 @@ export class Query { } #inQuotes: string[] - constructor(text = '', options: { ignoreDiacritics: boolean }) { + constructor(text = '', options: { ignoreDiacritics: boolean, ignoreArabicDiacritics: boolean}) { if (options.ignoreDiacritics) { - text = removeDiacritics(text) + text = removeDiacritics(text, options.ignoreArabicDiacritics) } const parsed = parse(text.toLowerCase(), { tokenize: true, diff --git a/src/search/search-engine.ts b/src/search/search-engine.ts index dba9a80..90f495b 100644 --- a/src/search/search-engine.ts +++ b/src/search/search-engine.ts @@ -304,7 +304,12 @@ export class SearchEngine { const title = document?.path.toLowerCase() ?? '' const content = (document?.cleanedContent ?? '').toLowerCase() return exactTerms.every( - q => content.includes(q) || removeDiacritics(title).includes(q) + q => + content.includes(q) || + removeDiacritics( + title, + this.plugin.settings.ignoreArabicDiacritics + ).includes(q) ) }) } @@ -434,7 +439,7 @@ export class SearchEngine { }, processTerm: (term: string) => (this.plugin.settings.ignoreDiacritics - ? removeDiacritics(term) + ? removeDiacritics(term, this.plugin.settings.ignoreArabicDiacritics) : term ).toLowerCase(), idField: 'path', diff --git a/src/settings.ts b/src/settings.ts index a02503e..8f5fe31 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -32,6 +32,8 @@ export interface OmnisearchSettings extends WeightingSettings { downrankedFoldersFilters: string[] /** Ignore diacritics when indexing files */ ignoreDiacritics: boolean + ignoreArabicDiacritics: boolean + /** Extensions of plain text files to index, in addition to .md */ indexedFileTypes: string[] /** Enable PDF indexing */ @@ -661,6 +663,17 @@ export class SettingsTab extends PluginSettingTab { }) ) + new Setting(containerEl) + .setName('Ignore Arabic diacritics (beta)') + .setDesc(diacriticsDesc) + .addToggle(toggle => + toggle.setValue(settings.ignoreArabicDiacritics).onChange(async v => { + await database.clearCache() + settings.ignoreArabicDiacritics = v + await saveSettings(this.plugin) + }) + ) + // Disable Omnisearch const disableDesc = new DocumentFragment() disableDesc.createSpan({}, span => { @@ -720,6 +733,7 @@ export function getDefaultSettings(app: App): OmnisearchSettings { hideExcluded: false, downrankedFoldersFilters: [] as string[], ignoreDiacritics: true, + ignoreArabicDiacritics: false, indexedFileTypes: [] as string[], PDFIndexing: false, officeIndexing: false, diff --git a/src/tools/text-processing.ts b/src/tools/text-processing.ts index 64574f3..497c513 100644 --- a/src/tools/text-processing.ts +++ b/src/tools/text-processing.ts @@ -115,7 +115,7 @@ export class TextProcessor { const originalText = text // text = text.toLowerCase().replace(new RegExp(SEPARATORS, 'gu'), ' ') if (this.plugin.settings.ignoreDiacritics) { - text = removeDiacritics(text) + text = removeDiacritics(text, this.plugin.settings.ignoreArabicDiacritics) } const startTime = new Date().getTime() let match: RegExpExecArray | null = null diff --git a/src/tools/utils.ts b/src/tools/utils.ts index 1c32558..74d7beb 100644 --- a/src/tools/utils.ts +++ b/src/tools/utils.ts @@ -109,15 +109,30 @@ export function getTagsFromMetadata(metadata: CachedMetadata | null): string[] { /** * https://stackoverflow.com/a/37511463 */ -export function removeDiacritics(str: string): string { - // Japanese diacritics that should be distinguished - const excludeDiacritics: string[] = ['\\u30FC', '\\u309A', '\\u3099'] - const regexpExclude: string = excludeDiacritics.join('|') - const regexp: RegExp = new RegExp(`(?!${regexpExclude})\\p{Diacritic}`, 'gu') - +export function removeDiacritics(str: string, arabic = false): string { if (str === null || str === undefined) { return '' } + + // Japanese diacritics that should be distinguished + const japaneseDiacritics: string[] = ['\\u30FC', '\\u309A', '\\u3099'] + const regexpExclude: string = japaneseDiacritics.join('|') + const regexp: RegExp = new RegExp(`(?!${regexpExclude})\\p{Diacritic}`, 'gu') + + if (arabic) { + // Arabic diacritics + // https://stackoverflow.com/a/40959537 + str = str + .replace(/([^\u0621-\u063A\u0641-\u064A\u0660-\u0669a-zA-Z 0-9])/g, '') + .replace(/(آ|إ|أ)/g, 'ا') + .replace(/(ة)/g, 'ه') + .replace(/(ئ|ؤ)/g, 'ء') + .replace(/(ى)/g, 'ي') + for (let i = 0; i < 10; i++) { + str.replace(String.fromCharCode(0x660 + i), String.fromCharCode(48 + i)) + } + } + // Keep backticks for code blocks, because otherwise they are removed by the .normalize() function // https://stackoverflow.com/a/36100275 str = str.replaceAll('`', '[__omnisearch__backtick__]') @@ -223,7 +238,7 @@ export function warnDebug(...args: any[]): void { printDebug(console.warn, ...args) } -let printDebugEnabled= false +let printDebugEnabled = false export function enablePrintDebug(enable: boolean): void { printDebugEnabled = enable } From 3611884bc530b25670e09d2b6887a829210fc1f5 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Thu, 27 Jun 2024 21:25:06 +0200 Subject: [PATCH 07/18] #349 - Added a "danger" setting to disable the cache killswitch --- src/main.ts | 17 ++++++++++++----- src/settings.ts | 28 +++++++++++++++++++++++----- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/main.ts b/src/main.ts index 4817945..b68c195 100644 --- a/src/main.ts +++ b/src/main.ts @@ -137,7 +137,9 @@ export default class OmnisearchPlugin extends Plugin { }) ) - this.refreshIndexCallback = this.notesIndexer.refreshIndex.bind(this.notesIndexer) + this.refreshIndexCallback = this.notesIndexer.refreshIndex.bind( + this.notesIndexer + ) addEventListener('blur', this.refreshIndexCallback) removeEventListener @@ -263,15 +265,20 @@ export default class OmnisearchPlugin extends Plugin { indexingStep.set(IndexingStepType.WritingCache) // Disable settings.useCache while writing the cache, in case it freezes - this.settings.useCache = false - await saveSettings(this) + const cacheEnabled = this.settings.useCache + if (cacheEnabled && !this.settings.DANGER_forceSaveCache) { + this.settings.useCache = false + await saveSettings(this) + } // Write the cache await searchEngine.writeToCache() // Re-enable settings.caching - this.settings.useCache = true - await saveSettings(this) + if (cacheEnabled) { + this.settings.useCache = true + await saveSettings(this) + } } console.timeEnd('Omnisearch - Indexing total time') diff --git a/src/settings.ts b/src/settings.ts index 8f5fe31..303c5e9 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -11,7 +11,7 @@ import { import { writable } from 'svelte/store' import { K_DISABLE_OMNISEARCH } from './globals' import type OmnisearchPlugin from './main' -import { enablePrintDebug } from "./tools/utils"; +import { enablePrintDebug } from './tools/utils' interface WeightingSettings { weightBasename: number @@ -71,6 +71,7 @@ export interface OmnisearchSettings extends WeightingSettings { httpApiNotice: boolean DANGER_httpHost: string | null + DANGER_forceSaveCache: boolean } /** @@ -139,9 +140,7 @@ export class SettingsTab extends PluginSettingTab { span.innerHTML = `Omnisearch will use Text Extractor to index the content of your PDFs.` }) new Setting(containerEl) - .setName( - `PDFs content indexing ${textExtractor ? '' : '⚠️ Disabled'}` - ) + .setName(`PDFs content indexing ${textExtractor ? '' : '⚠️ Disabled'}`) .setDesc(indexPDFsDesc) .addToggle(toggle => toggle.setValue(settings.PDFIndexing).onChange(async v => { @@ -663,7 +662,7 @@ export class SettingsTab extends PluginSettingTab { }) ) - new Setting(containerEl) + new Setting(containerEl) .setName('Ignore Arabic diacritics (beta)') .setDesc(diacriticsDesc) .addToggle(toggle => @@ -695,6 +694,23 @@ export class SettingsTab extends PluginSettingTab { }) ) + // Force save cache + const forceSaveCacheDesc = new DocumentFragment() + forceSaveCacheDesc.createSpan({}, span => { + span.innerHTML = `Omnisearch has a security feature that automatically disables cache writing if it cannot fully perform the operation.
+ Use this option to force the cache to be saved, even if it causes a crash.
+ ⚠️ Enabling this setting could lead to crash loops` + }) + new Setting(containerEl) + .setName('Force save the cache') + .setDesc(forceSaveCacheDesc) + .addToggle(toggle => + toggle.setValue(settings.DANGER_forceSaveCache).onChange(async v => { + settings.DANGER_forceSaveCache = v + await saveSettings(this.plugin) + }) + ) + // Clear cache data if (isCacheEnabled()) { const resetCacheDesc = new DocumentFragment() @@ -713,6 +729,7 @@ export class SettingsTab extends PluginSettingTab { }) }) } + //#endregion Danger Zone } @@ -769,6 +786,7 @@ export function getDefaultSettings(app: App): OmnisearchSettings { verboseLogging: false, DANGER_httpHost: null, + DANGER_forceSaveCache: false, } } From da161b8ffe4e543f6f5c3acc6430f8337e77a0f3 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sat, 29 Jun 2024 16:04:30 +0200 Subject: [PATCH 08/18] chore: cleaned up html descriptions in settings --- src/settings.ts | 151 ++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 82 deletions(-) diff --git a/src/settings.ts b/src/settings.ts index 303c5e9..9143bff 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -117,31 +117,27 @@ export class SettingsTab extends PluginSettingTab { //#region Indexing - const indexingDesc = new DocumentFragment() - indexingDesc.createSpan({}, span => { - span.innerHTML = `⚠️ Changing indexing settings will clear the cache, and requires a restart of Obsidian.

` - if (textExtractor) { - span.innerHTML += ` - 👍 You have installed Text Extractor, Omnisearch can use it to index PDFs and images contents. -
Text extraction only works on desktop, but the cache can be synchronized with your mobile device.` - } else { - span.innerHTML += `⚠️ Omnisearch requires Text Extractor to index PDFs and images.` - } - }) - new Setting(containerEl) .setName('Indexing') .setHeading() - .setDesc(indexingDesc) + .setDesc( + htmlDescription(`⚠️ Changing indexing settings will clear the cache, and requires a restart of Obsidian.

+ ${ + textExtractor + ? `👍 You have installed Text Extractor, Omnisearch can use it to index PDFs and images contents. +
Text extraction only works on desktop, but the cache can be synchronized with your mobile device.` + : `⚠️ Omnisearch requires Text Extractor to index PDFs and images.` + }`) + ) // PDF Indexing - const indexPDFsDesc = new DocumentFragment() - indexPDFsDesc.createSpan({}, span => { - span.innerHTML = `Omnisearch will use Text Extractor to index the content of your PDFs.` - }) new Setting(containerEl) .setName(`PDFs content indexing ${textExtractor ? '' : '⚠️ Disabled'}`) - .setDesc(indexPDFsDesc) + .setDesc( + htmlDescription( + `Omnisearch will use Text Extractor to index the content of your PDFs.` + ) + ) .addToggle(toggle => toggle.setValue(settings.PDFIndexing).onChange(async v => { await database.clearCache() @@ -152,13 +148,13 @@ export class SettingsTab extends PluginSettingTab { .setDisabled(!textExtractor) // Images Indexing - const indexImagesDesc = new DocumentFragment() - indexImagesDesc.createSpan({}, span => { - span.innerHTML = `Omnisearch will use Text Extractor to OCR your images and index their content.` - }) new Setting(containerEl) .setName(`Images OCR indexing ${textExtractor ? '' : '⚠️ Disabled'}`) - .setDesc(indexImagesDesc) + .setDesc( + htmlDescription( + `Omnisearch will use Text Extractor to OCR your images and index their content.` + ) + ) .addToggle(toggle => toggle.setValue(settings.imagesIndexing).onChange(async v => { await database.clearCache() @@ -188,16 +184,14 @@ export class SettingsTab extends PluginSettingTab { .setDisabled(!textExtractor) // Index filenames of unsupported files - const indexUnsupportedDesc = new DocumentFragment() - indexUnsupportedDesc.createSpan({}, span => { - span.innerHTML = ` - Omnisearch can index filenames of "unsupported" files, such as e.g. 
.mp4
- or non-extracted PDFs & images.
- "Obsidian setting" will respect the value of "Files & Links > Detect all file extensions".` - }) new Setting(containerEl) .setName('Index paths of unsupported files') - .setDesc(indexUnsupportedDesc) + .setDesc( + htmlDescription(` + Omnisearch can index filenames of "unsupported" files, such as e.g. 
.mp4
+ or non-extracted PDFs & images.
+ "Obsidian setting" will respect the value of "Files & Links > Detect all file extensions".`) + ) .addDropdown(dropdown => { dropdown .addOptions({ yes: 'Yes', no: 'No', default: 'Obsidian setting' }) @@ -210,16 +204,14 @@ export class SettingsTab extends PluginSettingTab { }) // Additional text files to index - const indexedFileTypesDesc = new DocumentFragment() - indexedFileTypesDesc.createSpan({}, span => { - span.innerHTML = `In addition to standard md files, Omnisearch can also index other PLAINTEXT files.
- Add extensions separated by a space, without the dot. Example: "txt org csv".
- ⚠️ Using extensions of non-plaintext files (like .pptx) WILL cause crashes, - because Omnisearch will try to index their content.` - }) new Setting(containerEl) .setName('Additional TEXT files to index') - .setDesc(indexedFileTypesDesc) + .setDesc( + htmlDescription(`In addition to standard md files, Omnisearch can also index other PLAINTEXT files.
+ Add extensions separated by a space, without the dot. Example: "txt org csv".
+ ⚠️ Using extensions of non-plaintext files (like .pptx) WILL cause crashes, + because Omnisearch will try to index their content.`) + ) .addText(component => { component .setValue(settings.indexedFileTypes.join(' ')) @@ -294,16 +286,13 @@ export class SettingsTab extends PluginSettingTab { }) // Split CamelCaseWords - const camelCaseDesc = new DocumentFragment() - camelCaseDesc.createSpan({}, span => { - span.innerHTML = `Enable this if you want to be able to search for CamelCaseWords as separate words.
- ⚠️ Changing this setting will clear the cache.
- ${needsARestart} - ` - }) new Setting(containerEl) .setName('Split CamelCaseWords') - .setDesc(camelCaseDesc) + .setDesc( + htmlDescription(`Enable this if you want to be able to search for CamelCaseWords as separate words.
+ ⚠️ Changing this setting will clear the cache.
+ ${needsARestart}`) + ) .addToggle(toggle => toggle.setValue(settings.splitCamelCase).onChange(async v => { await database.clearCache() @@ -444,14 +433,12 @@ export class SettingsTab extends PluginSettingTab { ) // Show "Create note" button - const createBtnDesc = new DocumentFragment() - createBtnDesc.createSpan({}, span => { - span.innerHTML = `Shows a button next to the search input, to create a note. - Acts the same as the shift ↵ shortcut, can be useful for mobile device users.` - }) new Setting(containerEl) .setName('Show "Create note" button') - .setDesc(createBtnDesc) + .setDesc( + htmlDescription(`Shows a button next to the search input, to create a note. + Acts the same as the shift ↵ shortcut, can be useful for mobile device users.`) + ) .addToggle(toggle => toggle.setValue(settings.showCreateButton).onChange(async v => { settings.showCreateButton = v @@ -564,14 +551,14 @@ export class SettingsTab extends PluginSettingTab { //#region HTTP Server if (!Platform.isMobile) { - const httpServerDesc = new DocumentFragment() - httpServerDesc.createSpan({}, span => { - span.innerHTML = `Omnisearch can be used through a simple HTTP server (more information).` - }) new Setting(containerEl) .setName('API Access Through HTTP') .setHeading() - .setDesc(httpServerDesc) + .setDesc( + htmlDescription( + `Omnisearch can be used through a simple HTTP server (more information).` + ) + ) new Setting(containerEl) .setName('Enable the HTTP server') @@ -643,17 +630,14 @@ export class SettingsTab extends PluginSettingTab { new Setting(containerEl).setName('Danger Zone').setHeading() // Ignore diacritics - const diacriticsDesc = new DocumentFragment() - diacriticsDesc.createSpan({}, span => { - span.innerHTML = `Normalize diacritics in search terms. Words like "brûlée" or "žluťoučký" will be indexed as "brulee" and "zlutoucky".
- ⚠️ You probably should NOT disable this.
- ⚠️ Changing this setting will clear the cache.
- ${needsARestart} - ` - }) new Setting(containerEl) .setName('Ignore diacritics') - .setDesc(diacriticsDesc) + .setDesc( + htmlDescription(`Normalize diacritics in search terms. Words like "brûlée" or "žluťoučký" will be indexed as "brulee" and "zlutoucky".
+ ⚠️ You probably should NOT disable this.
+ ⚠️ Changing this setting will clear the cache.
+ ${needsARestart}`) + ) .addToggle(toggle => toggle.setValue(settings.ignoreDiacritics).onChange(async v => { await database.clearCache() @@ -664,7 +648,6 @@ export class SettingsTab extends PluginSettingTab { new Setting(containerEl) .setName('Ignore Arabic diacritics (beta)') - .setDesc(diacriticsDesc) .addToggle(toggle => toggle.setValue(settings.ignoreArabicDiacritics).onChange(async v => { await database.clearCache() @@ -695,15 +678,13 @@ export class SettingsTab extends PluginSettingTab { ) // Force save cache - const forceSaveCacheDesc = new DocumentFragment() - forceSaveCacheDesc.createSpan({}, span => { - span.innerHTML = `Omnisearch has a security feature that automatically disables cache writing if it cannot fully perform the operation.
- Use this option to force the cache to be saved, even if it causes a crash.
- ⚠️ Enabling this setting could lead to crash loops` - }) new Setting(containerEl) .setName('Force save the cache') - .setDesc(forceSaveCacheDesc) + .setDesc( + htmlDescription(`Omnisearch has a security feature that automatically disables cache writing if it cannot fully perform the operation.
+ Use this option to force the cache to be saved, even if it causes a crash.
+ ⚠️ Enabling this setting could lead to crash loops`) + ) .addToggle(toggle => toggle.setValue(settings.DANGER_forceSaveCache).onChange(async v => { settings.DANGER_forceSaveCache = v @@ -713,15 +694,13 @@ export class SettingsTab extends PluginSettingTab { // Clear cache data if (isCacheEnabled()) { - const resetCacheDesc = new DocumentFragment() - resetCacheDesc.createSpan({}, span => { - span.innerHTML = `Erase all Omnisearch cache data. - Use this if Omnisearch results are inconsistent, missing, or appear outdated.
- ${needsARestart}` - }) new Setting(containerEl) .setName('Clear cache data') - .setDesc(resetCacheDesc) + .setDesc( + htmlDescription(`Erase all Omnisearch cache data. + Use this if Omnisearch results are inconsistent, missing, or appear outdated.
+ ${needsARestart}`) + ) .addButton(btn => { btn.setButtonText('Clear cache') btn.onClick(async () => { @@ -792,6 +771,14 @@ export function getDefaultSettings(app: App): OmnisearchSettings { let settings: OmnisearchSettings +function htmlDescription(innerHTML: string): DocumentFragment { + const desc = new DocumentFragment() + desc.createSpan({}, span => { + span.innerHTML = innerHTML + }) + return desc +} + // /** // * @deprecated // */ From 1d64ca52cc5401e278c1493a8a5601477bf60700 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sat, 29 Jun 2024 16:14:02 +0200 Subject: [PATCH 09/18] fix: using a debounced function to clear the cache when triggered by a text field --- src/settings.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/settings.ts b/src/settings.ts index 9143bff..a558c64 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -12,6 +12,7 @@ import { writable } from 'svelte/store' import { K_DISABLE_OMNISEARCH } from './globals' import type OmnisearchPlugin from './main' import { enablePrintDebug } from './tools/utils' +import { debounce } from 'lodash-es' interface WeightingSettings { weightBasename: number @@ -98,6 +99,11 @@ export class SettingsTab extends PluginSettingTab { const { containerEl } = this const database = this.plugin.database const textExtractor = this.plugin.getTextExtractor() + + const clearCacheDebounced = debounce(async () => { + await database.clearCache() + }, 1000) + containerEl.empty() if (this.app.loadLocalStorage(K_DISABLE_OMNISEARCH) == '1') { @@ -197,7 +203,7 @@ export class SettingsTab extends PluginSettingTab { .addOptions({ yes: 'Yes', no: 'No', default: 'Obsidian setting' }) .setValue(settings.unsupportedFilesIndexing) .onChange(async v => { - await database.clearCache() + await clearCacheDebounced() ;(settings.unsupportedFilesIndexing as any) = v await saveSettings(this.plugin) }) From 94d687aeaaeb56eb2f9b5e9f7857b7d7512f42c1 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sat, 29 Jun 2024 16:51:16 +0200 Subject: [PATCH 10/18] fix: missing the new arabic diacritics settings in a few places (#373) --- src/__tests__/query-tests.ts | 15 ++++++++++++--- src/components/ModalVault.svelte | 1 + src/tools/api.ts | 1 + 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/__tests__/query-tests.ts b/src/__tests__/query-tests.ts index 8ef016e..b2a5b42 100644 --- a/src/__tests__/query-tests.ts +++ b/src/__tests__/query-tests.ts @@ -6,7 +6,10 @@ describe('The Query class', () => { it('should correctly parse string queries', () => { // Act - const query = new Query(stringQuery, { ignoreDiacritics: true }) + const query = new Query(stringQuery, { + ignoreDiacritics: true, + ignoreArabicDiacritics: true, + }) // Assert const segments = query.query.text @@ -25,7 +28,10 @@ describe('The Query class', () => { it('should not exclude words when there is no space before', () => { // Act - const query = new Query('foo bar-baz', { ignoreDiacritics: true }) + const query = new Query('foo bar-baz', { + ignoreDiacritics: true, + ignoreArabicDiacritics: true, + }) // Assert expect(query.query.exclude.text).toHaveLength(0) @@ -34,7 +40,10 @@ describe('The Query class', () => { describe('.getExactTerms()', () => { it('should an array of strings containg "exact" values', () => { // Act - const query = new Query(stringQuery, { ignoreDiacritics: true }) + const query = new Query(stringQuery, { + ignoreDiacritics: true, + ignoreArabicDiacritics: true, + }) // Assert expect(query.getExactTerms()).toEqual(['lorem ipsum', 'sit amet']) diff --git a/src/components/ModalVault.svelte b/src/components/ModalVault.svelte index 8f17a75..e992d99 100644 --- a/src/components/ModalVault.svelte +++ b/src/components/ModalVault.svelte @@ -141,6 +141,7 @@ } query = new Query(searchQuery, { ignoreDiacritics: plugin.settings.ignoreDiacritics, + ignoreArabicDiacritics: plugin.settings.ignoreArabicDiacritics, }) cancelableQuery = cancelable( new Promise(resolve => { diff --git a/src/tools/api.ts b/src/tools/api.ts index 7761c8a..5a480e6 100644 --- a/src/tools/api.ts +++ b/src/tools/api.ts @@ -88,6 +88,7 @@ export function getApi(plugin: OmnisearchPlugin) { async search(q: string): Promise { const query = new Query(q, { ignoreDiacritics: plugin.settings.ignoreDiacritics, + ignoreArabicDiacritics: plugin.settings.ignoreArabicDiacritics, }) const raw = await plugin.searchEngine.getSuggestions(query) return mapResults(plugin, raw) From 38c964bec2208c223e65b5566025abfaf55a91f6 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sat, 29 Jun 2024 17:02:11 +0200 Subject: [PATCH 11/18] feat: Make any property of frontmatter a display name (#329) --- src/cache-manager.ts | 3 ++- src/components/ResultItemVault.svelte | 2 +- src/globals.ts | 2 ++ src/notes-indexer.ts | 1 + src/search/search-engine.ts | 3 ++- src/settings.ts | 18 ++++++++++++++++++ 6 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/cache-manager.ts b/src/cache-manager.ts index 8b94e9e..fc2841a 100644 --- a/src/cache-manager.ts +++ b/src/cache-manager.ts @@ -221,10 +221,11 @@ export class CacheManager { } } } - + const displayTitle = metadata?.frontmatter?.[this.plugin.settings.displayTitle] ?? '' const tags = getTagsFromMetadata(metadata) return { basename: file.basename, + displayTitle, content, /** Content without diacritics and markdown chars */ cleanedContent: stripMarkdownCharacters(removeDiacritics(content)), diff --git a/src/components/ResultItemVault.svelte b/src/components/ResultItemVault.svelte index 22f9ea6..a0290df 100644 --- a/src/components/ResultItemVault.svelte +++ b/src/components/ResultItemVault.svelte @@ -36,7 +36,7 @@ $: cleanedContent = plugin.textProcessor.makeExcerpt(note.content, note.matches[0]?.offset ?? -1) $: glyph = false //cacheManager.getLiveDocument(note.path)?.doesNotExist $: { - title = note.basename + title = note.displayTitle || note.basename notePath = pathWithoutFilename(note.path) // Icons diff --git a/src/globals.ts b/src/globals.ts index 716fbe2..cfad186 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -46,6 +46,7 @@ export type DocumentRef = { path: string; mtime: number } export type IndexedDocument = { path: string basename: string + displayTitle: string mtime: number content: string @@ -76,6 +77,7 @@ export type ResultNote = { score: number path: string basename: string + displayTitle: string content: string foundWords: string[] matches: SearchMatch[] diff --git a/src/notes-indexer.ts b/src/notes-indexer.ts index c5e56f6..431157f 100644 --- a/src/notes-indexer.ts +++ b/src/notes-indexer.ts @@ -88,6 +88,7 @@ export class NotesIndexer { return { path: filename, basename: name, + displayTitle: '', mtime: 0, content: '', diff --git a/src/search/search-engine.ts b/src/search/search-engine.ts index 90f495b..bfaae53 100644 --- a/src/search/search-engine.ts +++ b/src/search/search-engine.ts @@ -154,8 +154,9 @@ export class SearchEngine { term.length <= 3 ? 0 : term.length <= 5 ? fuzziness / 2 : fuzziness, boost: { basename: settings.weightBasename, - directory: settings.weightDirectory, aliases: settings.weightBasename, + displayTitle: settings.weightBasename, + directory: settings.weightDirectory, headings1: settings.weightH1, headings2: settings.weightH2, headings3: settings.weightH3, diff --git a/src/settings.ts b/src/settings.ts index a558c64..c493a88 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -37,6 +37,8 @@ export interface OmnisearchSettings extends WeightingSettings { /** Extensions of plain text files to index, in addition to .md */ indexedFileTypes: string[] + /** Custom title field */ + displayTitle: string /** Enable PDF indexing */ PDFIndexing: boolean /** Enable Images indexing */ @@ -209,6 +211,21 @@ export class SettingsTab extends PluginSettingTab { }) }) + // Custom display title + new Setting(containerEl) + .setName('Set frontmatter property key as title') + .setDesc( + htmlDescription(`If you have a custom property in your notes that you want to use as the title in search results.
+ Leave empty to disable.`) + ) + .addText(component => { + component.setValue(settings.displayTitle).onChange(async v => { + await clearCacheDebounced() + settings.displayTitle = v + await saveSettings(this.plugin) + }) + }) + // Additional text files to index new Setting(containerEl) .setName('Additional TEXT files to index') @@ -737,6 +754,7 @@ export function getDefaultSettings(app: App): OmnisearchSettings { ignoreDiacritics: true, ignoreArabicDiacritics: false, indexedFileTypes: [] as string[], + displayTitle: '', PDFIndexing: false, officeIndexing: false, imagesIndexing: false, From a7a24155c32606875ae5d9e5f0cc289c2e8b8c3b Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sat, 29 Jun 2024 20:15:57 +0200 Subject: [PATCH 12/18] chore(repository): updated release.yml and trying git-cliff --- .github/workflows/release.yml | 21 +++++++++++++++------ cliff.toml | 12 ++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 cliff.toml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77cefeb..fec51be 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,17 +14,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2.1.0 + - uses: pnpm/action-setup@v4 with: - version: 7.17.0 + version: 9.3.0 run_install: true - name: Use Node.js - uses: actions/setup-node@v1 + uses: actions/setup-node@v4 with: - node-version: "18.x" + cache: 'pnpm' + node-version: "20.x" - name: Build id: build @@ -37,6 +38,13 @@ jobs: ls echo "::set-output name=tag_name::$(git tag --sort version:refname | tail -n 1)" + - name: Generate a changelog + uses: orhun/git-cliff-action@v3 + id: git-cliff + with: + config: cliff.toml + args: --verbose + - name: Create Release id: create_release uses: actions/create-release@v1 @@ -46,7 +54,8 @@ jobs: with: tag_name: ${{ github.ref }} release_name: ${{ github.ref }} - draft: false + body: ${{ steps.git-cliff.outputs.changelog }} + draft: true prerelease: false - name: Upload zip file diff --git a/cliff.toml b/cliff.toml new file mode 100644 index 0000000..8a8ce7c --- /dev/null +++ b/cliff.toml @@ -0,0 +1,12 @@ +[changelog] +header = "Changelog" +body = """ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | upper_first }} + {% for commit in commits %} + - {{ commit.message | upper_first }} + {% endfor %} +{% endfor %} +""" +trim = true +footer = "" From 8e550efa4e60879ea20603ed179aae5ce26b3962 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sat, 29 Jun 2024 20:32:37 +0200 Subject: [PATCH 13/18] 1.24.0-beta.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 89cd06b..ef31874 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "scambier.obsidian-search", - "version": "1.24.0-beta.1", + "version": "1.24.0-beta.2", "description": "A search engine for Obsidian", "main": "dist/main.js", "scripts": { From 3ad15ea847934f1d2584411a70a58041695ae366 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sat, 29 Jun 2024 20:36:12 +0200 Subject: [PATCH 14/18] Removed useless field --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index ef31874..349910a 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,5 @@ "overrides": { "moment@>=2.18.0 <2.29.4": ">=2.29.4" } - }, - "packageManager": "pnpm@9.1.0+sha512.67f5879916a9293e5cf059c23853d571beaf4f753c707f40cb22bed5fb1578c6aad3b6c4107ccb3ba0b35be003eb621a16471ac836c87beb53f9d54bb4612724" + } } From e79ffe873ab432ec90b8337557d207a4e62dbd22 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sat, 29 Jun 2024 21:16:11 +0200 Subject: [PATCH 15/18] release.yml --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fec51be..471d9f7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: pnpm/action-setup@v4 with: From 6e62df2c3692de2f8a4de2ef0bcbcc2e6f7e331a Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sat, 29 Jun 2024 21:21:17 +0200 Subject: [PATCH 16/18] release.yml --- .github/workflows/release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 471d9f7..974b243 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,6 +46,11 @@ jobs: with: config: cliff.toml args: --verbose + env: + GITHUB_REPO: ${{ github.repository }} + + - name: Print the changelog + run: cat "${{ steps.git-cliff.outputs.changelog }}" - name: Create Release id: create_release From a778937292df1c468715283b636ce363a7cfac9a Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Tue, 23 Jul 2024 09:58:10 +0200 Subject: [PATCH 17/18] 1.24.0-beta.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 349910a..436e24d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "scambier.obsidian-search", - "version": "1.24.0-beta.2", + "version": "1.24.0-beta.3", "description": "A search engine for Obsidian", "main": "dist/main.js", "scripts": { From 439150a1f0ac5532435c392533f9a8472606094b Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Wed, 31 Jul 2024 11:17:35 +0200 Subject: [PATCH 18/18] fix: tentative workaround for #383 --- src/search/tokenizer.ts | 61 ++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/src/search/tokenizer.ts b/src/search/tokenizer.ts index 9ede21a..a155025 100644 --- a/src/search/tokenizer.ts +++ b/src/search/tokenizer.ts @@ -15,36 +15,41 @@ export class Tokenizer { * @returns */ public tokenizeForIndexing(text: string): string[] { - const words = this.tokenizeWords(text) - let urls: string[] = [] - if (this.plugin.settings.tokenizeUrls) { - try { - urls = markdownLinkExtractor(text) - } catch (e) { - logDebug('Error extracting urls', e) + try { + const words = this.tokenizeWords(text) + let urls: string[] = [] + if (this.plugin.settings.tokenizeUrls) { + try { + urls = markdownLinkExtractor(text) + } catch (e) { + logDebug('Error extracting urls', e) + } } + + let tokens = this.tokenizeTokens(text, { skipChs: true }) + + // Split hyphenated tokens + tokens = [...tokens, ...tokens.flatMap(splitHyphens)] + + // Split camelCase tokens into "camel" and "case + tokens = [...tokens, ...tokens.flatMap(splitCamelCase)] + + // Add whole words (aka "not tokens") + tokens = [...tokens, ...words] + + // Add urls + if (urls.length) { + tokens = [...tokens, ...urls] + } + + // Remove duplicates + tokens = [...new Set(tokens)] + + return tokens + } catch (e) { + console.error('Error tokenizing text, skipping document', e) + return [] } - - let tokens = this.tokenizeTokens(text, { skipChs: true }) - - // Split hyphenated tokens - tokens = [...tokens, ...tokens.flatMap(splitHyphens)] - - // Split camelCase tokens into "camel" and "case - tokens = [...tokens, ...tokens.flatMap(splitCamelCase)] - - // Add whole words (aka "not tokens") - tokens = [...tokens, ...words] - - // Add urls - if (urls.length) { - tokens = [...tokens, ...urls] - } - - // Remove duplicates - tokens = [...new Set(tokens)] - - return tokens } /**