From 3c32e5f70fc0be278467eb093d126157d052458e Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Wed, 18 Jan 2023 22:17:26 +0100 Subject: [PATCH 01/11] #182 - Added back an opt-out for the cache, now automatically disabled in case of crash --- src/globals.ts | 10 ++- src/main.ts | 167 ++++++++++++++++++++++----------------- src/search/omnisearch.ts | 3 - src/settings.ts | 20 ++++- 4 files changed, 120 insertions(+), 80 deletions(-) diff --git a/src/globals.ts b/src/globals.ts index 3ea1ce0..29c7013 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -2,6 +2,7 @@ import { EventBus } from './tools/event-bus' import { writable } from 'svelte/store' import { settings } from './settings' import type { TFile } from 'obsidian' +import { Platform } from 'obsidian' export const regexLineSplit = /\r?\n|\r|((\.|\?|!)( |\r?\n|\r))/g export const regexYaml = /^---\s*\n(.*?)\n?^---\s?/ms @@ -77,10 +78,9 @@ export function isInputComposition(): boolean { return inComposition } - /** * Plugin dependency - Chs Patch for Chinese word segmentation - * @returns + * @returns */ export function getChsSegmenter(): any | undefined { return (app as any).plugins.plugins['cm-chs-patch'] @@ -93,11 +93,15 @@ export type TextExtractorApi = { /** * Plugin dependency - Text Extractor - * @returns + * @returns */ export function getTextExtractor(): TextExtractorApi | undefined { return (app as any).plugins?.plugins?.['text-extractor']?.api } +export function isCacheEnabled(): boolean { + return !Platform.isIosApp && settings.useCache +} + export const SPACE_OR_PUNCTUATION = /[|\n\r -#%-*,-/:;?@[-\]_{}\u00A0\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u09FD\u0A76\u0AF0\u0C77\u0C84\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166E\u1680\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2000-\u200A\u2010-\u2029\u202F-\u2043\u2045-\u2051\u2053-\u205F\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E4F\u3000-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]+/u diff --git a/src/main.ts b/src/main.ts index 28f10ae..7469075 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,10 +1,22 @@ -import { Notice, Platform, Plugin } from 'obsidian' +import { Notice, Plugin } from 'obsidian' import { OmnisearchInFileModal, OmnisearchVaultModal, } from './components/modals' -import { loadSettings, settings, SettingsTab, showExcerpt } from './settings' -import { eventBus, EventNames, indexingStep, IndexingStepType } from './globals' +import { + loadSettings, + saveSettings, + settings, + SettingsTab, + showExcerpt, +} from './settings' +import { + eventBus, + EventNames, + indexingStep, + IndexingStepType, + isCacheEnabled, +} from './globals' import api from './tools/api' import { isFileIndexable } from './tools/utils' import { database, OmnisearchCache } from './database' @@ -87,10 +99,24 @@ export default class OmnisearchPlugin extends Plugin { }) ) - await populateIndex() + await this.populateIndex() }) - executeFirstLaunchTasks(this) + this.executeFirstLaunchTasks() + } + + executeFirstLaunchTasks(): void { + const code = '1.10.0-beta.1' + if (settings.welcomeMessage !== code) { + const welcome = new DocumentFragment() + welcome.createSpan({}, span => { + span.innerHTML = `๐Ÿ”Ž Omnisearch will soon require the Text Extractor plugin to index PDF and images. See Omnisearch settings for more information.` + }) + new Notice(welcome, 20_000) + } + settings.welcomeMessage = code + + this.saveData(settings) } async onunload(): Promise { @@ -114,64 +140,75 @@ export default class OmnisearchPlugin extends Plugin { this.ribbonButton.parentNode?.removeChild(this.ribbonButton) } } + + private async populateIndex(): Promise { + console.time('Omnisearch - Indexing total time') + indexingStep.set(IndexingStepType.ReadingFiles) + const files = app.vault.getFiles().filter(f => isFileIndexable(f.path)) + console.log(`Omnisearch - ${files.length} files total`) + + // Map documents in the background + // Promise.all(files.map(f => cacheManager.addToLiveCache(f.path))) + + if (isCacheEnabled()) { + console.time('Omnisearch - Loading index from cache') + indexingStep.set(IndexingStepType.LoadingCache) + await searchEngine.loadCache() + console.timeEnd('Omnisearch - Loading index from cache') + } + + const diff = searchEngine.getDiff( + files.map(f => ({ path: f.path, mtime: f.stat.mtime })) + ) + + if (diff.toAdd.length) { + console.log( + 'Omnisearch - Total number of files to add/update: ' + diff.toAdd.length + ) + } + if (diff.toRemove.length) { + console.log( + 'Omnisearch - Total number of files to remove: ' + diff.toRemove.length + ) + } + + if (diff.toAdd.length >= 1000 && isCacheEnabled()) { + new Notice( + `Omnisearch - ${diff.toAdd.length} files need to be indexed. Obsidian may experience stutters and freezes during the process`, + 10_000 + ) + } + + indexingStep.set(IndexingStepType.IndexingFiles) + searchEngine.removeFromPaths(diff.toRemove.map(o => o.path)) + await searchEngine.addFromPaths(diff.toAdd.map(o => o.path)) + + if ((diff.toRemove.length || diff.toAdd.length) && isCacheEnabled()) { + indexingStep.set(IndexingStepType.WritingCache) + + // Disable settings.useCache while writing the cache, in case it freezes + settings.useCache = false + saveSettings(this) + + // Write the cache + await searchEngine.writeToCache() + + // Re-enable settings.caching + settings.useCache = true + saveSettings(this) + } + + console.timeEnd('Omnisearch - Indexing total time') + if (diff.toAdd.length >= 1000) { + new Notice(`Omnisearch - Your files have been indexed.`) + } + indexingStep.set(IndexingStepType.Done) + } } /** * Read the files and feed them to Minisearch */ -async function populateIndex(): Promise { - console.time('Omnisearch - Indexing total time') - indexingStep.set(IndexingStepType.ReadingFiles) - const files = app.vault.getFiles().filter(f => isFileIndexable(f.path)) - console.log(`Omnisearch - ${files.length} files total`) - - // Map documents in the background - // Promise.all(files.map(f => cacheManager.addToLiveCache(f.path))) - - if (!Platform.isIosApp) { - console.time('Omnisearch - Loading index from cache') - indexingStep.set(IndexingStepType.LoadingCache) - await searchEngine.loadCache() - console.timeEnd('Omnisearch - Loading index from cache') - } - - const diff = searchEngine.getDiff( - files.map(f => ({ path: f.path, mtime: f.stat.mtime })) - ) - - if (diff.toAdd.length) { - console.log( - 'Omnisearch - Total number of files to add/update: ' + diff.toAdd.length - ) - } - if (diff.toRemove.length) { - console.log( - 'Omnisearch - Total number of files to remove: ' + diff.toRemove.length - ) - } - - if (diff.toAdd.length >= 1000 && !Platform.isIosApp) { - new Notice( - `Omnisearch - ${diff.toAdd.length} files need to be indexed. Obsidian may experience stutters and freezes during the process`, - 10_000 - ) - } - - indexingStep.set(IndexingStepType.IndexingFiles) - searchEngine.removeFromPaths(diff.toRemove.map(o => o.path)) - await searchEngine.addFromPaths(diff.toAdd.map(o => o.path)) - - if (diff.toRemove.length || diff.toAdd.length) { - indexingStep.set(IndexingStepType.WritingCache) - await searchEngine.writeToCache() - } - - console.timeEnd('Omnisearch - Indexing total time') - if (diff.toAdd.length >= 1000) { - new Notice(`Omnisearch - Your files have been indexed.`) - } - indexingStep.set(IndexingStepType.Done) -} async function cleanOldCacheFiles() { const toDelete = [ @@ -191,20 +228,6 @@ async function cleanOldCacheFiles() { } } -function executeFirstLaunchTasks(plugin: Plugin) { - const code = '1.10.0-beta.1' - if (settings.welcomeMessage !== code) { - const welcome = new DocumentFragment() - welcome.createSpan({}, span => { - span.innerHTML = `๐Ÿ”Ž Omnisearch will soon require the Text Extractor plugin to index PDF and images. See Omnisearch settings for more information.` - }) - new Notice(welcome, 20_000) - } - settings.welcomeMessage = code - - plugin.saveData(settings) -} - function registerAPI(plugin: OmnisearchPlugin): void { // Url scheme for obsidian://omnisearch?query=foobar plugin.registerObsidianProtocolHandler('omnisearch', params => { diff --git a/src/search/omnisearch.ts b/src/search/omnisearch.ts index 6265bdd..96cdd1a 100644 --- a/src/search/omnisearch.ts +++ b/src/search/omnisearch.ts @@ -348,9 +348,6 @@ export class Omnisearch { } public async writeToCache(): Promise { - if (Platform.isIosApp) { - return - } await cacheManager.writeMinisearchCache( this.minisearch, this.indexedDocuments diff --git a/src/settings.ts b/src/settings.ts index 777a6b6..e059de7 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -8,7 +8,7 @@ import { } from 'obsidian' import { writable } from 'svelte/store' import { database } from './database' -import { getTextExtractor } from './globals' +import { getTextExtractor, isCacheEnabled } from './globals' import type OmnisearchPlugin from './main' interface WeightingSettings { @@ -19,6 +19,8 @@ interface WeightingSettings { } export interface OmnisearchSettings extends WeightingSettings { + /** Enables caching to speed up indexing */ + useCache: boolean /** Respect the "excluded files" Obsidian setting by downranking results ignored files */ hideExcluded: boolean /** Ignore diacritics when indexing files */ @@ -160,6 +162,19 @@ export class SettingsTab extends PluginSettingTab { new Setting(containerEl).setName('Behavior').setHeading() + // Caching + new Setting(containerEl) + .setName('Save index to cache') + .setDesc( + 'Enable caching to speed up indexing time. In rare cases, the cache write may cause a freeze in Obsidian. This option will disable itself if it happens.' + ) + .addToggle(toggle => + toggle.setValue(settings.useCache).onChange(async v => { + settings.useCache = v + await saveSettings(this.plugin) + }) + ) + // Respect excluded files new Setting(containerEl) .setName('Respect Obsidian\'s "Excluded Files"') @@ -336,7 +351,7 @@ export class SettingsTab extends PluginSettingTab { //#endregion Results Weighting //#region Danger Zone - if (!Platform.isIosApp) { + if (isCacheEnabled()) { new Setting(containerEl).setName('Danger Zone').setHeading() const resetCacheDesc = new DocumentFragment() @@ -371,6 +386,7 @@ export class SettingsTab extends PluginSettingTab { } export const DEFAULT_SETTINGS: OmnisearchSettings = { + useCache: true, hideExcluded: false, ignoreDiacritics: true, indexedFileTypes: [] as string[], From c7d255d2778059f6c408965bf00bfeffd4b74d54 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Wed, 18 Jan 2023 22:25:20 +0100 Subject: [PATCH 02/11] Removed Omnisearch's own text extractor --- package.json | 1 - pnpm-lock.yaml | 181 +++++++++---------------------------------- src/cache-manager.ts | 16 +--- 3 files changed, 41 insertions(+), 157 deletions(-) diff --git a/package.json b/package.json index e6afa24..1533e69 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,6 @@ "dexie": "^3.2.2", "lodash-es": "4.17.21", "minisearch": "6.0.0-beta.1", - "obsidian-text-extract": "^1.0.4", "pure-md5": "^0.1.14" }, "pnpm": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0acb019..960560d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,7 +22,6 @@ specifiers: lodash-es: 4.17.21 minisearch: 6.0.0-beta.1 obsidian: latest - obsidian-text-extract: ^1.0.4 prettier: ^2.8.1 prettier-plugin-svelte: ^2.8.1 pure-md5: ^0.1.14 @@ -38,7 +37,6 @@ dependencies: dexie: 3.2.2 lodash-es: 4.17.21 minisearch: 6.0.0-beta.1 - obsidian-text-extract: 1.0.4 pure-md5: 0.1.14 devDependencies: @@ -86,6 +84,7 @@ packages: engines: {node: '>=6.9.0'} dependencies: '@babel/highlight': 7.18.6 + dev: true /@babel/compat-data/7.20.5: resolution: {integrity: sha512-KZXo2t10+/jxmkhNXc7pZTqRvSOIvVv/+lJwHS+B2rErwOyjuVRh60yVpb7liQ1U5t7lLJ1bz+t8tSypUZdm0g==} @@ -122,6 +121,7 @@ packages: '@babel/types': 7.20.5 '@jridgewell/gen-mapping': 0.3.2 jsesc: 2.5.2 + dev: true /@babel/helper-annotate-as-pure/7.18.6: resolution: {integrity: sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==} @@ -208,6 +208,7 @@ packages: /@babel/helper-environment-visitor/7.18.9: resolution: {integrity: sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg==} engines: {node: '>=6.9.0'} + dev: true /@babel/helper-explode-assignable-expression/7.18.6: resolution: {integrity: sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==} @@ -222,12 +223,14 @@ packages: dependencies: '@babel/template': 7.18.10 '@babel/types': 7.20.5 + dev: true /@babel/helper-hoist-variables/7.18.6: resolution: {integrity: sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==} engines: {node: '>=6.9.0'} dependencies: '@babel/types': 7.20.5 + dev: true /@babel/helper-member-expression-to-functions/7.18.9: resolution: {integrity: sha512-RxifAh2ZoVU67PyKIO4AMi1wTenGfMR/O/ae0CCRqwgBAt5v7xjdtRw7UoSbsreKrQn5t7r89eruK/9JjYHuDg==} @@ -317,14 +320,17 @@ packages: engines: {node: '>=6.9.0'} dependencies: '@babel/types': 7.20.5 + dev: true /@babel/helper-string-parser/7.19.4: resolution: {integrity: sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw==} engines: {node: '>=6.9.0'} + dev: true /@babel/helper-validator-identifier/7.19.1: resolution: {integrity: sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==} engines: {node: '>=6.9.0'} + dev: true /@babel/helper-validator-option/7.18.6: resolution: {integrity: sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==} @@ -361,6 +367,7 @@ packages: '@babel/helper-validator-identifier': 7.19.1 chalk: 2.4.2 js-tokens: 4.0.0 + dev: true /@babel/parser/7.20.5: resolution: {integrity: sha512-r27t/cy/m9uKLXQNWWebeCUHgnAZq0CpG1OwKRxzJMP1vpSU4bSIK2hq+/cp0bQxetkXx38n09rNu8jVkcK/zA==} @@ -368,6 +375,7 @@ packages: hasBin: true dependencies: '@babel/types': 7.20.5 + dev: true /@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/7.18.6: resolution: {integrity: sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ==} @@ -1291,6 +1299,7 @@ packages: '@babel/code-frame': 7.18.6 '@babel/parser': 7.20.5 '@babel/types': 7.20.5 + dev: true /@babel/traverse/7.20.5: resolution: {integrity: sha512-WM5ZNN3JITQIq9tFZaw1ojLU3WgWdtkxnhM1AegMS+PvHjkM5IXjmYEGY7yukz5XS4sJyEf2VzWjI8uAavhxBQ==} @@ -1308,6 +1317,7 @@ packages: globals: 11.12.0 transitivePeerDependencies: - supports-color + dev: true /@babel/types/7.20.5: resolution: {integrity: sha512-c9fst/h2/dcF7H+MJKZ2T0KjEQ8hY/BNnDk/H3XY8C4Aw/eWQXWn/lWntHF9ooUBnGmEvbfGrTgLWc+um0YDUg==} @@ -1316,6 +1326,7 @@ packages: '@babel/helper-string-parser': 7.19.4 '@babel/helper-validator-identifier': 7.19.1 to-fast-properties: 2.0.0 + dev: true /@bcoe/v8-coverage/0.2.3: resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} @@ -1561,23 +1572,28 @@ packages: '@jridgewell/set-array': 1.1.2 '@jridgewell/sourcemap-codec': 1.4.14 '@jridgewell/trace-mapping': 0.3.17 + dev: true /@jridgewell/resolve-uri/3.1.0: resolution: {integrity: sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==} engines: {node: '>=6.0.0'} + dev: true /@jridgewell/set-array/1.1.2: resolution: {integrity: sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==} engines: {node: '>=6.0.0'} + dev: true /@jridgewell/sourcemap-codec/1.4.14: resolution: {integrity: sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==} + dev: true /@jridgewell/trace-mapping/0.3.17: resolution: {integrity: sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g==} dependencies: '@jridgewell/resolve-uri': 3.1.0 '@jridgewell/sourcemap-codec': 1.4.14 + dev: true /@nodelib/fs.scandir/2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} @@ -1816,6 +1832,7 @@ packages: engines: {node: '>=4'} dependencies: color-convert: 1.9.3 + dev: true /ansi-styles/4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} @@ -1863,23 +1880,6 @@ packages: engines: {node: '>= 0.4'} dev: true - /babel-eslint/10.1.0: - resolution: {integrity: sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg==} - engines: {node: '>=6'} - deprecated: babel-eslint is now @babel/eslint-parser. This package will no longer receive updates. - peerDependencies: - eslint: '>= 4.12.1' - dependencies: - '@babel/code-frame': 7.18.6 - '@babel/parser': 7.20.5 - '@babel/traverse': 7.20.5 - '@babel/types': 7.20.5 - eslint-visitor-keys: 1.3.0 - resolve: 1.22.1 - transitivePeerDependencies: - - supports-color - dev: false - /babel-jest/27.5.1: resolution: {integrity: sha512-cdQ5dXjGRd0IBRATiQ4mZGlGlRE8kJpjPOixdNRdT+m3UcNqmYWN6rK6nvtXYfY3D76cb8s/O1Ss8ea24PIwcg==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} @@ -2042,10 +2042,6 @@ packages: engines: {node: '>=8'} dev: true - /bmp-js/0.1.0: - resolution: {integrity: sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==} - dev: false - /brace-expansion/1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -2127,6 +2123,7 @@ packages: ansi-styles: 3.2.1 escape-string-regexp: 1.0.5 supports-color: 5.5.0 + dev: true /chalk/3.0.0: resolution: {integrity: sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==} @@ -2194,6 +2191,7 @@ packages: resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} dependencies: color-name: 1.1.3 + dev: true /color-convert/2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} @@ -2204,6 +2202,7 @@ packages: /color-name/1.1.3: resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} + dev: true /color-name/1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} @@ -2277,6 +2276,7 @@ packages: optional: true dependencies: ms: 2.1.2 + dev: true /decimal.js/10.4.3: resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==} @@ -2802,6 +2802,7 @@ packages: /escape-string-regexp/1.0.5: resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} engines: {node: '>=0.8.0'} + dev: true /escape-string-regexp/2.0.0: resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} @@ -2821,11 +2822,6 @@ packages: source-map: 0.6.1 dev: true - /eslint-visitor-keys/1.3.0: - resolution: {integrity: sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==} - engines: {node: '>=4'} - dev: false - /esprima/4.0.1: resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} engines: {node: '>=4'} @@ -2903,11 +2899,6 @@ packages: bser: 2.1.1 dev: true - /file-type/12.4.2: - resolution: {integrity: sha512-UssQP5ZgIOKelfsaB5CuGAL+Y+q7EmONuiwF3N5HAH0t27rvrttgi6Ra9k/+DVaY9UF6+ybxu5pOXLUdA8N7Vg==} - engines: {node: '>=8'} - dev: false - /fill-range/7.0.1: resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} engines: {node: '>=8'} @@ -2961,6 +2952,7 @@ packages: /function-bind/1.1.1: resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} + dev: true /functions-have-names/1.2.3: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} @@ -3015,6 +3007,7 @@ packages: /globals/11.12.0: resolution: {integrity: sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==} engines: {node: '>=4'} + dev: true /globby/11.1.0: resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} @@ -3045,6 +3038,7 @@ packages: /has-flag/3.0.0: resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} engines: {node: '>=4'} + dev: true /has-flag/4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} @@ -3074,6 +3068,7 @@ packages: engines: {node: '>= 0.4.0'} dependencies: function-bind: 1.1.1 + dev: true /html-encoding-sniffer/2.0.1: resolution: {integrity: sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==} @@ -3119,10 +3114,6 @@ packages: safer-buffer: 2.1.2 dev: true - /idb-keyval/3.2.0: - resolution: {integrity: sha512-slx8Q6oywCCSfKgPgL0sEsXtPVnSbTLWpyiDcu6msHOyKOLari1TD1qocXVCft80umnkk3/Qqh3lwoFt8T/BPQ==} - dev: false - /ignore/5.2.1: resolution: {integrity: sha512-d2qQLzTJ9WxQftPAuEQpSPmKqzxePjzVbpAVv62AQ64NTL+wR4JkrVqR/LqFsFEUsHDAiId52mJteHDFuDkElA==} engines: {node: '>= 4'} @@ -3208,6 +3199,7 @@ packages: resolution: {integrity: sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==} dependencies: has: 1.0.3 + dev: true /is-date-object/1.0.5: resolution: {integrity: sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==} @@ -3216,10 +3208,6 @@ packages: has-tostringtag: 1.0.0 dev: true - /is-electron/2.2.1: - resolution: {integrity: sha512-r8EEQQsqT+Gn0aXFx7lTFygYQhILLCB+wn0WCDL5LZRINeLH/Rvw1j2oKodELLXYNImQ3CRlVsY8wW4cGOsyuw==} - dev: false - /is-extglob/2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -3308,10 +3296,6 @@ packages: resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} dev: true - /is-url/1.2.4: - resolution: {integrity: sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==} - dev: false - /is-weakmap/2.0.1: resolution: {integrity: sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==} dev: true @@ -3845,6 +3829,7 @@ packages: /js-tokens/4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + dev: true /js-yaml/3.14.1: resolution: {integrity: sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==} @@ -3905,6 +3890,7 @@ packages: resolution: {integrity: sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==} engines: {node: '>=4'} hasBin: true + dev: true /json-parse-even-better-errors/2.3.1: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} @@ -4062,6 +4048,7 @@ packages: /ms/2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + dev: true /nanoid/3.3.4: resolution: {integrity: sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==} @@ -4073,18 +4060,6 @@ packages: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: true - /node-fetch/2.6.7: - resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} - engines: {node: 4.x || >=6.0.0} - peerDependencies: - encoding: ^0.1.0 - peerDependenciesMeta: - encoding: - optional: true - dependencies: - whatwg-url: 5.0.0 - dev: false - /node-int64/0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} dev: true @@ -4136,20 +4111,6 @@ packages: object-keys: 1.1.1 dev: true - /obsidian-text-extract/1.0.4: - resolution: {integrity: sha512-lJ7HaEPGUoGVYr6iCSEU8oPrnA8xRBQgSU9KryWteq3wO+Yiw2Zue70G0rZf9Yj+DkgR92PtuR8aQZQxTy/uLA==} - deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. - dependencies: - dexie: 3.2.2 - p-limit: 4.0.0 - pure-md5: 0.1.14 - tesseract.js: 3.0.3 - transitivePeerDependencies: - - encoding - - eslint - - supports-color - dev: false - /obsidian/1.1.1: resolution: {integrity: sha512-GcxhsHNkPEkwHEjeyitfYNBcQuYGeAHFs1pEpZIv0CnzSfui8p8bPLm2YKLgcg20B764770B1sYGtxCvk9ptxg==} peerDependencies: @@ -4173,11 +4134,6 @@ packages: mimic-fn: 2.1.0 dev: true - /opencollective-postinstall/2.0.3: - resolution: {integrity: sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==} - hasBin: true - dev: false - /optionator/0.8.3: resolution: {integrity: sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==} engines: {node: '>= 0.8.0'} @@ -4197,13 +4153,6 @@ packages: p-try: 2.2.0 dev: true - /p-limit/4.0.0: - resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - yocto-queue: 1.0.0 - dev: false - /p-locate/4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -4254,6 +4203,7 @@ packages: /path-parse/1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + dev: true /path-type/4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} @@ -4381,6 +4331,7 @@ packages: /regenerator-runtime/0.13.11: resolution: {integrity: sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==} + dev: true /regenerator-transform/0.15.1: resolution: {integrity: sha512-knzmNAcuyxV+gQCufkYcvOqX/qIIfHLv0u5x79kRxuGojfYVky1f15TzZEu2Avte8QGepvUNTnLskf8E6X6Vyg==} @@ -4446,11 +4397,6 @@ packages: engines: {node: '>=8'} dev: true - /resolve-url/0.2.1: - resolution: {integrity: sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==} - deprecated: https://github.com/lydell/resolve-url#deprecated - dev: false - /resolve.exports/1.1.0: resolution: {integrity: sha512-J1l+Zxxp4XK3LUDZ9m60LRJF/mAe4z6a4xyabPHk7pvK5t35dACV32iIjJDFeWZFfZlO29w6SZ67knR0tHzJtQ==} engines: {node: '>=10'} @@ -4463,6 +4409,7 @@ packages: is-core-module: 2.11.0 path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 + dev: true /reusify/1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} @@ -4669,6 +4616,7 @@ packages: engines: {node: '>=4'} dependencies: has-flag: 3.0.0 + dev: true /supports-color/7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} @@ -4695,6 +4643,7 @@ packages: /supports-preserve-symlinks-flag/1.0.0: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} + dev: true /svelte-check/2.10.2_svelte@3.54.0: resolution: {integrity: sha512-h1Tuiir0m8J5yqN+Vx6qgKKk1L871e6a9o7rMwVWfu8Qs6Wg7x2R+wcxS3SO3VpW5JCxCat90rxPsZMYgz+HaQ==} @@ -4803,33 +4752,6 @@ packages: supports-hyperlinks: 2.3.0 dev: true - /tesseract.js-core/3.0.2: - resolution: {integrity: sha512-2fD76ka9nO/C616R0fq+M9Zu91DA3vEfyozp0jlxaJOBmpfeprtgRP3cqVweZh2darE1kK/DazoxZ65g7WU99Q==} - dev: false - - /tesseract.js/3.0.3: - resolution: {integrity: sha512-eZ1+OGWvF5IMExAzIwnDf3S3kf2FeC+i4qrMTRvBSlZeHc3ONy0vCmaKmBQz6scjB6C1W2w2x0r4lCEh95qBnw==} - requiresBuild: true - dependencies: - babel-eslint: 10.1.0 - bmp-js: 0.1.0 - file-type: 12.4.2 - idb-keyval: 3.2.0 - is-electron: 2.2.1 - is-url: 1.2.4 - node-fetch: 2.6.7 - opencollective-postinstall: 2.0.3 - regenerator-runtime: 0.13.11 - resolve-url: 0.2.1 - tesseract.js-core: 3.0.2 - wasm-feature-detect: 1.3.0 - zlibjs: 0.3.1 - transitivePeerDependencies: - - encoding - - eslint - - supports-color - dev: false - /test-exclude/6.0.0: resolution: {integrity: sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==} engines: {node: '>=8'} @@ -4850,6 +4772,7 @@ packages: /to-fast-properties/2.0.0: resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} engines: {node: '>=4'} + dev: true /to-regex-range/5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} @@ -4868,10 +4791,6 @@ packages: url-parse: 1.5.10 dev: true - /tr46/0.0.3: - resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} - dev: false - /tr46/2.1.0: resolution: {integrity: sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw==} engines: {node: '>=8'} @@ -5026,14 +4945,6 @@ packages: makeerror: 1.0.12 dev: true - /wasm-feature-detect/1.3.0: - resolution: {integrity: sha512-w9datO3OReMouWgKOelvu1CozmLK/VbkXOtlzNTanBJpR0uBHyUwS3EYdXf5vBPoHKYS0lpuYo91rpqMNIZM9g==} - dev: false - - /webidl-conversions/3.0.1: - resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} - dev: false - /webidl-conversions/5.0.0: resolution: {integrity: sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==} engines: {node: '>=8'} @@ -5054,13 +4965,6 @@ packages: resolution: {integrity: sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==} dev: true - /whatwg-url/5.0.0: - resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} - dependencies: - tr46: 0.0.3 - webidl-conversions: 3.0.1 - dev: false - /whatwg-url/8.7.0: resolution: {integrity: sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg==} engines: {node: '>=10'} @@ -5183,12 +5087,3 @@ packages: y18n: 5.0.8 yargs-parser: 20.2.9 dev: true - - /yocto-queue/1.0.0: - resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} - engines: {node: '>=12.20'} - dev: false - - /zlibjs/0.3.1: - resolution: {integrity: sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==} - dev: false diff --git a/src/cache-manager.ts b/src/cache-manager.ts index 1040be5..cb9f7f7 100644 --- a/src/cache-manager.ts +++ b/src/cache-manager.ts @@ -16,7 +16,6 @@ import { makeMD5, removeDiacritics, } from './tools/utils' -import { getImageText, getPdfText } from 'obsidian-text-extract' import type { CanvasData } from 'obsidian/canvas' import type { AsPlainObject } from 'minisearch' import type MiniSearch from 'minisearch' @@ -60,24 +59,15 @@ async function getAndMapIndexedDocument( content = texts.join('\r\n') } - // a) ** Image or PDF ** with Text Extractor - else if (extractor) { + // ** Image or PDF ** + if (extractor) { if (extractor.canFileBeExtracted(path)) { content = await extractor.extractText(file) } else { throw new Error('Invalid file format: ' + file.path) } } - // b) ** Image or PDF ** without the text-extractor plugin - else { - if (isFilePDF(path)) { - content = await getPdfText(file) - } else if (isFileImage(file.path)) { - content = await getImageText(file) - } else { - throw new Error('Invalid file format: ' + file.path) - } - } + if (content === null || content === undefined) { // This shouldn't happen console.warn(`Omnisearch: ${content} content for file`, file.path) From 8942e662d67d3b2fe1d58f45740ae40b79cd8b95 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Thu, 19 Jan 2023 08:36:57 +0100 Subject: [PATCH 03/11] Fixed manifest --- manifest.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/manifest.json b/manifest.json index bb476bc..39ad950 100644 --- a/manifest.json +++ b/manifest.json @@ -6,9 +6,9 @@ "description": "A search engine that just works", "author": "Simon Cambier", "authorUrl": "https://github.com/scambier/obsidian-omnisearch", - "isDesktopOnly": false, - "fundingUrl": { - "Github": "https://github.com/sponsors/scambier", - "Ko-fi": "https://ko-fi.com/scambier" - } + "fundingUrl": { + "Github": "https://github.com/sponsors/scambier", + "Ko-fi": "https://ko-fi.com/scambier" + }, + "isDesktopOnly": false } From ebee93a2aa98ae9ecea4f34049e410d842ebd21d Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Thu, 19 Jan 2023 08:37:43 +0100 Subject: [PATCH 04/11] Registering events sooner --- src/main.ts | 71 ++++++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/src/main.ts b/src/main.ts index 7469075..559f8c5 100644 --- a/src/main.ts +++ b/src/main.ts @@ -64,45 +64,44 @@ export default class OmnisearchPlugin extends Plugin { }, }) - app.workspace.onLayoutReady(async () => { - // Listeners to keep the search index up-to-date - this.registerEvent( - this.app.vault.on('create', async file => { - if (isFileIndexable(file.path)) { - await cacheManager.addToLiveCache(file.path) - searchEngine.addFromPaths([file.path]) - } - }) - ) - this.registerEvent( - this.app.vault.on('delete', file => { - cacheManager.removeFromLiveCache(file.path) - searchEngine.removeFromPaths([file.path]) - }) - ) - this.registerEvent( - this.app.vault.on('modify', async file => { - if (isFileIndexable(file.path)) { - await cacheManager.addToLiveCache(file.path) - NotesIndex.markNoteForReindex(file) - } - }) - ) - this.registerEvent( - this.app.vault.on('rename', async (file, oldPath) => { - if (isFileIndexable(file.path)) { - cacheManager.removeFromLiveCache(oldPath) - cacheManager.addToLiveCache(file.path) - searchEngine.removeFromPaths([oldPath]) - await searchEngine.addFromPaths([file.path]) - } - }) - ) + // Listeners to keep the search index up-to-date + this.registerEvent( + this.app.vault.on('create', async file => { + if (isFileIndexable(file.path)) { + await cacheManager.addToLiveCache(file.path) + searchEngine.addFromPaths([file.path]) + } + }) + ) + this.registerEvent( + this.app.vault.on('delete', file => { + cacheManager.removeFromLiveCache(file.path) + searchEngine.removeFromPaths([file.path]) + }) + ) + this.registerEvent( + this.app.vault.on('modify', async file => { + if (isFileIndexable(file.path)) { + await cacheManager.addToLiveCache(file.path) + NotesIndex.markNoteForReindex(file) + } + }) + ) + this.registerEvent( + this.app.vault.on('rename', async (file, oldPath) => { + if (isFileIndexable(file.path)) { + cacheManager.removeFromLiveCache(oldPath) + cacheManager.addToLiveCache(file.path) + searchEngine.removeFromPaths([oldPath]) + await searchEngine.addFromPaths([file.path]) + } + }) + ) + app.workspace.onLayoutReady(async () => { + this.executeFirstLaunchTasks() await this.populateIndex() }) - - this.executeFirstLaunchTasks() } executeFirstLaunchTasks(): void { From 9c97a71d209e80ac0e1d76e92bc9506a71795c51 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Thu, 19 Jan 2023 08:38:30 +0100 Subject: [PATCH 05/11] Cleaner code --- src/tools/utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tools/utils.ts b/src/tools/utils.ts index 993a703..cc9d9ff 100644 --- a/src/tools/utils.ts +++ b/src/tools/utils.ts @@ -279,9 +279,9 @@ export function makeMD5(data: BinaryLike): string { } export function chunkArray(arr: T[], len: number): T[][] { - var chunks = [], - i = 0, - n = arr.length + const chunks = [] + let i = 0 + const n = arr.length while (i < n) { chunks.push(arr.slice(i, (i += len))) From 42ba65364251bf67c36dc0a77ab7c7dd7050238b Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Thu, 19 Jan 2023 21:46:53 +0100 Subject: [PATCH 06/11] Text Extractor requirement --- src/main.ts | 4 +-- src/settings.ts | 71 ++++++++++++++++++++++------------------------ src/tools/utils.ts | 7 +++-- 3 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/main.ts b/src/main.ts index 559f8c5..e089de0 100644 --- a/src/main.ts +++ b/src/main.ts @@ -105,11 +105,11 @@ export default class OmnisearchPlugin extends Plugin { } executeFirstLaunchTasks(): void { - const code = '1.10.0-beta.1' + const code = '1.10.1' if (settings.welcomeMessage !== code) { const welcome = new DocumentFragment() welcome.createSpan({}, span => { - span.innerHTML = `๐Ÿ”Ž Omnisearch will soon require the Text Extractor plugin to index PDF and images. See Omnisearch settings for more information.` + span.innerHTML = `๐Ÿ”Ž Omnisearch now requires the Text Extractor plugin to index PDF and images. See Omnisearch settings for more information.` }) new Notice(welcome, 20_000) } diff --git a/src/settings.ts b/src/settings.ts index e059de7..0d2d4de 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -86,53 +86,50 @@ export class SettingsTab extends PluginSettingTab { new Setting(containerEl).setName('Indexing').setHeading() + const textExtractDesc = new DocumentFragment() if (getTextExtractor()) { - const desc = new DocumentFragment() - desc.createSpan({}, span => { + textExtractDesc.createSpan({}, span => { span.innerHTML = `๐Ÿ‘ You have installed Text Extractor, Omnisearch will use it to index PDFs and images.
Text extraction only works on desktop, but the cache can be synchronized with your mobile device.` }) - new Setting(containerEl).setDesc(desc) } else { - const label = new DocumentFragment() - label.createSpan({}, span => { - span.innerHTML = `โš ๏ธ Omnisearch will soon require Text Extractor to index PDFs and images. - You can already install it to get a head start.` + textExtractDesc.createSpan({}, span => { + span.innerHTML = `โš ๏ธ Omnisearch requires Text Extractor to index PDFs and images.` }) - new Setting(containerEl).setDesc(label) } + new Setting(containerEl).setDesc(textExtractDesc) // PDF Indexing - if (!Platform.isMobileApp || getTextExtractor()) { - const indexPDFsDesc = new DocumentFragment() - indexPDFsDesc.createSpan({}, span => { - span.innerHTML = `Include PDFs in search results - Will soon depend on Text Extractor.` - }) - new Setting(containerEl) - .setName(`PDFs Indexing`) - .setDesc(indexPDFsDesc) - .addToggle(toggle => - toggle.setValue(settings.PDFIndexing).onChange(async v => { - settings.PDFIndexing = v - await saveSettings(this.plugin) - }) - ) + const indexPDFsDesc = new DocumentFragment() + indexPDFsDesc.createSpan({}, span => { + span.innerHTML = `Include PDFs in search results` + }) + new Setting(containerEl) + .setName(`PDFs Indexing ${getTextExtractor() ? '' : 'โš ๏ธ Disabled'}`) + .setDesc(indexPDFsDesc) + .addToggle(toggle => + toggle.setValue(settings.PDFIndexing).onChange(async v => { + settings.PDFIndexing = v + await saveSettings(this.plugin) + }) + ) + .setDisabled(!getTextExtractor()) - // Images Indexing - const indexImagesDesc = new DocumentFragment() - indexImagesDesc.createSpan({}, span => { - span.innerHTML = `Include images in search results - Will soon depend on Text Extractor.` - }) - new Setting(containerEl) - .setName(`Images Indexing`) - .setDesc(indexImagesDesc) - .addToggle(toggle => - toggle.setValue(settings.imagesIndexing).onChange(async v => { - settings.imagesIndexing = v - await saveSettings(this.plugin) - }) - ) - } + // Images Indexing + const indexImagesDesc = new DocumentFragment() + indexImagesDesc.createSpan({}, span => { + span.innerHTML = `Include images in search results` + }) + new Setting(containerEl) + .setName(`Images Indexing ${getTextExtractor() ? '' : 'โš ๏ธ Disabled'}`) + .setDesc(indexImagesDesc) + .addToggle(toggle => + toggle.setValue(settings.imagesIndexing).onChange(async v => { + settings.imagesIndexing = v + await saveSettings(this.plugin) + }) + ) + .setDisabled(!getTextExtractor()) // Additional files to index const indexedFileTypesDesc = new DocumentFragment() diff --git a/src/tools/utils.ts b/src/tools/utils.ts index cc9d9ff..02f20f9 100644 --- a/src/tools/utils.ts +++ b/src/tools/utils.ts @@ -5,16 +5,17 @@ import { parseFrontMatterAliases, Platform, } from 'obsidian' -import { getTextExtractor, type SearchMatch } from '../globals' import { excerptAfter, excerptBefore, getChsSegmenter, + getTextExtractor, highlightClass, isSearchMatch, regexLineSplit, regexStripQuotes, regexYaml, + type SearchMatch, } from '../globals' import { settings } from '../settings' import { type BinaryLike, createHash } from 'crypto' @@ -236,8 +237,8 @@ export function getCtrlKeyLabel(): 'ctrl' | 'โŒ˜' { } export function isFileIndexable(path: string): boolean { - const canIndexPDF = (!Platform.isMobileApp || !!getTextExtractor()) && settings.PDFIndexing - const canIndexImages = (!Platform.isMobileApp || !!getTextExtractor()) && settings.imagesIndexing + const canIndexPDF = !!getTextExtractor() && settings.PDFIndexing + const canIndexImages = !!getTextExtractor() && settings.imagesIndexing return ( isFilePlaintext(path) || isFileCanvas(path) || From 5bc3ac045a8cd03a93227dc462f1cf50790aa711 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Thu, 19 Jan 2023 22:02:19 +0100 Subject: [PATCH 07/11] Fixed prefix length bug --- src/search/omnisearch.ts | 4 ++-- src/settings.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/search/omnisearch.ts b/src/search/omnisearch.ts index 96cdd1a..4fc0ad0 100644 --- a/src/search/omnisearch.ts +++ b/src/search/omnisearch.ts @@ -280,12 +280,12 @@ export class Omnisearch { let results: SearchResult[] if (settings.simpleSearch) { results = await this.search(query, { - prefixLength: 1, + prefixLength: 3, singleFilePath: options?.singleFilePath, }) } else { results = await this.search(query, { - prefixLength: 3, + prefixLength: 1, singleFilePath: options?.singleFilePath, }) } diff --git a/src/settings.ts b/src/settings.ts index 0d2d4de..2e6357a 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -210,7 +210,7 @@ export class SettingsTab extends PluginSettingTab { new Setting(containerEl) .setName('Simpler search') .setDesc( - `Enable this if Obsidian often freezes while making searches. This will return more strict results.` + `Enable this if Obsidian often freezes while making searches. This may return fewer results.` ) .addToggle(toggle => toggle.setValue(settings.simpleSearch).onChange(async v => { From 61d1654c0c8e1e9a59a3df7bfdb77d06272c08dd Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Thu, 19 Jan 2023 22:02:57 +0100 Subject: [PATCH 08/11] 1.11.0-beta.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1533e69..1d07df3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "scambier.obsidian-search", - "version": "1.10.0", + "version": "1.11.0-beta.1", "description": "A search engine for Obsidian", "main": "dist/main.js", "scripts": { From e243c32a8dc250743db9346af72d37827f62e108 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Fri, 20 Jan 2023 22:18:48 +0100 Subject: [PATCH 09/11] Updated README --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d4abc8d..4f5fdfc 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ Under the hood, it uses the excellent [MiniSearch](https://github.com/lucaong/mi > Omnisearch's first goal is to _locate_ files instantly. You can see it as a _Quick Switcher_ on steroids. - Find your **๐Ÿ“notes, ๐Ÿ“„PDFs, and ๐Ÿ–ผimages** faster than ever - - _Images OCR and PDF indexing are only available on desktop_ + - Indexing of images and PDF indexing is available + through [Text Extractor](https://github.com/scambier/obsidian-text-extractor) - Automatic document scoring using the [BM25 algorithm](https://github.com/lucaong/minisearch/issues/129#issuecomment-1046257399) - The relevance of a document against a query depends on the number of times the query terms appear in the document, @@ -130,15 +131,18 @@ See [styles.css](./assets/styles.css) for more information. - If you have several thousands of files, Obsidian may freeze a few seconds at startup while the Omnisearch cache is loaded in memory. +**Omnisearch seems to make Obsidian slower.** + +- Once Obsidian has indexed your files at startup, it doesn't do anything while its modal is closed. Your changes are + not indexed until you open the modal again. If you experience slowdowns while using Obsidian, it's unlikely that + Omnisearch is responsible. +- However, Text Extractor can make Obsidian slower when indexing PDFs and images for the first time. If you don't need + those features, you can disable them in the plugin settings. + **Omnisearch is slow to index my PDFs and images** -- The first time Omnisearch indexes those files, it needs to extract their text. This can take a long time, but - will only happen once. This process is also resumable, so you can temporarily disable PDFs/images indexing, or close - Obsidian without losing data. - -**Can I index PDFs/images on mobile?** - -- Not at the moment. On mobile devices, text extraction either doesn't work or consumes too much resources. +- The first time Text Extractor reads those files, it can take a long time to extract their text. The results are then + cached, and indexing shouldn't take more time than your average markdown note. **Omnisearch gives inconsistent/invalid results, there are errors in the developer console** @@ -151,10 +155,6 @@ See [styles.css](./assets/styles.css) for more information. - If you have modified them, reset weightings to their original values. - Rewrite your query and avoid numbers and common words. -**How do I highlight matches in search results?** - -See [here](https://github.com/scambier/obsidian-omnisearch#css-customization). - **I'm still having an issue** You can write your issue [here](https://github.com/scambier/obsidian-omnisearch/issues) with as much details as From c1b3c6d0ec11ef83cb820113d8837fc1a6cf006c Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Fri, 20 Jan 2023 22:56:02 +0100 Subject: [PATCH 10/11] Fixed potential indexing issues --- src/cache-manager.ts | 14 +++--- src/main.ts | 100 +++++++++++++++++++++------------------ src/search/omnisearch.ts | 14 +++++- src/tools/utils.ts | 5 +- 4 files changed, 78 insertions(+), 55 deletions(-) diff --git a/src/cache-manager.ts b/src/cache-manager.ts index cb9f7f7..7e48fda 100644 --- a/src/cache-manager.ts +++ b/src/cache-manager.ts @@ -60,12 +60,10 @@ async function getAndMapIndexedDocument( } // ** Image or PDF ** - if (extractor) { - if (extractor.canFileBeExtracted(path)) { - content = await extractor.extractText(file) - } else { - throw new Error('Invalid file format: ' + file.path) - } + else if (extractor?.canFileBeExtracted(path)) { + content = await extractor.extractText(file) + } else { + throw new Error(`Unsupported file type: "${path}"`) } if (content === null || content === undefined) { @@ -125,6 +123,10 @@ class CacheManager { */ private documents: Map = new Map() + /** + * Set or update the live cache with the content of the given file. + * @param path + */ public async addToLiveCache(path: string): Promise { try { const doc = await getAndMapIndexedDocument(path) diff --git a/src/main.ts b/src/main.ts index e089de0..524bb73 100644 --- a/src/main.ts +++ b/src/main.ts @@ -64,41 +64,41 @@ export default class OmnisearchPlugin extends Plugin { }, }) - // Listeners to keep the search index up-to-date - this.registerEvent( - this.app.vault.on('create', async file => { - if (isFileIndexable(file.path)) { - await cacheManager.addToLiveCache(file.path) - searchEngine.addFromPaths([file.path]) - } - }) - ) - this.registerEvent( - this.app.vault.on('delete', file => { - cacheManager.removeFromLiveCache(file.path) - searchEngine.removeFromPaths([file.path]) - }) - ) - this.registerEvent( - this.app.vault.on('modify', async file => { - if (isFileIndexable(file.path)) { - await cacheManager.addToLiveCache(file.path) - NotesIndex.markNoteForReindex(file) - } - }) - ) - this.registerEvent( - this.app.vault.on('rename', async (file, oldPath) => { - if (isFileIndexable(file.path)) { - cacheManager.removeFromLiveCache(oldPath) - cacheManager.addToLiveCache(file.path) - searchEngine.removeFromPaths([oldPath]) - await searchEngine.addFromPaths([file.path]) - } - }) - ) - app.workspace.onLayoutReady(async () => { + // Listeners to keep the search index up-to-date + this.registerEvent( + this.app.vault.on('create', file => { + if (isFileIndexable(file.path)) { + // await cacheManager.addToLiveCache(file.path) + searchEngine.addFromPaths([file.path]) + } + }) + ) + this.registerEvent( + this.app.vault.on('delete', file => { + cacheManager.removeFromLiveCache(file.path) + searchEngine.removeFromPaths([file.path]) + }) + ) + this.registerEvent( + this.app.vault.on('modify', async file => { + if (isFileIndexable(file.path)) { + await cacheManager.addToLiveCache(file.path) + NotesIndex.markNoteForReindex(file) + } + }) + ) + this.registerEvent( + this.app.vault.on('rename', async (file, oldPath) => { + if (isFileIndexable(file.path)) { + cacheManager.removeFromLiveCache(oldPath) + cacheManager.addToLiveCache(file.path) + searchEngine.removeFromPaths([oldPath]) + await searchEngine.addFromPaths([file.path]) + } + }) + ) + this.executeFirstLaunchTasks() await this.populateIndex() }) @@ -145,30 +145,38 @@ export default class OmnisearchPlugin extends Plugin { indexingStep.set(IndexingStepType.ReadingFiles) const files = app.vault.getFiles().filter(f => isFileIndexable(f.path)) console.log(`Omnisearch - ${files.length} files total`) - + console.log( + `Omnisearch - Cache is ${isCacheEnabled() ? 'enabled' : 'disabled'}` + ) // Map documents in the background // Promise.all(files.map(f => cacheManager.addToLiveCache(f.path))) if (isCacheEnabled()) { console.time('Omnisearch - Loading index from cache') indexingStep.set(IndexingStepType.LoadingCache) - await searchEngine.loadCache() - console.timeEnd('Omnisearch - Loading index from cache') + const hasCache = await searchEngine.loadCache() + if (hasCache) { + console.timeEnd('Omnisearch - Loading index from cache') + } } const diff = searchEngine.getDiff( files.map(f => ({ path: f.path, mtime: f.stat.mtime })) ) - if (diff.toAdd.length) { - console.log( - 'Omnisearch - Total number of files to add/update: ' + diff.toAdd.length - ) - } - if (diff.toRemove.length) { - console.log( - 'Omnisearch - Total number of files to remove: ' + diff.toRemove.length - ) + if (isCacheEnabled()) { + if (diff.toAdd.length) { + console.log( + 'Omnisearch - Total number of files to add/update: ' + + diff.toAdd.length + ) + } + if (diff.toRemove.length) { + console.log( + 'Omnisearch - Total number of files to remove: ' + + diff.toRemove.length + ) + } } if (diff.toAdd.length >= 1000 && isCacheEnabled()) { diff --git a/src/search/omnisearch.ts b/src/search/omnisearch.ts index 4fc0ad0..df88b74 100644 --- a/src/search/omnisearch.ts +++ b/src/search/omnisearch.ts @@ -16,6 +16,7 @@ import { import { Notice, Platform } from 'obsidian' import type { Query } from './query' import { cacheManager } from '../cache-manager' +import { sortBy } from 'lodash-es' const tokenize = (text: string): string[] => { const tokens = text.split(SPACE_OR_PUNCTUATION) @@ -59,12 +60,19 @@ export class Omnisearch { this.minisearch = new MiniSearch(Omnisearch.options) } - async loadCache(): Promise { + /** + * Return true if the cache is valid + */ + async loadCache(): Promise { const cache = await cacheManager.getMinisearchCache() if (cache) { + // console.log('Omnisearch - Cache', cache) this.minisearch = MiniSearch.loadJS(cache.data, Omnisearch.options) this.indexedDocuments = new Map(cache.paths.map(o => [o.path, o.mtime])) + return true } + console.log('Omnisearch - No cache found') + return false } /** @@ -77,11 +85,13 @@ export class Omnisearch { } { const docsMap = new Map(docs.map(d => [d.path, d.mtime])) + // console.log(this.indexedDocuments) const toAdd = docs.filter( d => !this.indexedDocuments.has(d.path) || this.indexedDocuments.get(d.path) !== d.mtime ) + // console.log(toAdd) const toRemove = [...this.indexedDocuments] .filter( ([path, mtime]) => !docsMap.has(path) || docsMap.get(path) !== mtime @@ -100,6 +110,8 @@ export class Omnisearch { paths.map(async path => await cacheManager.getDocument(path)) ) ).filter(d => !!d?.path) + // Index markdown files first + documents = sortBy(documents, d => (d.path.endsWith('.md') ? 0 : 1)) // If a document is already added, discard it this.removeFromPaths( diff --git a/src/tools/utils.ts b/src/tools/utils.ts index 02f20f9..fb392c9 100644 --- a/src/tools/utils.ts +++ b/src/tools/utils.ts @@ -237,8 +237,9 @@ export function getCtrlKeyLabel(): 'ctrl' | 'โŒ˜' { } export function isFileIndexable(path: string): boolean { - const canIndexPDF = !!getTextExtractor() && settings.PDFIndexing - const canIndexImages = !!getTextExtractor() && settings.imagesIndexing + const hasTextExtractor = !!getTextExtractor() + const canIndexPDF = hasTextExtractor && settings.PDFIndexing + const canIndexImages = hasTextExtractor && settings.imagesIndexing return ( isFilePlaintext(path) || isFileCanvas(path) || From c8db324183076c67bc600cf61750f3b381adb075 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Fri, 20 Jan 2023 22:59:13 +0100 Subject: [PATCH 11/11] Updated readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4f5fdfc..b6e63ea 100644 --- a/README.md +++ b/README.md @@ -136,13 +136,13 @@ See [styles.css](./assets/styles.css) for more information. - Once Obsidian has indexed your files at startup, it doesn't do anything while its modal is closed. Your changes are not indexed until you open the modal again. If you experience slowdowns while using Obsidian, it's unlikely that Omnisearch is responsible. -- However, Text Extractor can make Obsidian slower when indexing PDFs and images for the first time. If you don't need +- However, Text Extractor can make Obsidian slower while indexing PDFs and images for the first time. If you don't need those features, you can disable them in the plugin settings. **Omnisearch is slow to index my PDFs and images** - The first time Text Extractor reads those files, it can take a long time to extract their text. The results are then - cached, and indexing shouldn't take more time than your average markdown note. + cached for the text startup. **Omnisearch gives inconsistent/invalid results, there are errors in the developer console**