diff --git a/src/main.ts b/src/main.ts index e0c804c..55d8bde 100644 --- a/src/main.ts +++ b/src/main.ts @@ -6,7 +6,7 @@ import { eventBus } from './globals' import { registerAPI } from '@vanakat/plugin-api' import api from './api' import { loadSearchHistory } from './search-history' -import {isFileIndexable, showWelcomeNotice} from './utils' +import {isFilePlaintext, showWelcomeNotice} from './utils' import { addNoteToReindex, addToIndex, removeFromIndex } from './notes-index' function _registerAPI(plugin: OmnisearchPlugin): void { @@ -22,10 +22,7 @@ export default class OmnisearchPlugin extends Plugin { // additional files to index by Omnisearch await loadSettings(this) - - this.registerExtensions(settings.indexedFileTypes, 'markdown') await loadSearchHistory() - _registerAPI(this) if (settings.ribbonIcon) { @@ -75,7 +72,7 @@ export default class OmnisearchPlugin extends Plugin { ) this.registerEvent( this.app.vault.on('rename', async (file, oldPath) => { - if (file instanceof TFile && isFileIndexable(file.path)) { + if (file instanceof TFile && isFilePlaintext(file.path)) { removeFromIndex(oldPath) await addToIndex(file) } diff --git a/src/notes-index.ts b/src/notes-index.ts index 4b57ae8..c9592e9 100644 --- a/src/notes-index.ts +++ b/src/notes-index.ts @@ -2,8 +2,8 @@ import { Notice, TAbstractFile, TFile } from 'obsidian' import { extractHeadingsFromCache, getAliasesFromMetadata, - getTagsFromMetadata, - isFileIndexable, + getTagsFromMetadata, isFileIndexable, + isFilePlaintext, removeDiacritics, wait, } from './utils' @@ -129,7 +129,7 @@ export function addNonExistingToIndex(name: string, parent: string): void { * @param path */ export function removeFromIndex(path: string): void { - if (!isFileIndexable(path)) { + if (!isFilePlaintext(path)) { console.info(`"${path}" is not an indexable file`) return } diff --git a/src/pdf-parser.ts b/src/pdf-parser.ts index 4ac550f..886f25c 100644 --- a/src/pdf-parser.ts +++ b/src/pdf-parser.ts @@ -1,11 +1,11 @@ import type { TFile } from 'obsidian' -import {loadPdfJs} from "obsidian"; +import { loadPdfJs } from 'obsidian' let PDFJs: any = null // https://stackoverflow.com/a/59929946 export async function getPdfText(file: TFile): Promise { - PDFJs = PDFJs ?? await loadPdfJs() + PDFJs = PDFJs ?? (await loadPdfJs()) const data = await app.vault.readBinary(file) const doc = await PDFJs.getDocument(data).promise const pageTexts = Array.from({ length: doc.numPages }, async (v, i) => { diff --git a/src/search.ts b/src/search.ts index 0c22cd9..5fef912 100644 --- a/src/search.ts +++ b/src/search.ts @@ -9,7 +9,7 @@ import { SPACE_OR_PUNCTUATION, } from './globals' import { - isFileIndexable, + isFilePlaintext, removeDiacritics, stringsToRegex, stripMarkdownCharacters, @@ -42,7 +42,7 @@ const tokenize = (text: string): string[] => { * Initializes the MiniSearch instance, * and adds all the notes to the index */ -export async function initGlobalSearchIndex(force = false): Promise { +export async function initGlobalSearchIndex(): Promise { const options: Options = { tokenize, processTerm: (term: string) => @@ -76,7 +76,7 @@ export async function initGlobalSearchIndex(force = false): Promise { } } - if (!minisearchInstance || force) { + if (!minisearchInstance) { minisearchInstance = new MiniSearch(options) resetNotesCache() } @@ -84,7 +84,7 @@ export async function initGlobalSearchIndex(force = false): Promise { // Index files that are already present const start = new Date().getTime() - const allFiles = app.vault.getFiles().filter(f => isFileIndexable(f.path)) + const allFiles = app.vault.getFiles().filter(f => isFilePlaintext(f.path)) let files let notesSuffix @@ -99,16 +99,14 @@ export async function initGlobalSearchIndex(force = false): Promise { console.log(`Omnisearch - indexing ${files.length} ${notesSuffix}`) // This is basically the same behavior as MiniSearch's `addAllAsync()`. - // We index files by batches of 10 + // We index markdown and plaintext files by batches of 10 for (let i = 0; i < files.length; ++i) { if (i % 10 === 0) await wait(0) const file = files[i] - if (file) { - if (getNoteFromCache(file.path)) { - removeFromIndex(file.path) - } - await addToIndex(file) + if (getNoteFromCache(file.path)) { + removeFromIndex(file.path) } + await addToIndex(file) } if (files.length > 0) { @@ -123,6 +121,27 @@ export async function initGlobalSearchIndex(force = false): Promise { } await saveIndexToFile() + + // PDFs are indexed later, since they're heavier + await indexPDFs() + } +} + +async function indexPDFs() { + if (settings.indexPDFs) { + console.warn("Omnisearch - Warnings on pdf.worker.min are due to some issues while reading PDFs file.") + const files = app.vault.getFiles().filter(f => f.path.endsWith('.pdf')) + for (const file of files) { + await wait(0) + if (getNoteFromCache(file.path)) { + removeFromIndex(file.path) + } + await addToIndex(file) + console.log(file.path) + } + if (settings.showIndexingNotices) { + new Notice(`Omnisearch - Indexed ${files.length} PDFs`) + } } } diff --git a/src/settings.ts b/src/settings.ts index be58fa0..2b90f45 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -2,7 +2,6 @@ import { Plugin, PluginSettingTab, Setting, SliderComponent } from 'obsidian' import { writable } from 'svelte/store' import { notesCacheFilePath, searchIndexFilePath } from './globals' import type OmnisearchPlugin from './main' -import { initGlobalSearchIndex } from './search' interface WeightingSettings { weightBasename: number @@ -75,7 +74,7 @@ export class SettingsTab extends PluginSettingTab { const diacriticsDesc = new DocumentFragment() diacriticsDesc.createSpan({}, span => { span.innerHTML = `Normalize diacritics in search terms. Words like "brûlée" or "žluťoučký" will be indexed as "brulee" and "zlutoucky".
- Changing this will trigger a full reindex.` + Needs a restart to fully take effect.` }) new Setting(containerEl) .setName('Ignore diacritics') @@ -84,7 +83,6 @@ export class SettingsTab extends PluginSettingTab { toggle.setValue(settings.ignoreDiacritics).onChange(async v => { settings.ignoreDiacritics = v await saveSettings(this.plugin) - await initGlobalSearchIndex(true) }) ) @@ -93,7 +91,6 @@ export class SettingsTab extends PluginSettingTab { indexedFileTypesDesc.createSpan({}, span => { span.innerHTML = `In addition to standard md files, Omnisearch can also index other plain text files.
Add extensions separated by a space. Example: txt org.
- This setting will also add these files in the navigation, and they will be treated as markdown.
Needs a restart to fully take effect.` }) new Setting(containerEl) @@ -114,7 +111,8 @@ export class SettingsTab extends PluginSettingTab { indexPDFsDesc.createSpan({}, span => { span.innerHTML = `Omnisearch will index your PDFs, and return them in search results. This feature is currently a work-in-progress, please report slowdowns or issues that you might experience.
- Changing this will trigger a full reindex.` + PDFs being quite slow to index, it is strongly recommended to also enable "Store index in file".
+ Needs a restart to fully take effect.` }) new Setting(containerEl) .setName('BETA - Index PDFs') @@ -123,9 +121,9 @@ export class SettingsTab extends PluginSettingTab { toggle.setValue(settings.indexPDFs).onChange(async v => { settings.indexPDFs = v await saveSettings(this.plugin) - await initGlobalSearchIndex(true) }) ) + // Store index const serializedIndexDesc = new DocumentFragment() serializedIndexDesc.createSpan({}, span => { @@ -133,7 +131,7 @@ export class SettingsTab extends PluginSettingTab { This results in faster loading times for bigger vaults and mobile devices.
⚠️ Note: the index can become corrupted - if you notice any issue, disable and re-enable this option to clear the cache.
⚠️ Cache files in .obsidian/plugins/omnisearch/ must not be synchronized.
- Changing this will trigger a full reindex. + Needs a restart to fully take effect. ` }) new Setting(containerEl) @@ -141,11 +139,18 @@ export class SettingsTab extends PluginSettingTab { .setDesc(serializedIndexDesc) .addToggle(toggle => toggle.setValue(settings.storeIndexInFile).onChange(async v => { - await app.vault.adapter.remove(notesCacheFilePath) - await app.vault.adapter.remove(searchIndexFilePath) + try { + await app.vault.adapter.remove(notesCacheFilePath) + } catch (e) { + console.warn(e) + } + try { + await app.vault.adapter.remove(searchIndexFilePath) + } catch (e) { + console.warn(e) + } settings.storeIndexInFile = v await saveSettings(this.plugin) - await initGlobalSearchIndex(true) }) ) @@ -311,7 +316,7 @@ export const DEFAULT_SETTINGS: OmnisearchSettings = { storeIndexInFile: false, - welcomeMessage: '' + welcomeMessage: '', } as const export let settings = Object.assign({}, DEFAULT_SETTINGS) as OmnisearchSettings diff --git a/src/utils.ts b/src/utils.ts index d90d50e..89889b7 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -173,11 +173,15 @@ export function getCtrlKeyLabel(): 'ctrl' | '⌘' { } export function isFileIndexable(path: string): boolean { - return ( - path.endsWith('.md') || - (settings.indexPDFs && path.endsWith('.pdf')) || - settings.indexedFileTypes.some(t => path.endsWith(`.${t}`)) - ) + return (settings.indexPDFs && path.endsWith('.pdf')) || isFilePlaintext(path) +} + +export function isFilePlaintext(path: string): boolean { + return getPlaintextExtensions().some(t => path.endsWith(`.${t}`)) +} + +export function getPlaintextExtensions(): string[] { + return [...settings.indexedFileTypes, 'md'] } export function getExtension(path: string): string {