tanner/obsidian-tannersearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#58 - new behavior

Browse Source 
Omnisearch no longer register extensions, PDFs are indexed after plaintext files, updated settings descriptions
This commit is contained in:
Simon Cambier
2022-10-01 13:58:11 +02:00
parent 7dc01ecc8f
commit 124e5f5d61
6 changed files with 61 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/main.ts
View File

@@ -6,7 +6,7 @@ import { eventBus } from './globals'
import { registerAPI } from '@vanakat/plugin-api'
import api from './api'
import { loadSearchHistory } from './search-history'
import {isFileIndexable, showWelcomeNotice} from './utils'
import {isFilePlaintext, showWelcomeNotice} from './utils'
import { addNoteToReindex, addToIndex, removeFromIndex } from './notes-index'
function _registerAPI(plugin: OmnisearchPlugin): void {
@@ -22,10 +22,7 @@ export default class OmnisearchPlugin extends Plugin {
// additional files to index by Omnisearch
await loadSettings(this)
this.registerExtensions(settings.indexedFileTypes, 'markdown')
await loadSearchHistory()
_registerAPI(this)
if (settings.ribbonIcon) {
@@ -75,7 +72,7 @@ export default class OmnisearchPlugin extends Plugin {
)
this.registerEvent(
this.app.vault.on('rename', async (file, oldPath) => {
if (file instanceof TFile && isFileIndexable(file.path)) {
if (file instanceof TFile && isFilePlaintext(file.path)) {
removeFromIndex(oldPath)
await addToIndex(file)
}

6
src/notes-index.ts
View File

@@ -2,8 +2,8 @@ import { Notice, TAbstractFile, TFile } from 'obsidian'
import {
extractHeadingsFromCache,
getAliasesFromMetadata,
getTagsFromMetadata,
isFileIndexable,
getTagsFromMetadata, isFileIndexable,
isFilePlaintext,
removeDiacritics,
wait,
} from './utils'
@@ -129,7 +129,7 @@ export function addNonExistingToIndex(name: string, parent: string): void {
* @param path
*/
export function removeFromIndex(path: string): void {
if (!isFileIndexable(path)) {
if (!isFilePlaintext(path)) {
console.info(`"${path}" is not an indexable file`)
return
}

4
src/pdf-parser.ts
View File

@@ -1,11 +1,11 @@
import type { TFile } from 'obsidian'
import {loadPdfJs} from "obsidian";
import { loadPdfJs } from 'obsidian'
let PDFJs: any = null
// https://stackoverflow.com/a/59929946
export async function getPdfText(file: TFile): Promise<string> {
PDFJs = PDFJs ?? await loadPdfJs()
PDFJs = PDFJs ?? (await loadPdfJs())
const data = await app.vault.readBinary(file)
const doc = await PDFJs.getDocument(data).promise
const pageTexts = Array.from({ length: doc.numPages }, async (v, i) => {

33
src/search.ts
View File

@@ -9,7 +9,7 @@ import {
SPACE_OR_PUNCTUATION,
} from './globals'
import {
isFileIndexable,
isFilePlaintext,
removeDiacritics,
stringsToRegex,
stripMarkdownCharacters,
@@ -42,7 +42,7 @@ const tokenize = (text: string): string[] => {
* Initializes the MiniSearch instance,
* and adds all the notes to the index
*/
export async function initGlobalSearchIndex(force = false): Promise<void> {
export async function initGlobalSearchIndex(): Promise<void> {
const options: Options<IndexedNote> = {
tokenize,
processTerm: (term: string) =>
@@ -76,7 +76,7 @@ export async function initGlobalSearchIndex(force = false): Promise<void> {
}
}
if (!minisearchInstance || force) {
if (!minisearchInstance) {
minisearchInstance = new MiniSearch(options)
resetNotesCache()
}
@@ -84,7 +84,7 @@ export async function initGlobalSearchIndex(force = false): Promise<void> {
// Index files that are already present
const start = new Date().getTime()
const allFiles = app.vault.getFiles().filter(f => isFileIndexable(f.path))
const allFiles = app.vault.getFiles().filter(f => isFilePlaintext(f.path))
let files
let notesSuffix
@@ -99,17 +99,15 @@ export async function initGlobalSearchIndex(force = false): Promise<void> {
console.log(`Omnisearch - indexing ${files.length} ${notesSuffix}`)
// This is basically the same behavior as MiniSearch's `addAllAsync()`.
// We index files by batches of 10
// We index markdown and plaintext files by batches of 10
for (let i = 0; i < files.length; ++i) {
if (i % 10 === 0) await wait(0)
const file = files[i]
if (file) {
if (getNoteFromCache(file.path)) {
removeFromIndex(file.path)
}
await addToIndex(file)
}
}
if (files.length > 0) {
const message = `Omnisearch - Indexed ${files.length} ${notesSuffix} in ${
@@ -123,6 +121,27 @@ export async function initGlobalSearchIndex(force = false): Promise<void> {
}
await saveIndexToFile()
// PDFs are indexed later, since they're heavier
await indexPDFs()
}
}
async function indexPDFs() {
if (settings.indexPDFs) {
console.warn("Omnisearch - Warnings on pdf.worker.min are due to some issues while reading PDFs file.")
const files = app.vault.getFiles().filter(f => f.path.endsWith('.pdf'))
for (const file of files) {
await wait(0)
if (getNoteFromCache(file.path)) {
removeFromIndex(file.path)
}
await addToIndex(file)
console.log(file.path)
}
if (settings.showIndexingNotices) {
new Notice(`Omnisearch - Indexed ${files.length} PDFs`)
}
}
}

23
src/settings.ts
View File

@@ -2,7 +2,6 @@ import { Plugin, PluginSettingTab, Setting, SliderComponent } from 'obsidian'
import { writable } from 'svelte/store'
import { notesCacheFilePath, searchIndexFilePath } from './globals'
import type OmnisearchPlugin from './main'
import { initGlobalSearchIndex } from './search'
interface WeightingSettings {
weightBasename: number
@@ -75,7 +74,7 @@ export class SettingsTab extends PluginSettingTab {
const diacriticsDesc = new DocumentFragment()
diacriticsDesc.createSpan({}, span => {
span.innerHTML = `Normalize diacritics in search terms. Words like "brûlée" or "žluťoučký" will be indexed as "brulee" and "zlutoucky".<br/>
<strong>Changing this will trigger a full reindex.</strong>`
<strong>Needs a restart to fully take effect.</strong>`
})
new Setting(containerEl)
.setName('Ignore diacritics')
@@ -84,7 +83,6 @@ export class SettingsTab extends PluginSettingTab {
toggle.setValue(settings.ignoreDiacritics).onChange(async v => {
settings.ignoreDiacritics = v
await saveSettings(this.plugin)
await initGlobalSearchIndex(true)
})
)
@@ -93,7 +91,6 @@ export class SettingsTab extends PluginSettingTab {
indexedFileTypesDesc.createSpan({}, span => {
span.innerHTML = `In addition to standard <code>md</code> files, Omnisearch can also index other plain text files.<br/>
Add extensions separated by a space. Example: <code>txt org</code>.<br />
This setting will also add these files in the navigation, and they will be treated as markdown.<br />
<strong>Needs a restart to fully take effect.</strong>`
})
new Setting(containerEl)
@@ -114,7 +111,8 @@ export class SettingsTab extends PluginSettingTab {
indexPDFsDesc.createSpan({}, span => {
span.innerHTML = `Omnisearch will index your PDFs, and return them in search results.
This feature is currently a work-in-progress, please report slowdowns or issues that you might experience.<br>
<strong>Changing this will trigger a full reindex.</strong>`
PDFs being quite slow to index, <strong style="color: var(--text-accent)">it is strongly recommended to also enable "Store index in file"</strong>.<br>
<strong>Needs a restart to fully take effect.</strong>`
})
new Setting(containerEl)
.setName('BETA - Index PDFs')
@@ -123,9 +121,9 @@ export class SettingsTab extends PluginSettingTab {
toggle.setValue(settings.indexPDFs).onChange(async v => {
settings.indexPDFs = v
await saveSettings(this.plugin)
await initGlobalSearchIndex(true)
})
)
// Store index
const serializedIndexDesc = new DocumentFragment()
serializedIndexDesc.createSpan({}, span => {
@@ -133,7 +131,7 @@ export class SettingsTab extends PluginSettingTab {
This results in faster loading times for bigger vaults and mobile devices.<br />
<em>⚠️ Note: the index can become corrupted - if you notice any issue, disable and re-enable this option to clear the cache.</em><br/>
<em>⚠️ Cache files in <code>.obsidian/plugins/omnisearch/</code> must not be synchronized.</em><br/>
<strong>Changing this will trigger a full reindex.</strong>
<strong>Needs a restart to fully take effect.</strong>
`
})
new Setting(containerEl)
@@ -141,11 +139,18 @@ export class SettingsTab extends PluginSettingTab {
.setDesc(serializedIndexDesc)
.addToggle(toggle =>
toggle.setValue(settings.storeIndexInFile).onChange(async v => {
try {
await app.vault.adapter.remove(notesCacheFilePath)
} catch (e) {
console.warn(e)
}
try {
await app.vault.adapter.remove(searchIndexFilePath)
} catch (e) {
console.warn(e)
}
settings.storeIndexInFile = v
await saveSettings(this.plugin)
await initGlobalSearchIndex(true)
})
)
@@ -311,7 +316,7 @@ export const DEFAULT_SETTINGS: OmnisearchSettings = {
storeIndexInFile: false,
welcomeMessage: ''
welcomeMessage: '',
} as const
export let settings = Object.assign({}, DEFAULT_SETTINGS) as OmnisearchSettings

14
src/utils.ts
View File

@@ -173,11 +173,15 @@ export function getCtrlKeyLabel(): 'ctrl' | '⌘' {
}
export function isFileIndexable(path: string): boolean {
return (
path.endsWith('.md') ||
(settings.indexPDFs && path.endsWith('.pdf')) ||
settings.indexedFileTypes.some(t => path.endsWith(`.${t}`))
)
return (settings.indexPDFs && path.endsWith('.pdf')) || isFilePlaintext(path)
}
export function isFilePlaintext(path: string): boolean {
return getPlaintextExtensions().some(t => path.endsWith(`.${t}`))
}
export function getPlaintextExtensions(): string[] {
return [...settings.indexedFileTypes, 'md']
}
export function getExtension(path: string): string {