tanner/obsidian-tannersearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Squashed commit of the following:

Browse Source 
commit 3b229cad538ad88ef2d366964c4261bc0e02fb7c
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Sat Nov 5 14:30:08 2022 +0100

    1.8.0-beta.1

commit f43c369b2dd0a1083b171724e3f7466429505629
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Sat Nov 5 13:39:45 2022 +0100

    Squashed commit of the following:

    commit 93508ee95046385baf62475e5bd835ed9fafe6d3
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Sat Nov 5 13:35:56 2022 +0100

        Cleaning

    commit 205e6a7cce4c1939338820f366f7ae8a067ec7fb
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Fri Nov 4 08:53:46 2022 +0100

        Added logs

    commit ea19b94e164581829908ac71d09a60e230925a7f
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:27:24 2022 +0100

        Notices

    commit 53ff4e822b3c292a56da150b94a1cfe43e199d44
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:27:09 2022 +0100

        Custom minisearch build + Notice when the cache could be corrupted

    commit 498408afd1c350dd68969318c3533fff8aa6c172
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:26:22 2022 +0100

        Added a button to manually clear the cache

    commit 90afe5d3868989626ba4613b064e24ac7efa88be
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 22:03:41 2022 +0100

        Optimized loading minisearch from cache

    commit 719dcb9c82f09f56dabb828ac13c9c1db7f795bb
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Thu Nov 3 21:43:49 2022 +0100

        #92 - Refactored cache to make it behave like pre-indexedDb

    commit 2164ccfa39d83eef23231d01e8aa35ac30e0d31c
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Wed Nov 2 23:13:59 2022 +0100

        Removed cache & tmp engine

    commit 50eb33bbd4d074be9a9952eaf871cd8f58b327e6
    Author: Simon Cambier <simon.cambier@protonmail.com>
    Date:   Wed Nov 2 22:56:04 2022 +0100

        More efficient loading of PDFs

commit a6342a675f
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Wed Nov 2 10:34:02 2022 +0100

    #120 - Cleaning of old cache databases

commit b6890567f3
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Mon Oct 31 17:28:17 2022 +0100

    Updated Readme
This commit is contained in:
Simon Cambier
2022-11-05 14:58:25 +01:00
parent 777b172904
commit 087ec5cc99
12 changed files with 367 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
src/file-loader.ts
View File

@@ -11,6 +11,7 @@ import type { TFile } from 'obsidian'
import type { IndexedDocument } from './globals'
import { pdfManager } from './pdf/pdf-manager'
import { getNonExistingNotes } from './tools/notes'
import { database } from './database'
/**
* Return all plaintext files as IndexedDocuments
@@ -21,7 +22,7 @@ export async function getPlainTextFiles(): Promise<IndexedDocument[]> {
for (const file of allFiles) {
const doc = await fileToIndexedDocument(file)
data.push(doc)
await cacheManager.updateDocument(file.path, doc)
await cacheManager.updateLiveDocument(file.path, doc)
}
return data
}
@@ -31,15 +32,19 @@ export async function getPlainTextFiles(): Promise<IndexedDocument[]> {
* If a PDF isn't cached, it will be read from the disk and added to the IndexedDB
*/
export async function getPDFFiles(): Promise<IndexedDocument[]> {
const allFiles = app.vault.getFiles().filter(f => f.path.endsWith('.pdf'))
const data: IndexedDocument[] = []
const fromDisk = app.vault.getFiles().filter(f => f.path.endsWith('.pdf'))
const fromDb = await database.pdf.toArray()
const data: IndexedDocument[] = []
const input = []
for (const file of allFiles) {
for (const file of fromDisk) {
input.push(
NotesIndex.processQueue(async () => {
const doc = await fileToIndexedDocument(file)
await cacheManager.updateDocument(file.path, doc)
const doc = await fileToIndexedDocument(
file,
fromDb.find(o => o.path === file.path)?.text
)
await cacheManager.updateLiveDocument(file.path, doc)
data.push(doc)
})
)
@@ -52,38 +57,45 @@ export async function getPDFFiles(): Promise<IndexedDocument[]> {
* Convert a file into an IndexedDocument.
* Will use the cache if possible.
* @param file
* @param content If we give a text content, will skip the fetching part
*/
export async function fileToIndexedDocument(
file: TFile
file: TFile,
content?: string
): Promise<IndexedDocument> {
let content: string
if (isFilePlaintext(file.path)) {
content = removeDiacritics(await app.vault.cachedRead(file))
} else if (file.path.endsWith('.pdf')) {
content = removeDiacritics(await pdfManager.getPdfText(file))
} else {
throw new Error('Invalid file: ' + file.path)
if (!content) {
if (isFilePlaintext(file.path)) {
content = await app.vault.cachedRead(file)
} else if (file.path.endsWith('.pdf')) {
content = await pdfManager.getPdfText(file)
} else {
throw new Error('Invalid file: ' + file.path)
}
}
content = removeDiacritics(content)
const metadata = app.metadataCache.getFileCache(file)
// EXCALIDRAW
// Remove the json code
if (metadata?.frontmatter?.['excalidraw-plugin']) {
const comments = metadata.sections?.filter(s => s.type === 'comment') ?? []
for (const { start, end } of comments.map(c => c.position)) {
content = content.substring(0, start.offset-1) + content.substring(end.offset)
}
}
// Look for links that lead to non-existing files,
// and add them to the index.
if (metadata) {
const nonExisting = getNonExistingNotes(file, metadata)
for (const name of nonExisting.filter(o => !cacheManager.getDocument(o))) {
for (const name of nonExisting.filter(
o => !cacheManager.getLiveDocument(o)
)) {
NotesIndex.addNonExistingToIndex(name, file.path)
}
// EXCALIDRAW
// Remove the json code
if (metadata.frontmatter?.['excalidraw-plugin']) {
const comments =
metadata.sections?.filter(s => s.type === 'comment') ?? []
for (const { start, end } of comments.map(c => c.position)) {
content =
content.substring(0, start.offset - 1) + content.substring(end.offset)
}
}
}
return {