tanner/obsidian-tannersearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Squashed commit of the following:

Browse Source 
commit 603b9bbde4c6efc90c81032e4e765c64d3075e75
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Tue Oct 11 21:47:03 2022 +0200

    Basic PDF indexing ok

commit 200331bb5c5111493af1e1f6ef8cd4bbfbdbfd4f
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Tue Oct 11 20:56:44 2022 +0200

    Tweaks and comments

commit 434b9662d40c5fea9d8b28d43828b11916db8c94
Author: Simon Cambier <simon.cambier@ores.be>
Date:   Tue Oct 11 16:22:55 2022 +0200

    Refactoring notes & minisearch cache

commit 7253c676c8ed161782ba8e33f0c4c162880925ad
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Tue Oct 11 09:50:33 2022 +0200

    wip

commit 77736e6ef6f28ccfddb64fb768732927d43bbd77
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Mon Oct 10 20:49:02 2022 +0200

    Small rewrites & deps updates

commit 59845fdb89eb6a3ad3f3f9ad75b39e7a3e604c45
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Mon Oct 10 12:22:11 2022 +0200

    wasm + worker ok

commit 1cf3b506e56147586cd0ebcc003642c5230e04cc
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Sun Oct 2 20:04:49 2022 +0200

    no disk access, of course

commit eb3dd9dd4f616a479a53e10856f6c96c6725e911
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Sun Oct 2 19:08:48 2022 +0200

    Rollup build ok

commit 54f2b7e615456c0e1b1504691689d1ba2c72d9e8
Author: Simon Cambier <simon.cambier@protonmail.com>
Date:   Sun Oct 2 16:03:31 2022 +0200

    Rollup build + wasm PoC
This commit is contained in:
Simon Cambier
2022-10-11 21:54:11 +02:00
parent cf7f6af257
commit 7ddae6dc08
28 changed files with 18437 additions and 923 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@@ -20,3 +20,8 @@ data.json
dist dist
.pnpm-debug.log .pnpm-debug.log
coverage coverage
# Added by cargo
/target

608
Cargo.lock generated Normal file
View File

@@ -0,0 +1,608 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "adobe-cmap-parser"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3aaf5066d68c8ec9656cfd3a96bc9de83d4883f183d6c6b8d742e36a4819dda"
dependencies = [
"pom 1.1.0",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "base-x"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270"
[[package]]
name = "bumpalo"
version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "const_fn"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935"
[[package]]
name = "crc32fast"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if",
]
[[package]]
name = "discard"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0"
[[package]]
name = "dtoa"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
[[package]]
name = "encoding"
version = "0.2.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
dependencies = [
"encoding-index-japanese",
"encoding-index-korean",
"encoding-index-simpchinese",
"encoding-index-singlebyte",
"encoding-index-tradchinese",
]
[[package]]
name = "encoding-index-japanese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-korean"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-simpchinese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-singlebyte"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-tradchinese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding_index_tests"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
[[package]]
name = "euclid"
version = "0.20.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad"
dependencies = [
"num-traits",
]
[[package]]
name = "flate2"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]]
name = "itoa"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "itoa"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
[[package]]
name = "js-sys"
version = "0.3.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "libc"
version = "0.2.134"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb"
[[package]]
name = "linked-hash-map"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a"
[[package]]
name = "log"
version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if",
]
[[package]]
name = "lopdf"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b49a0272112719d0037ab63d4bb67f73ba659e1e90bc38f235f163a457ac16f3"
dependencies = [
"dtoa",
"encoding",
"flate2",
"itoa 0.4.8",
"linked-hash-map",
"log",
"lzw",
"pom 3.2.0",
"time",
]
[[package]]
name = "lzw"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084"
[[package]]
name = "miniz_oxide"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
dependencies = [
"adler",
]
[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg",
]
[[package]]
name = "obsidian-search"
version = "0.1.0"
dependencies = [
"js-sys",
"pdf-extract",
"wasm-bindgen",
]
[[package]]
name = "once_cell"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
[[package]]
name = "pdf-extract"
version = "0.6.5-alpha.0"
source = "git+https://github.com/scambier/pdf-extract#8f01969a0bb49bd71195dd4fd5c87a4a0b5f4b48"
dependencies = [
"adobe-cmap-parser",
"encoding",
"euclid",
"linked-hash-map",
"lopdf",
"postscript",
"type1-encoding-parser",
"unicode-normalization",
]
[[package]]
name = "pom"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6"
[[package]]
name = "pom"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07e2192780e9f8e282049ff9bffcaa28171e1cb0844f49ed5374e518ae6024ec"
[[package]]
name = "postscript"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac1825c05c4f9e2f781202d1a02fff5e5f722bbafca542d818364e1b1ea22575"
[[package]]
name = "proc-macro-hack"
version = "0.5.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
[[package]]
name = "proc-macro2"
version = "1.0.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rustc_version"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
dependencies = [
"semver",
]
[[package]]
name = "ryu"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
[[package]]
name = "semver"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
dependencies = [
"semver-parser",
]
[[package]]
name = "semver-parser"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "serde"
version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b"
[[package]]
name = "serde_derive"
version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
dependencies = [
"itoa 1.0.3",
"ryu",
"serde",
]
[[package]]
name = "sha1"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770"
dependencies = [
"sha1_smol",
]
[[package]]
name = "sha1_smol"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
[[package]]
name = "standback"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff"
dependencies = [
"version_check",
]
[[package]]
name = "stdweb"
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5"
dependencies = [
"discard",
"rustc_version",
"stdweb-derive",
"stdweb-internal-macros",
"stdweb-internal-runtime",
"wasm-bindgen",
]
[[package]]
name = "stdweb-derive"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef"
dependencies = [
"proc-macro2",
"quote",
"serde",
"serde_derive",
"syn",
]
[[package]]
name = "stdweb-internal-macros"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11"
dependencies = [
"base-x",
"proc-macro2",
"quote",
"serde",
"serde_derive",
"serde_json",
"sha1",
"syn",
]
[[package]]
name = "stdweb-internal-runtime"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
[[package]]
name = "syn"
version = "1.0.101"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "time"
version = "0.2.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242"
dependencies = [
"const_fn",
"libc",
"standback",
"stdweb",
"time-macros",
"version_check",
"winapi",
]
[[package]]
name = "time-macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1"
dependencies = [
"proc-macro-hack",
"time-macros-impl",
]
[[package]]
name = "time-macros-impl"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f"
dependencies = [
"proc-macro-hack",
"proc-macro2",
"quote",
"standback",
"syn",
]
[[package]]
name = "tinyvec"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "type1-encoding-parser"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3d6cc09e1a99c7e01f2afe4953789311a1c50baebbdac5b477ecf78e2e92a5b"
dependencies = [
"pom 1.1.0",
]
[[package]]
name = "unicode-ident"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
[[package]]
name = "unicode-normalization"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
dependencies = [
"tinyvec",
]
[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasm-bindgen"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

18
Cargo.toml Normal file
View File

@@ -0,0 +1,18 @@
[package]
name = "obsidian-search"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
crate-type = ["cdylib"]
[dependencies]
wasm-bindgen = "0.2"
js-sys = "0.3.49"
pdf-extract = { git = "https://github.com/scambier/pdf-extract" }
[profile.release]
lto = true
opt-level = 'z'

71
esbuild.config.mjs
View File

@@ -1,71 +0,0 @@
import esbuild from 'esbuild'
import sveltePlugin from 'esbuild-svelte'
import sveltePreprocess from 'svelte-preprocess'
import { copy } from 'esbuild-plugin-copy'
import process from 'process'
import builtins from 'builtin-modules'
import path from 'path'
const banner = `/*
THIS IS A GENERATED/BUNDLED FILE BY ESBUILD
if you want to view the source, please visit the github repository of this plugin
*/
`
const prod = process.argv[2] === 'production'
esbuild
.build({
banner: {
js: banner,
},
entryPoints: ['./src/main.ts'],
bundle: true,
external: [
'obsidian',
'electron',
'@codemirror/autocomplete',
'@codemirror/closebrackets',
'@codemirror/collab',
'@codemirror/commands',
'@codemirror/comment',
'@codemirror/fold',
'@codemirror/gutter',
'@codemirror/highlight',
'@codemirror/history',
'@codemirror/language',
'@codemirror/lint',
'@codemirror/matchbrackets',
'@codemirror/panel',
'@codemirror/rangeset',
'@codemirror/rectangular-selection',
'@codemirror/search',
'@codemirror/state',
'@codemirror/stream-parser',
'@codemirror/text',
'@codemirror/tooltip',
'@codemirror/view',
...builtins,
],
outfile: path.join('./dist', 'main.js'),
plugins: [
sveltePlugin({
preprocess: sveltePreprocess(),
}),
copy({
assets: {
from: ['./assets/styles.css', './manifest.json'],
to: ['./'],
},
}),
],
format: 'cjs',
watch: !prod,
target: 'chrome98',
logLevel: 'info',
sourcemap: prod ? false : 'inline',
treeShaking: true,
minify: prod,
legalComments: 'none',
})
.catch(() => process.exit(1))

16523
package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

36
package.json
View File

@@ -4,8 +4,8 @@
"description": "A search engine for Obsidian", "description": "A search engine for Obsidian",
"main": "dist/main.js", "main": "dist/main.js",
"scripts": { "scripts": {
"dev": "pnpm run check && node esbuild.config.mjs", "dev": "wasm-pack build --target web && rollup -c -w",
"build": "pnpm run check && node esbuild.config.mjs production", "build": "wasm-pack build --target web && rollup -c",
"check": "tsc -noEmit -skipLibCheck", "check": "tsc -noEmit -skipLibCheck",
"version": "node version-bump.mjs && git add manifest.json versions.json package.json", "version": "node version-bump.mjs && git add manifest.json versions.json package.json",
"test": "jest" "test": "jest"
@@ -14,31 +14,41 @@
"author": "Simon Cambier", "author": "Simon Cambier",
"license": "GPL-3", "license": "GPL-3",
"devDependencies": { "devDependencies": {
"@babel/preset-env": "^7.19.0", "@babel/preset-env": "^7.19.4",
"@babel/preset-typescript": "^7.18.6", "@babel/preset-typescript": "^7.18.6",
"@rollup/plugin-commonjs": "^23.0.0",
"@rollup/plugin-node-resolve": "^13.3.0",
"@rollup/plugin-typescript": "^8.5.0",
"@testing-library/jest-dom": "^5.16.5", "@testing-library/jest-dom": "^5.16.5",
"@tsconfig/svelte": "^3.0.0", "@tsconfig/svelte": "^3.0.0",
"@types/jest": "^27.5.2", "@types/jest": "^27.5.2",
"@types/node": "^16.11.58", "@types/lodash-es": "^4.17.6",
"@types/node": "^16.11.64",
"@types/pako": "^2.0.0",
"babel-jest": "^27.5.1", "babel-jest": "^27.5.1",
"builtin-modules": "^3.3.0", "builtin-modules": "^3.3.0",
"esbuild": "0.13.12",
"esbuild-plugin-copy": "^1.3.0",
"esbuild-svelte": "^0.7.1",
"jest": "^27.5.1", "jest": "^27.5.1",
"obsidian": "latest", "obsidian": "latest",
"prettier": "^2.7.1", "prettier": "^2.7.1",
"prettier-plugin-svelte": "^2.7.0", "prettier-plugin-svelte": "^2.8.0",
"svelte": "^3.50.1", "rollup": "^2.79.1",
"rollup-plugin-base64": "^1.0.1",
"rollup-plugin-copy": "^3.4.0",
"rollup-plugin-svelte": "^7.1.0",
"rollup-plugin-terser": "^7.0.2",
"rollup-plugin-web-worker-loader": "^1.6.1",
"svelte": "^3.51.0",
"svelte-jester": "^2.3.2", "svelte-jester": "^2.3.2",
"svelte-preprocess": "^4.10.7", "svelte-preprocess": "^4.10.7",
"tslib": "2.3.1", "tslib": "2.3.1",
"typescript": "^4.8.3" "typescript": "^4.8.4"
}, },
"dependencies": { "dependencies": {
"@vanakat/plugin-api": "^0.1.0", "@vanakat/plugin-api": "0.1.0",
"minisearch": "^5.0.0", "lodash-es": "4.17.21",
"pdfjs-dist": "^2.16.105" "minisearch": "5.0.0",
"p-queue-compat": "1.0.187",
"pako": "^2.0.4"
}, },
"pnpm": { "pnpm": {
"overrides": { "overrides": {

1281
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

46
rollup.config.js Normal file
View File

@@ -0,0 +1,46 @@
import { nodeResolve } from '@rollup/plugin-node-resolve'
import commonjs from '@rollup/plugin-commonjs'
import { base64 } from 'rollup-plugin-base64'
import typescript from '@rollup/plugin-typescript'
import svelte from 'rollup-plugin-svelte'
import autoPreprocess from 'svelte-preprocess'
import copy from 'rollup-plugin-copy'
import { terser } from 'rollup-plugin-terser'
import webWorkerLoader from 'rollup-plugin-web-worker-loader'
const banner = `/*
THIS IS A GENERATED/BUNDLED FILE BY ROLLUP
if you want to view the source visit the plugins github repository
*/
`
const production = !process.env.ROLLUP_WATCH
export default {
input: './src/main.ts',
output: {
file: './dist/main.js',
sourcemap: !production && 'inline',
format: 'cjs',
exports: 'default',
banner,
},
external: ['obsidian'],
plugins: [
nodeResolve({ browser: true }),
svelte({
preprocess: autoPreprocess(),
}),
typescript(),
commonjs(),
base64({ include: '**/*.wasm' }),
copy({
targets: [
{ src: 'assets/styles.css', dest: 'dist' },
{ src: 'manifest.json', dest: 'dist' },
],
}),
webWorkerLoader({ inline: true, forceInline: true, targetPlatform: "browser" }),
production && terser(),
],
}

4
src/api.ts
View File

@@ -1,6 +1,6 @@
import type { ResultNote, SearchMatch } from './globals' import type { ResultNote, SearchMatch } from './globals'
import { Query } from './query' import { Query } from './query'
import { getSuggestions } from './search' import * as Search from './search'
type ResultNoteApi = { type ResultNoteApi = {
score: number score: number
@@ -30,7 +30,7 @@ function mapResults(results: ResultNote[]): ResultNoteApi[] {
async function search(q: string): Promise<ResultNoteApi[]> { async function search(q: string): Promise<ResultNoteApi[]> {
const query = new Query(q) const query = new Query(q)
const raw = await getSuggestions(query) const raw = await Search.getSuggestions(query)
return mapResults(raw) return mapResults(raw)
} }

120
src/cache-manager.ts Normal file
View File

@@ -0,0 +1,120 @@
import { throttle } from 'lodash-es'
import type MiniSearch from 'minisearch'
import type { TFile } from 'obsidian'
import { deflate, inflate } from 'pako'
import {
notesCacheFilePath,
minisearchCacheFilePath,
type IndexedNote,
} from './globals'
import { settings } from './settings'
class CacheManager {
notesCache: Record<string, IndexedNote> = {}
compress = true
writeInterval = 5_000 // In milliseconds
//#region Minisearch
/**
* Serializes and writes the Minisearch index on the disk
*/
public writeMinisearchIndex = throttle(
this._writeMinisearchIndex,
this.writeInterval,
{
leading: true,
trailing: true,
}
)
private async _writeMinisearchIndex(minisearch: MiniSearch): Promise<void> {
if (!settings.persistCache) {
return
}
const json = JSON.stringify(minisearch)
const data = this.compress ? deflate(json) : json
await app.vault.adapter.writeBinary(minisearchCacheFilePath, data as any)
console.log('Omnisearch - Minisearch index saved on disk')
}
public async readMinisearchIndex(): Promise<string | null> {
if (!settings.persistCache) {
return null
}
if (await app.vault.adapter.exists(minisearchCacheFilePath)) {
try {
const data = await app.vault.adapter.readBinary(minisearchCacheFilePath)
return (
this.compress ? new TextDecoder('utf8').decode(inflate(data)) : data
) as any
} catch (e) {
console.trace(
'Omnisearch - Could not load MiniSearch index from the file:'
)
console.warn(e)
app.vault.adapter.remove(minisearchCacheFilePath)
}
}
return null
}
//#endregion Minisearch
public async loadNotesCache() {
if (!settings.persistCache) {
return null
}
if (await app.vault.adapter.exists(notesCacheFilePath)) {
try {
const data = await app.vault.adapter.readBinary(notesCacheFilePath)
const json = (
this.compress ? new TextDecoder('utf8').decode(inflate(data)) : data
) as any
this.notesCache = JSON.parse(json)
} catch (e) {
console.trace('Omnisearch - Could not load notes cache:')
console.warn(e)
app.vault.adapter.remove(notesCacheFilePath)
}
}
return null
}
public saveNotesCache = throttle(this._saveNotesCache, this.writeInterval, {
leading: true,
trailing: true,
})
private async _saveNotesCache() {
if (!settings.persistCache) {
return
}
const json = JSON.stringify(this.notesCache)
const data = this.compress ? deflate(json) : json
await app.vault.adapter.writeBinary(notesCacheFilePath, data as any)
console.log('Omnisearch - Notes cache saved on disk')
}
public addNoteToCache(path: string, note: IndexedNote) {
this.notesCache[path] = note
this.saveNotesCache()
}
public removeNoteFromCache(key: string): void {
delete this.notesCache[key]
}
public getNoteFromCache(key: string): IndexedNote | undefined {
return this.notesCache[key]
}
public getNonExistingNotesFromCache(): IndexedNote[] {
return Object.values(this.notesCache).filter(note => note.doesNotExist)
}
public isCacheOutdated(file: TFile): boolean {
const indexedNote = this.getNoteFromCache(file.path)
return !indexedNote || indexedNote.mtime !== file.stat.mtime
}
}
export const cacheManager = new CacheManager()

22
src/components/ModalInFile.svelte
View File

@@ -13,13 +13,13 @@
import { loopIndex } from 'src/utils' import { loopIndex } from 'src/utils'
import { onDestroy, onMount, tick } from 'svelte' import { onDestroy, onMount, tick } from 'svelte'
import { MarkdownView } from 'obsidian' import { MarkdownView } from 'obsidian'
import { getSuggestions } from 'src/search' import * as Search from 'src/search'
import ModalContainer from './ModalContainer.svelte' import ModalContainer from './ModalContainer.svelte'
import { OmnisearchInFileModal, OmnisearchVaultModal } from 'src/modals' import { OmnisearchInFileModal, OmnisearchVaultModal } from 'src/modals'
import ResultItemInFile from './ResultItemInFile.svelte' import ResultItemInFile from './ResultItemInFile.svelte'
import { Query } from 'src/query' import { Query } from 'src/query'
import { openNote } from 'src/notes' import { openNote } from 'src/notes'
import {saveSearchHistory} from "../search-history"; import { saveSearchHistory } from '../search-history'
export let modal: OmnisearchInFileModal export let modal: OmnisearchInFileModal
export let parent: OmnisearchVaultModal | null = null export let parent: OmnisearchVaultModal | null = null
@@ -50,7 +50,7 @@
$: (async () => { $: (async () => {
if (searchQuery) { if (searchQuery) {
query = new Query(searchQuery) query = new Query(searchQuery)
note = (await getSuggestions(query, { singleFilePath }))[0] ?? null note = (await Search.getSuggestions(query, { singleFilePath }))[0] ?? null
lastSearch = searchQuery lastSearch = searchQuery
} }
selectedIndex = 0 selectedIndex = 0
@@ -143,20 +143,20 @@
</script> </script>
<InputSearch <InputSearch
value={searchQuery} value="{searchQuery}"
on:input={e => (searchQuery = e.detail)} on:input="{e => (searchQuery = e.detail)}"
placeholder="Omnisearch - File" /> placeholder="Omnisearch - File" />
<ModalContainer> <ModalContainer>
{#if groupedOffsets.length && note} {#if groupedOffsets.length && note}
{#each groupedOffsets as offset, i} {#each groupedOffsets as offset, i}
<ResultItemInFile <ResultItemInFile
{offset} offset="{offset}"
{note} note="{note}"
index={i} index="{i}"
selected={i === selectedIndex} selected="{i === selectedIndex}"
on:mousemove={_e => (selectedIndex = i)} on:mousemove="{_e => (selectedIndex = i)}"
on:click={openSelection} /> on:click="{openSelection}" />
{/each} {/each}
{:else} {:else}
<div style="text-align: center;"> <div style="text-align: center;">

8
src/components/ModalVault.svelte
View File

@@ -5,14 +5,14 @@
import ModalContainer from './ModalContainer.svelte' import ModalContainer from './ModalContainer.svelte'
import { eventBus, type ResultNote } from 'src/globals' import { eventBus, type ResultNote } from 'src/globals'
import { createNote, openNote } from 'src/notes' import { createNote, openNote } from 'src/notes'
import { getSuggestions } from 'src/search' import * as Search from 'src/search'
import { getCtrlKeyLabel, getExtension, loopIndex } from 'src/utils' import { getCtrlKeyLabel, getExtension, loopIndex } from 'src/utils'
import { OmnisearchInFileModal, type OmnisearchVaultModal } from 'src/modals' import { OmnisearchInFileModal, type OmnisearchVaultModal } from 'src/modals'
import ResultItemVault from './ResultItemVault.svelte' import ResultItemVault from './ResultItemVault.svelte'
import { Query } from 'src/query' import { Query } from 'src/query'
import { saveSearchHistory, searchHistory } from 'src/search-history' import { saveSearchHistory, searchHistory } from 'src/search-history'
import { settings } from '../settings' import { settings } from '../settings'
import { refreshIndex } from '../notes-index' import * as NotesIndex from '../notes-index'
export let modal: OmnisearchVaultModal export let modal: OmnisearchVaultModal
let selectedIndex = 0 let selectedIndex = 0
@@ -29,7 +29,7 @@
} }
onMount(async () => { onMount(async () => {
await refreshIndex() await NotesIndex.refreshIndex()
searchQuery = searchHistory[historySearchIndex] searchQuery = searchHistory[historySearchIndex]
eventBus.enable('vault') eventBus.enable('vault')
eventBus.on('vault', 'enter', openNoteAndCloseModal) eventBus.on('vault', 'enter', openNoteAndCloseModal)
@@ -63,7 +63,7 @@
async function updateResults() { async function updateResults() {
query = new Query(searchQuery) query = new Query(searchQuery)
resultNotes = (await getSuggestions(query)).sort( resultNotes = (await Search.getSuggestions(query)).sort(
(a, b) => b.score - a.score (a, b) => b.score - a.score
) )
selectedIndex = 0 selectedIndex = 0

15
src/components/ResultItemVault.svelte
View File

@@ -1,8 +1,8 @@
<script lang="ts"> <script lang="ts">
import { getNoteFromCache } from 'src/notes' import { cacheManager } from 'src/cache-manager'
import { settings, showExcerpt } from 'src/settings' import { settings, showExcerpt } from 'src/settings'
import type { ResultNote } from '../globals' import type { ResultNote } from '../globals'
import { getMatches } from '../search' import * as Search from '../search'
import { highlighter, makeExcerpt, stringsToRegex } from '../utils' import { highlighter, makeExcerpt, stringsToRegex } from '../utils'
import ResultItemContainer from './ResultItemContainer.svelte' import ResultItemContainer from './ResultItemContainer.svelte'
@@ -10,13 +10,18 @@
export let note: ResultNote export let note: ResultNote
$: reg = stringsToRegex(note.foundWords) $: reg = stringsToRegex(note.foundWords)
$: matches = getMatches(note.content, reg) $: matches = Search.getMatches(note.content, reg)
$: cleanedContent = makeExcerpt(note.content, note.matches[0]?.offset ?? -1) $: cleanedContent = makeExcerpt(note.content, note.matches[0]?.offset ?? -1)
$: glyph = getNoteFromCache(note.path)?.doesNotExist $: glyph = cacheManager.getNoteFromCache(note.path)?.doesNotExist
$: title = settings.showShortName ? note.basename : note.path $: title = settings.showShortName ? note.basename : note.path
</script> </script>
<ResultItemContainer id={note.path} {selected} on:mousemove on:click {glyph}> <ResultItemContainer
id="{note.path}"
selected="{selected}"
on:mousemove
on:click
glyph="{glyph}">
<div> <div>
<span class="omnisearch-result__title"> <span class="omnisearch-result__title">
{@html title.replace(reg, highlighter)} {@html title.replace(reg, highlighter)}

5
src/globals.ts
View File

@@ -12,8 +12,9 @@ export const highlightClass = 'suggestion-highlight omnisearch-highlight'
export const eventBus = new EventBus() export const eventBus = new EventBus()
export const searchIndexFilePath = `${app.vault.configDir}/plugins/omnisearch/searchIndex.json` export const minisearchCacheFilePath = `${app.vault.configDir}/plugins/omnisearch/searchIndex.data`
export const notesCacheFilePath = `${app.vault.configDir}/plugins/omnisearch/notesCache.json` export const notesCacheFilePath = `${app.vault.configDir}/plugins/omnisearch/notesCache.data`
export const pdfCacheFilePath = `${app.vault.configDir}/plugins/omnisearch/pdfCache.data`
export const historyFilePath = `${app.vault.configDir}/plugins/omnisearch/historyCache.json` export const historyFilePath = `${app.vault.configDir}/plugins/omnisearch/historyCache.json`
export const EventNames = { export const EventNames = {

57
src/lib.rs Normal file
View File

@@ -0,0 +1,57 @@
use js_sys::Uint8Array;
use pdf_extract::extract_text_from_mem;
use wasm_bindgen::prelude::*;
mod obsidian;
#[wasm_bindgen]
pub fn extract_pdf_text(arr: Uint8Array) -> String {
// FIXME: return a Result<> here, to throw in JS in case of an error
let txt = match extract_text_from_mem(&arr.to_vec()) {
Ok(txt) => txt,
Err(e) => e.to_string(),
};
txt
}
// #[wasm_bindgen]
// pub struct ExampleCommand {
// id: JsString,
// name: JsString,
// }
// #[wasm_bindgen]
// impl ExampleCommand {
// #[wasm_bindgen(getter)]
// pub fn id(&self) -> JsString {
// self.id.clone()
// }
// #[wasm_bindgen(setter)]
// pub fn set_id(&mut self, id: &str) {
// self.id = JsString::from(id)
// }
// #[wasm_bindgen(getter)]
// pub fn name(&self) -> JsString {
// self.name.clone()
// }
// #[wasm_bindgen(setter)]
// pub fn set_name(&mut self, name: &str) {
// self.name = JsString::from(name)
// }
// pub fn callback(&self) {
// obsidian::Notice::new("hello from rust");
// }
// }
// #[wasm_bindgen]
// pub fn onload(plugin: &obsidian::Plugin) {
// let cmd = ExampleCommand {
// id: JsString::from("example"),
// name: JsString::from("Example"),
// };
// plugin.addCommand(JsValue::from(cmd))
// }

31
src/main.ts
View File

@@ -1,13 +1,15 @@
import { Plugin, TFile } from 'obsidian' import { Plugin, TFile } from 'obsidian'
import { initGlobalSearchIndex } from './search' import * as Search from './search'
import { OmnisearchInFileModal, OmnisearchVaultModal } from './modals' import { OmnisearchInFileModal, OmnisearchVaultModal } from './modals'
import { loadSettings, settings, SettingsTab, showExcerpt } from './settings' import { loadSettings, settings, SettingsTab, showExcerpt } from './settings'
import {eventBus, EventNames} from './globals' import { eventBus, EventNames } from './globals'
import { registerAPI } from '@vanakat/plugin-api' import { registerAPI } from '@vanakat/plugin-api'
import api from './api' import api from './api'
import { loadSearchHistory } from './search-history' import { loadSearchHistory } from './search-history'
import {isFilePlaintext, showWelcomeNotice} from './utils' import { isFilePlaintext, showWelcomeNotice } from './utils'
import { addNoteToReindex, addToIndex, removeFromIndex } from './notes-index' import * as NotesIndex from './notes-index'
import { cacheManager } from './cache-manager'
import { pdfManager } from './pdf-manager'
function _registerAPI(plugin: OmnisearchPlugin): void { function _registerAPI(plugin: OmnisearchPlugin): void {
registerAPI('omnisearch', api, plugin as any) registerAPI('omnisearch', api, plugin as any)
@@ -19,10 +21,12 @@ function _registerAPI(plugin: OmnisearchPlugin): void {
export default class OmnisearchPlugin extends Plugin { export default class OmnisearchPlugin extends Plugin {
async onload(): Promise<void> { async onload(): Promise<void> {
// additional files to index by Omnisearch
await loadSettings(this) await loadSettings(this)
await loadSearchHistory() await loadSearchHistory()
await cacheManager.loadNotesCache()
await pdfManager.loadPDFCache()
_registerAPI(this) _registerAPI(this)
if (settings.ribbonIcon) { if (settings.ribbonIcon) {
@@ -57,35 +61,38 @@ export default class OmnisearchPlugin extends Plugin {
// Listeners to keep the search index up-to-date // Listeners to keep the search index up-to-date
this.registerEvent( this.registerEvent(
this.app.vault.on('create', file => { this.app.vault.on('create', file => {
addToIndex(file) NotesIndex.addToIndexAndCache(file)
}) })
) )
this.registerEvent( this.registerEvent(
this.app.vault.on('delete', file => { this.app.vault.on('delete', file => {
removeFromIndex(file.path) NotesIndex.removeFromIndex(file.path)
}) })
) )
this.registerEvent( this.registerEvent(
this.app.vault.on('modify', async file => { this.app.vault.on('modify', async file => {
addNoteToReindex(file) NotesIndex.addNoteToReindex(file)
}) })
) )
this.registerEvent( this.registerEvent(
this.app.vault.on('rename', async (file, oldPath) => { this.app.vault.on('rename', async (file, oldPath) => {
if (file instanceof TFile && isFilePlaintext(file.path)) { if (file instanceof TFile && isFilePlaintext(file.path)) {
removeFromIndex(oldPath) NotesIndex.removeFromIndex(oldPath)
await addToIndex(file) await NotesIndex.addToIndexAndCache(file)
} }
}) })
) )
await initGlobalSearchIndex() await Search.initGlobalSearchIndex()
}) })
// showWelcomeNotice(this) // showWelcomeNotice(this)
} }
onunload(): void {} onunload(): void {
console.log('Omnisearch - Interrupting PDF indexing')
NotesIndex.pdfQueue.pause()
}
addRibbonButton(): void { addRibbonButton(): void {
this.addRibbonIcon('search', 'Omnisearch', _evt => { this.addRibbonIcon('search', 'Omnisearch', _evt => {

121
src/notes-index.ts
View File

@@ -1,6 +1,5 @@
import {Notice, TAbstractFile, TFile} from 'obsidian' import { Notice, TAbstractFile, TFile } from 'obsidian'
import { import {
canIndexPDFs,
extractHeadingsFromCache, extractHeadingsFromCache,
getAliasesFromMetadata, getAliasesFromMetadata,
getTagsFromMetadata, getTagsFromMetadata,
@@ -9,36 +8,33 @@ import {
removeDiacritics, removeDiacritics,
wait, wait,
} from './utils' } from './utils'
import { import { getNonExistingNotes, removeAnchors } from './notes'
addNoteToCache, import * as PDF from './pdf-manager'
getNonExistingNotes, import type { IndexedNote } from './globals'
getNonExistingNotesFromCache, import { settings } from './settings'
getNoteFromCache, import * as Search from './search'
removeAnchors, import PQueue from 'p-queue-compat'
removeNoteFromCache, import { cacheManager } from './cache-manager'
saveNotesCacheToFile,
} from './notes'
import {getPdfText} from './pdf-parser'
import type {IndexedNote} from './globals'
import {searchIndexFilePath} from './globals'
import {settings} from './settings'
import {minisearchInstance} from './search'
let isIndexChanged: boolean let isIndexChanged: boolean
export const pdfQueue = new PQueue({
concurrency: settings.backgroundProcesses,
})
/** /**
* Adds a file to the index * Adds a file to the index
* @param file * @param file
* @returns * @returns
*/ */
export async function addToIndex(file: TAbstractFile): Promise<void> { export async function addToIndexAndCache(file: TAbstractFile): Promise<void> {
if (!(file instanceof TFile) || !isFileIndexable(file.path)) { if (!(file instanceof TFile) || !isFileIndexable(file.path)) {
return return
} }
// Check if the file was already indexed as non-existent, // Check if the file was already indexed as non-existent,
// and if so, remove it from the index (before adding it again) // and if so, remove it from the index (before adding it again)
if (getNoteFromCache(file.path)?.doesNotExist) { if (cacheManager.getNoteFromCache(file.path)?.doesNotExist) {
removeFromIndex(file.path) removeFromIndex(file.path)
} }
@@ -50,18 +46,20 @@ export async function addToIndex(file: TAbstractFile): Promise<void> {
const metadata = app.metadataCache.getFileCache(file) const metadata = app.metadataCache.getFileCache(file)
if (metadata) { if (metadata) {
const nonExisting = getNonExistingNotes(file, metadata) const nonExisting = getNonExistingNotes(file, metadata)
for (const name of nonExisting.filter(o => !getNoteFromCache(o))) { for (const name of nonExisting.filter(
o => !cacheManager.getNoteFromCache(o)
)) {
addNonExistingToIndex(name, file.path) addNonExistingToIndex(name, file.path)
} }
} }
if (getNoteFromCache(file.path)) { if (cacheManager.getNoteFromCache(file.path)) {
throw new Error(`${file.basename} is already indexed`) throw new Error(`${file.basename} is already indexed`)
} }
let content let content
if (file.path.endsWith('.pdf')) { if (file.path.endsWith('.pdf')) {
content = removeDiacritics(await getPdfText(file as TFile)) content = removeDiacritics(await PDF.pdfManager.getPdfText(file as TFile))
} else { } else {
// Fetch content from the cache to index it as-is // Fetch content from the cache to index it as-is
content = removeDiacritics(await app.vault.cachedRead(file)) content = removeDiacritics(await app.vault.cachedRead(file))
@@ -87,9 +85,9 @@ export async function addToIndex(file: TAbstractFile): Promise<void> {
: '', : '',
} }
minisearchInstance.add(note) Search.minisearchInstance.add(note)
isIndexChanged = true isIndexChanged = true
addNoteToCache(note.path, note) cacheManager.addNoteToCache(note.path, note)
} catch (e) { } catch (e) {
console.trace('Error while indexing ' + file.basename) console.trace('Error while indexing ' + file.basename)
console.error(e) console.error(e)
@@ -105,7 +103,7 @@ export async function addToIndex(file: TAbstractFile): Promise<void> {
export function addNonExistingToIndex(name: string, parent: string): void { export function addNonExistingToIndex(name: string, parent: string): void {
name = removeAnchors(name) name = removeAnchors(name)
const filename = name + (name.endsWith('.md') ? '' : '.md') const filename = name + (name.endsWith('.md') ? '' : '.md')
if (getNoteFromCache(filename)) return if (cacheManager.getNoteFromCache(filename)) return
const note = { const note = {
path: filename, path: filename,
@@ -121,9 +119,9 @@ export function addNonExistingToIndex(name: string, parent: string): void {
doesNotExist: true, doesNotExist: true,
parent, parent,
} as IndexedNote } as IndexedNote
minisearchInstance.add(note) Search.minisearchInstance.add(note)
isIndexChanged = true isIndexChanged = true
addNoteToCache(filename, note) cacheManager.addNoteToCache(filename, note)
} }
/** /**
@@ -135,18 +133,19 @@ export function removeFromIndex(path: string): void {
console.info(`"${path}" is not an indexable file`) console.info(`"${path}" is not an indexable file`)
return return
} }
const note = getNoteFromCache(path) const note = cacheManager.getNoteFromCache(path)
if (note) { if (note) {
minisearchInstance.remove(note) Search.minisearchInstance.remove(note)
isIndexChanged = true isIndexChanged = true
removeNoteFromCache(path) cacheManager.removeNoteFromCache(path)
getNonExistingNotesFromCache() cacheManager
.getNonExistingNotesFromCache()
.filter(n => n.parent === path) .filter(n => n.parent === path)
.forEach(n => { .forEach(n => {
removeFromIndex(n.path) removeFromIndex(n.path)
}) })
} else { } else {
console.warn(`not not found under path ${path}`) console.warn(`Omnisearch - Note not found under path ${path}`)
} }
} }
@@ -157,54 +156,40 @@ export function addNoteToReindex(note: TAbstractFile): void {
} }
export async function refreshIndex(): Promise<void> { export async function refreshIndex(): Promise<void> {
if (settings.showIndexingNotices && notesToReindex.size > 0) { if (notesToReindex.size > 0) {
new Notice(`Omnisearch - Reindexing ${notesToReindex.size} notes`, 2000) if (settings.showIndexingNotices) {
} new Notice(`Omnisearch - Reindexing ${notesToReindex.size} notes`, 2000)
for (const note of notesToReindex) { }
removeFromIndex(note.path) for (const note of notesToReindex) {
await addToIndex(note) removeFromIndex(note.path)
await wait(0) await addToIndexAndCache(note)
} await wait(0)
notesToReindex.clear() }
notesToReindex.clear()
await saveIndexToFile() await cacheManager.writeMinisearchIndex(Search.minisearchInstance)
}
export async function saveIndexToFile(): Promise<void> {
if (settings.storeIndexInFile && minisearchInstance && isIndexChanged) {
const json = JSON.stringify(minisearchInstance)
await app.vault.adapter.write(searchIndexFilePath, json)
console.log('Omnisearch - Index saved on disk')
await saveNotesCacheToFile()
isIndexChanged = false
} }
} }
export async function indexPDFs() { export async function indexPDFs() {
if (canIndexPDFs()) { if (settings.PDFIndexing) {
const start = new Date().getTime()
const files = app.vault.getFiles().filter(f => f.path.endsWith('.pdf')) const files = app.vault.getFiles().filter(f => f.path.endsWith('.pdf'))
if (files.length > 50) { console.time('PDF Indexing')
new Notice(`⚠️ Omnisearch is indexing ${files.length} PDFs. You can experience slowdowns while this work is in progress.`) console.log(`Omnisearch - Indexing ${files.length} PDFs`)
}
const promises: Promise<void>[] = []
for (const file of files) { for (const file of files) {
if (getNoteFromCache(file.path)) { if (cacheManager.getNoteFromCache(file.path)) {
removeFromIndex(file.path) removeFromIndex(file.path)
} }
promises.push(addToIndex(file)) pdfQueue.add(async () => {
await addToIndexAndCache(file)
await cacheManager.writeMinisearchIndex(Search.minisearchInstance)
})
} }
await Promise.all(promises)
// Notice & log await pdfQueue.onEmpty()
const message = `Omnisearch - Indexed ${files.length} PDFs in ${ console.timeEnd('PDF Indexing')
new Date().getTime() - start
}ms`
if (settings.showIndexingNotices) { if (settings.showIndexingNotices) {
new Notice(message) new Notice(`Omnisearch - Indexed ${files.length} PDFs`)
} }
console.log(message)
} }
} }

62
src/notes.ts
View File

@@ -1,55 +1,6 @@
import { type CachedMetadata, MarkdownView, TFile } from 'obsidian' import { type CachedMetadata, MarkdownView, TFile } from 'obsidian'
import {
type IndexedNote,
notesCacheFilePath,
type ResultNote,
} from './globals'
import { stringsToRegex } from './utils' import { stringsToRegex } from './utils'
import { settings } from './settings' import type { ResultNote } from './globals'
/**
* This is an in-memory cache of the notes, with all their computed fields
* used by the search engine.
* This cache allows us to quickly de-index notes when they are deleted or updated.
*/
export let notesCache: Record<string, IndexedNote> = {}
export function resetNotesCache(): void {
notesCache = {}
}
export async function loadNotesCache(): Promise<void> {
if (
settings.storeIndexInFile &&
(await app.vault.adapter.exists(notesCacheFilePath))
) {
try {
const json = await app.vault.adapter.read(notesCacheFilePath)
notesCache = JSON.parse(json)
console.log('Omnisearch - Notes cache loaded from the file')
} catch (e) {
console.trace('Omnisearch - Could not load Notes cache from the file')
console.error(e)
}
}
notesCache ||= {}
}
export function getNoteFromCache(key: string): IndexedNote | undefined {
return notesCache[key]
}
export function getNonExistingNotesFromCache(): IndexedNote[] {
return Object.values(notesCache).filter(note => note.doesNotExist)
}
export function addNoteToCache(filename: string, note: IndexedNote): void {
notesCache[filename] = note
}
export function removeNoteFromCache(key: string): void {
delete notesCache[key]
}
export async function openNote( export async function openNote(
item: ResultNote, item: ResultNote,
@@ -145,14 +96,3 @@ export function getNonExistingNotes(
export function removeAnchors(name: string): string { export function removeAnchors(name: string): string {
return name.split(/[\^#]+/)[0] return name.split(/[\^#]+/)[0]
} }
export async function saveNotesCacheToFile(): Promise<void> {
const json = JSON.stringify(notesCache)
await app.vault.adapter.write(notesCacheFilePath, json)
console.log('Omnisearch - Notes cache saved to the file')
}
export function isCacheOutdated(file: TFile): boolean {
const indexedNote = getNoteFromCache(file.path)
return !indexedNote || indexedNote.mtime !== file.stat.mtime
}

14
src/obsidian.rs Normal file
View File

@@ -0,0 +1,14 @@
use wasm_bindgen::prelude::*;
#[wasm_bindgen(module = "obsidian")]
extern "C" {
pub type Plugin;
#[wasm_bindgen(structural, method)]
pub fn addCommand(this: &Plugin, command: JsValue);
pub type Notice;
#[wasm_bindgen(constructor)]
pub fn new(message: &str) -> Notice;
}

53
src/pdf-manager.ts Normal file
View File

@@ -0,0 +1,53 @@
import type { TFile } from 'obsidian'
import PQueue from 'p-queue-compat'
import PDFWorker from 'web-worker:./pdf-worker.ts'
import { pdfCacheFilePath } from './globals'
import { deflate, inflate } from 'pako'
import { md5 } from './utils'
class PDFManager {
private cache: Map<string, { content: string }> = new Map()
private serializeQueue = new PQueue({ concurrency: 1 })
public async loadPDFCache(): Promise<void> {
if (await app.vault.adapter.exists(pdfCacheFilePath)) {
try {
const data = await app.vault.adapter.readBinary(pdfCacheFilePath)
const json = new TextDecoder('utf8').decode(inflate(data))
this.cache = new Map(JSON.parse(json))
} catch (e) {
console.error(e)
this.cache = new Map()
}
}
}
public async getPdfText(file: TFile): Promise<string> {
const data = new Uint8Array(await app.vault.readBinary(file))
const hash = md5(data)
if (this.cache.has(hash)) {
return this.cache.get(hash)!.content
}
const worker = new PDFWorker({ name: 'PDF Text Extractor' })
return new Promise(async (resolve, reject) => {
// @ts-ignore
worker.postMessage({ data })
worker.onmessage = (evt: any) => {
const txt = evt.data.text
this.updatePDFCache(hash, txt)
resolve(txt)
}
})
}
private async updatePDFCache(hash: string, content: string): Promise<void> {
this.serializeQueue.add(() => {
this.cache.set(hash, { content })
const data = deflate(JSON.stringify(Array.from(this.cache), null, 1))
app.vault.adapter.writeBinary(pdfCacheFilePath, data as any)
})
}
}
export const pdfManager = new PDFManager()

17
src/pdf-parser.ts
View File

@@ -1,17 +0,0 @@
import type { TFile } from 'obsidian'
import PDFJs from 'pdfjs-dist'
import pdfjsWorker from 'pdfjs-dist/build/pdf.worker.entry'
PDFJs.GlobalWorkerOptions.workerSrc = pdfjsWorker
// https://stackoverflow.com/a/59929946
export async function getPdfText(file: TFile): Promise<string> {
const data = await app.vault.readBinary(file)
const doc = await PDFJs.getDocument(data).promise
const pageTexts = Array.from({ length: doc.numPages }, async (v, i) => {
const page = await doc.getPage(i + 1)
const content = await page.getTextContent()
return (content.items as any[]).map(token => token.str).join('')
})
return (await Promise.all(pageTexts)).join('')
}

16
src/pdf-worker.ts Normal file
View File

@@ -0,0 +1,16 @@
import rustPlugin from '../pkg/obsidian_search_bg.wasm'
import * as plugin from '../pkg/obsidian_search'
const decodedPlugin = decodeBase64(rustPlugin as any)
onmessage = async evt => {
const buffer = Uint8Array.from(decodedPlugin, c => c.charCodeAt(0))
await plugin.default(Promise.resolve(buffer))
const text = plugin.extract_pdf_text(evt.data.data as Uint8Array)
self.postMessage({ text })
}
function decodeBase64(data: string) {
return atob(data)
// return Buffer.from(data, 'base64').toString()
}

91
src/search.ts
View File

@@ -4,12 +4,12 @@ import {
chsRegex, chsRegex,
type IndexedNote, type IndexedNote,
type ResultNote, type ResultNote,
searchIndexFilePath, minisearchCacheFilePath,
type SearchMatch, type SearchMatch,
SPACE_OR_PUNCTUATION, SPACE_OR_PUNCTUATION,
} from './globals' } from './globals'
import { import {
canIndexPDFs, isFileIndexable,
isFilePlaintext, isFilePlaintext,
removeDiacritics, removeDiacritics,
stringsToRegex, stringsToRegex,
@@ -18,13 +18,15 @@ import {
} from './utils' } from './utils'
import type { Query } from './query' import type { Query } from './query'
import { settings } from './settings' import { settings } from './settings'
import { // import {
getNoteFromCache, // getNoteFromCache,
isCacheOutdated, // isCacheOutdated,
loadNotesCache, // loadNotesCache,
resetNotesCache, // resetNotesCache,
} from './notes' // } from './notes'
import {addToIndex, indexPDFs, removeFromIndex, saveIndexToFile} from './notes-index' import * as NotesIndex from './notes-index'
import PQueue from 'p-queue-compat'
import { cacheManager } from './cache-manager'
export let minisearchInstance: MiniSearch<IndexedNote> export let minisearchInstance: MiniSearch<IndexedNote>
@@ -60,15 +62,18 @@ export async function initGlobalSearchIndex(): Promise<void> {
storeFields: ['tags'], storeFields: ['tags'],
} }
if ( // Default instance
settings.storeIndexInFile && minisearchInstance = new MiniSearch(options)
(await app.vault.adapter.exists(searchIndexFilePath))
) { // Load Minisearch cache, if it exists
if (await app.vault.adapter.exists(minisearchCacheFilePath)) {
try { try {
const json = await app.vault.adapter.read(searchIndexFilePath) const json = await cacheManager.readMinisearchIndex()
minisearchInstance = MiniSearch.loadJSON(json, options) if (json) {
// If we have cache data, reload it
minisearchInstance = MiniSearch.loadJSON(json, options)
}
console.log('Omnisearch - MiniSearch index loaded from the file') console.log('Omnisearch - MiniSearch index loaded from the file')
await loadNotesCache()
} catch (e) { } catch (e) {
console.trace( console.trace(
'Omnisearch - Could not load MiniSearch index from the file' 'Omnisearch - Could not load MiniSearch index from the file'
@@ -77,10 +82,9 @@ export async function initGlobalSearchIndex(): Promise<void> {
} }
} }
if (!minisearchInstance) { // if (!minisearchInstance) {
minisearchInstance = new MiniSearch(options) // resetNotesCache()
resetNotesCache() // }
}
// Index files that are already present // Index files that are already present
const start = new Date().getTime() const start = new Date().getTime()
@@ -89,32 +93,28 @@ export async function initGlobalSearchIndex(): Promise<void> {
let files let files
let notesSuffix let notesSuffix
if (settings.storeIndexInFile) { if (settings.persistCache) {
files = allFiles.filter(file => isCacheOutdated(file)) files = allFiles.filter(file => cacheManager.isCacheOutdated(file))
notesSuffix = 'modified notes' notesSuffix = 'modified notes'
} else { } else {
files = allFiles files = allFiles
notesSuffix = 'notes' notesSuffix = 'notes'
} }
console.log(`Omnisearch - indexing ${files.length} ${notesSuffix}`) if (files.length > 0) {
console.log(`Omnisearch - Indexing ${files.length} ${notesSuffix}`)
// This is basically the same behavior as MiniSearch's `addAllAsync()`.
// We index markdown and plaintext files by batches of 10
let promises: Promise<void>[] = []
for (let i = 0; i < files.length; ++i) {
const file = files[i]
if (getNoteFromCache(file.path)) {
removeFromIndex(file.path)
}
promises.push(addToIndex(file))
if (i % 10 === 0) {
await wait(1)
await Promise.all(promises)
promises = []
}
} }
await Promise.all(promises)
// Read and index all the files into the search engine
const queue = new PQueue({ concurrency: 10 })
for (const file of files) {
if (cacheManager.getNoteFromCache(file.path)) {
NotesIndex.removeFromIndex(file.path)
}
queue.add(() => NotesIndex.addToIndexAndCache(file))
}
await queue.onEmpty()
if (files.length > 0) { if (files.length > 0) {
const message = `Omnisearch - Indexed ${files.length} ${notesSuffix} in ${ const message = `Omnisearch - Indexed ${files.length} ${notesSuffix} in ${
@@ -127,10 +127,10 @@ export async function initGlobalSearchIndex(): Promise<void> {
new Notice(message) new Notice(message)
} }
await saveIndexToFile() await cacheManager.writeMinisearchIndex(minisearchInstance)
// PDFs are indexed later, since they're heavier // PDFs are indexed later, since they're heavier
await indexPDFs() await NotesIndex.indexPDFs()
} }
} }
@@ -172,9 +172,10 @@ async function search(query: Query): Promise<SearchResult[]> {
const exactTerms = query.getExactTerms() const exactTerms = query.getExactTerms()
if (exactTerms.length) { if (exactTerms.length) {
results = results.filter(r => { results = results.filter(r => {
const title = getNoteFromCache(r.id)?.path.toLowerCase() ?? '' const title =
cacheManager.getNoteFromCache(r.id)?.path.toLowerCase() ?? ''
const content = stripMarkdownCharacters( const content = stripMarkdownCharacters(
getNoteFromCache(r.id)?.content ?? '' cacheManager.getNoteFromCache(r.id)?.content ?? ''
).toLowerCase() ).toLowerCase()
return exactTerms.every(q => content.includes(q) || title.includes(q)) return exactTerms.every(q => content.includes(q) || title.includes(q))
}) })
@@ -185,7 +186,7 @@ async function search(query: Query): Promise<SearchResult[]> {
if (exclusions.length) { if (exclusions.length) {
results = results.filter(r => { results = results.filter(r => {
const content = stripMarkdownCharacters( const content = stripMarkdownCharacters(
getNoteFromCache(r.id)?.content ?? '' cacheManager.getNoteFromCache(r.id)?.content ?? ''
).toLowerCase() ).toLowerCase()
return exclusions.every(q => !content.includes(q.value)) return exclusions.every(q => !content.includes(q.value))
}) })
@@ -253,7 +254,7 @@ export async function getSuggestions(
// Map the raw results to get usable suggestions // Map the raw results to get usable suggestions
return results.map(result => { return results.map(result => {
const note = getNoteFromCache(result.id) const note = cacheManager.getNoteFromCache(result.id)
if (!note) { if (!note) {
throw new Error(`Note "${result.id}" not indexed`) throw new Error(`Note "${result.id}" not indexed`)
} }

114
src/settings.ts
View File

@@ -1,6 +1,6 @@
import { Plugin, PluginSettingTab, Setting, SliderComponent } from 'obsidian' import { Plugin, PluginSettingTab, Setting, SliderComponent } from 'obsidian'
import { writable } from 'svelte/store' import { writable } from 'svelte/store'
import { notesCacheFilePath, searchIndexFilePath } from './globals' import { notesCacheFilePath, minisearchCacheFilePath } from './globals'
import type OmnisearchPlugin from './main' import type OmnisearchPlugin from './main'
interface WeightingSettings { interface WeightingSettings {
@@ -11,20 +11,33 @@ interface WeightingSettings {
} }
export interface OmnisearchSettings extends WeightingSettings { export interface OmnisearchSettings extends WeightingSettings {
/** Respect the "excluded files" Obsidian setting by downranking results ignored files */
respectExcluded: boolean respectExcluded: boolean
/** Ignore diacritics when indexing files */
ignoreDiacritics: boolean ignoreDiacritics: boolean
/** Extensions of plain text files to index, in addition to .md */
indexedFileTypes: string[] indexedFileTypes: string[]
indexPDFs: boolean /** Enable PDF indexing */
storeIndexInFile: boolean PDFIndexing: boolean
/** Max number of spawned processes for background tasks, such as extracting text from PDFs */
backgroundProcesses: number
/** Write cache files on disk (unrelated to PDFs) */
persistCache: boolean
/** Display Omnisearch popup notices over Obsidian */
showIndexingNotices: boolean showIndexingNotices: boolean
/** Activate the small 🔍 button on Obsidian's ribbon */
ribbonIcon: boolean ribbonIcon: boolean
/** Display short filenames in search results, instead of the full path */
showShortName: boolean showShortName: boolean
/** Display the small contextual excerpt in search results */
showExcerpt: boolean showExcerpt: boolean
/** Enable a "create note" button in the Vault Search modal */
showCreateButton: boolean showCreateButton: boolean
/** Vim mode shortcuts */
CtrlJK: boolean CtrlJK: boolean
/** Vim mode shortcuts */
CtrlNP: boolean CtrlNP: boolean
/** Key for the welcome message when Obsidian is updated. A message is only shown once. */
welcomeMessage: string welcomeMessage: string
} }
@@ -74,7 +87,7 @@ export class SettingsTab extends PluginSettingTab {
const diacriticsDesc = new DocumentFragment() const diacriticsDesc = new DocumentFragment()
diacriticsDesc.createSpan({}, span => { diacriticsDesc.createSpan({}, span => {
span.innerHTML = `Normalize diacritics in search terms. Words like "brûlée" or "žluťoučký" will be indexed as "brulee" and "zlutoucky".<br/> span.innerHTML = `Normalize diacritics in search terms. Words like "brûlée" or "žluťoučký" will be indexed as "brulee" and "zlutoucky".<br/>
<strong>Needs a restart to fully take effect.</strong>` <strong style="color: var(--text-accent)">Needs a restart to fully take effect.</strong>`
}) })
new Setting(containerEl) new Setting(containerEl)
.setName('Ignore diacritics') .setName('Ignore diacritics')
@@ -91,7 +104,7 @@ export class SettingsTab extends PluginSettingTab {
indexedFileTypesDesc.createSpan({}, span => { indexedFileTypesDesc.createSpan({}, span => {
span.innerHTML = `In addition to standard <code>md</code> files, Omnisearch can also index other plain text files.<br/> span.innerHTML = `In addition to standard <code>md</code> files, Omnisearch can also index other plain text files.<br/>
Add extensions separated by a space. Example: <code>txt org</code>.<br /> Add extensions separated by a space. Example: <code>txt org</code>.<br />
<strong>Needs a restart to fully take effect.</strong>` <strong style="color: var(--text-accent)">Needs a restart to fully take effect.</strong>`
}) })
new Setting(containerEl) new Setting(containerEl)
.setName('Additional files to index') .setName('Additional files to index')
@@ -106,50 +119,68 @@ export class SettingsTab extends PluginSettingTab {
}) })
}) })
// Index PDFs // // Background processes
const indexPDFsDesc = new DocumentFragment() // new Setting(containerEl)
indexPDFsDesc.createSpan({}, span => { // .setName(
span.innerHTML = `Omnisearch will index your PDFs, and return them in search results. // `Background processes (default: ${DEFAULT_SETTINGS.backgroundProcesses})`
This feature is currently a work-in-progress, please report slowdowns or issues that you might experience.<br> // )
PDFs being quite slow to index, <strong style="color: var(--text-accent)">it is strongly recommended to also enable "Store index in file"</strong>.<br> // .setDesc('The maximum number of processes for background work, like PDF indexing. This value should not be higher than your number of CPU cores.')
<strong>Needs a restart to fully take effect.</strong>` // .addSlider(cb => {
}) // cb.setLimits(1, 16, 1)
new Setting(containerEl) // .setValue(settings.backgroundProcesses)
.setName('BETA - Index PDFs') // .setDynamicTooltip()
.setDesc(indexPDFsDesc) // .onChange(v => {
.addToggle(toggle => // settings.backgroundProcesses = v
toggle.setValue(settings.indexPDFs).onChange(async v => { // saveSettings(this.plugin)
settings.indexPDFs = v // })
await saveSettings(this.plugin) // })
})
)
// Store index // Store index
const serializedIndexDesc = new DocumentFragment() const serializedIndexDesc = new DocumentFragment()
serializedIndexDesc.createSpan({}, span => { serializedIndexDesc.createSpan({}, span => {
span.innerHTML = `The search index is stored on disk, instead of being rebuilt at every startup. span.innerHTML = `This will speedup startup times after the initial indexing. Do not activate it unless indexing is too slow on your device:
This results in faster loading times for bigger vaults and mobile devices.<br /> <ul>
<em>⚠️ Note: the index can become corrupted - if you notice any issue, disable and re-enable this option to clear the cache.</em><br/> <li>PDF indexing is not affected by this setting</li>
<em>⚠️ Cache files in <code>.obsidian/plugins/omnisearch/</code> must not be synchronized.</em><br/> <li>⚠️ The index can become corrupted - if you notice any issue, disable and re-enable this option to clear the cache.</li>
<strong>Needs a restart to fully take effect.</strong> <li>⚠️ Cache files in <code>.obsidian/plugins/omnisearch/*.data</code> must not be synchronized between your devices.</li>
</ul>
<strong style="color: var(--text-accent)">Needs a restart to fully take effect.</strong>
` `
}) })
new Setting(containerEl) new Setting(containerEl)
.setName('Store index in file') .setName('Persist cache on disk')
.setDesc(serializedIndexDesc) .setDesc(serializedIndexDesc)
.addToggle(toggle => .addToggle(toggle =>
toggle.setValue(settings.storeIndexInFile).onChange(async v => { toggle.setValue(settings.persistCache).onChange(async v => {
try { try {
await app.vault.adapter.remove(notesCacheFilePath) await app.vault.adapter.remove(notesCacheFilePath)
} catch (e) { } catch (e) {
console.warn(e) console.warn(e)
} }
try { try {
await app.vault.adapter.remove(searchIndexFilePath) await app.vault.adapter.remove(minisearchCacheFilePath)
} catch (e) { } catch (e) {
console.warn(e) console.warn(e)
} }
settings.storeIndexInFile = v settings.persistCache = v
await saveSettings(this.plugin)
})
)
// PDF Indexing
const indexPDFsDesc = new DocumentFragment()
indexPDFsDesc.createSpan({}, span => {
span.innerHTML = `Omnisearch will include PDFs in search results.
This feature is currently a work-in-progress, please report slowdowns or issues that you might experience.<br>
Each PDF can take a few seconds to be indexed, so it may not appear immediately in search results.<br>
<strong style="color: var(--text-accent)">Needs a restart to fully take effect.</strong>`
})
new Setting(containerEl)
.setName('BETA - PDF Indexing')
.setDesc(indexPDFsDesc)
.addToggle(toggle =>
toggle.setValue(settings.PDFIndexing).onChange(async v => {
settings.PDFIndexing = v
await saveSettings(this.plugin) await saveSettings(this.plugin)
}) })
) )
@@ -285,12 +316,12 @@ export class SettingsTab extends PluginSettingTab {
weightSlider(cb: SliderComponent, key: keyof WeightingSettings): void { weightSlider(cb: SliderComponent, key: keyof WeightingSettings): void {
cb.setLimits(1, 3, 0.1) cb.setLimits(1, 3, 0.1)
cb.setValue(settings[key]) .setValue(settings[key])
cb.setDynamicTooltip() .setDynamicTooltip()
cb.onChange(v => { .onChange(v => {
settings[key] = v settings[key] = v
saveSettings(this.plugin) saveSettings(this.plugin)
}) })
} }
} }
@@ -298,7 +329,8 @@ export const DEFAULT_SETTINGS: OmnisearchSettings = {
respectExcluded: true, respectExcluded: true,
ignoreDiacritics: true, ignoreDiacritics: true,
indexedFileTypes: [] as string[], indexedFileTypes: [] as string[],
indexPDFs: false, PDFIndexing: false,
backgroundProcesses: Math.max(1, Math.floor(require('os').cpus().length / 2)),
showIndexingNotices: false, showIndexingNotices: false,
showShortName: false, showShortName: false,
@@ -314,7 +346,7 @@ export const DEFAULT_SETTINGS: OmnisearchSettings = {
CtrlJK: false, CtrlJK: false,
CtrlNP: false, CtrlNP: false,
storeIndexInFile: false, persistCache: false,
welcomeMessage: '', welcomeMessage: '',
} as const } as const

0
src/types-obsidian.d.ts → src/typings/types-obsidian.d.ts vendored
View File

0
src/types.d.ts → src/typings/types.d.ts vendored
View File

4
src/typings/workers.d.ts vendored Normal file
View File

@@ -0,0 +1,4 @@
declare module "web-worker:*" {
const WorkerFactory: new (options: any) => Worker;
export default WorkerFactory;
}

14
src/utils.ts
View File

@@ -10,6 +10,7 @@ import {
regexYaml, regexYaml,
} from './globals' } from './globals'
import { settings } from './settings' import { settings } from './settings'
import { createHash, type BinaryLike } from 'crypto'
export function highlighter(str: string): string { export function highlighter(str: string): string {
return `<span class="${highlightClass}">${str}</span>` return `<span class="${highlightClass}">${str}</span>`
@@ -172,12 +173,10 @@ export function getCtrlKeyLabel(): 'ctrl' | '⌘' {
return Platform.isMacOS ? '⌘' : 'ctrl' return Platform.isMacOS ? '⌘' : 'ctrl'
} }
export function canIndexPDFs(): boolean {
return settings.indexPDFs
}
export function isFileIndexable(path: string): boolean { export function isFileIndexable(path: string): boolean {
return (canIndexPDFs() && path.endsWith('.pdf')) || isFilePlaintext(path) return (
(settings.PDFIndexing && path.endsWith('.pdf')) || isFilePlaintext(path)
)
} }
export function isFilePlaintext(path: string): boolean { export function isFilePlaintext(path: string): boolean {
@@ -194,6 +193,7 @@ export function getExtension(path: string): string {
} }
export function showWelcomeNotice(plugin: Plugin) { export function showWelcomeNotice(plugin: Plugin) {
return
const code = '1.6.0' const code = '1.6.0'
if (settings.welcomeMessage !== code) { if (settings.welcomeMessage !== code) {
const welcome = new DocumentFragment() const welcome = new DocumentFragment()
@@ -208,3 +208,7 @@ New beta feature: PDF search 🔎📄
plugin.saveData(settings) plugin.saveData(settings)
} }
export function md5(data: BinaryLike): string {
return createHash('md5').update(data).digest('hex')
}