From dacca2a8153181e4587b376448aeab91f92e774e Mon Sep 17 00:00:00 2001 From: frc4533-lincoln <132951735+frc4533-lincoln@users.noreply.github.com> Date: Sun, 27 Oct 2024 15:24:29 -0400 Subject: [PATCH] Added config schema --- Cargo.lock | 678 ++++++++++---------------------------------------- Cargo.toml | 20 +- searched.toml | 15 ++ src/config.rs | 28 +++ src/lib.rs | 9 +- src/main.rs | 10 +- 6 files changed, 191 insertions(+), 569 deletions(-) create mode 100644 searched.toml create mode 100644 src/config.rs diff --git a/Cargo.lock b/Cargo.lock index bc43f08..df685d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,12 +39,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" - [[package]] name = "android-tzdata" version = "0.1.1" @@ -109,18 +103,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "anyhow" -version = "1.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" - -[[package]] -name = "arc-swap" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" - [[package]] name = "async-compression" version = "0.4.17" @@ -256,15 +238,6 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" -[[package]] -name = "bitpacking" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" -dependencies = [ - "crunchy", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -274,17 +247,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bstr" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" -dependencies = [ - "lazy_static", - "memchr", - "regex-automata 0.1.10", -] - [[package]] name = "bstr" version = "1.10.0" @@ -319,17 +281,9 @@ version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" dependencies = [ - "jobserver", - "libc", "shlex", ] -[[package]] -name = "census" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" - [[package]] name = "cfg-if" version = "1.0.0" @@ -438,15 +392,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -472,12 +417,6 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - [[package]] name = "crypto-common" version = "0.1.6" @@ -518,7 +457,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", - "serde", ] [[package]] @@ -548,12 +486,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "downcast-rs" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" - [[package]] name = "dtoa" version = "1.0.9" @@ -639,12 +571,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "fastdivide" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59668941c55e5c186b8b58c391629af56774ec768f73c08bbcd56f09348eb00b" - [[package]] name = "fastrand" version = "2.1.1" @@ -667,12 +593,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" - [[package]] name = "foreign-types" version = "0.3.2" @@ -707,16 +627,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fs4" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" -dependencies = [ - "rustix", - "windows-sys 0.52.0", -] - [[package]] name = "futf" version = "0.1.5" @@ -776,30 +686,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "gc-arena" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd70cf88a32937834aae9614ff2569b5d9467fa0c42c5d7762fd94a8de88266" -dependencies = [ - "allocator-api2", - "gc-arena-derive", - "hashbrown 0.14.5", - "sptr", -] - -[[package]] -name = "gc-arena-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c612a69f5557a11046b77a7408d2836fe77077f842171cd211c5ef504bd3cddd" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.85", - "synstructure", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -834,9 +720,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" dependencies = [ "aho-corasick", - "bstr 1.10.0", + "bstr", "log", - "regex-automata 0.4.8", + "regex-automata", "regex-syntax", ] @@ -870,26 +756,11 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", -] - [[package]] name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] [[package]] name = "hermit-abi" @@ -897,6 +768,15 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "html5ever" version = "0.27.0" @@ -911,12 +791,6 @@ dependencies = [ "syn 2.0.85", ] -[[package]] -name = "htmlescape" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" - [[package]] name = "http" version = "1.1.0" @@ -1104,7 +978,7 @@ dependencies = [ "globset", "log", "memchr", - "regex-automata 0.4.8", + "regex-automata", "same-file", "walkdir", "winapi-util", @@ -1117,7 +991,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown", ] [[package]] @@ -1127,9 +1001,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", ] [[package]] @@ -1144,15 +1015,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -1168,15 +1030,6 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" -[[package]] -name = "jobserver" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.72" @@ -1192,18 +1045,22 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "levenshtein_automata" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" - [[package]] name = "libc" version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +[[package]] +name = "libloading" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +dependencies = [ + "cfg-if", + "windows-targets", +] + [[package]] name = "libm" version = "0.2.9" @@ -1233,12 +1090,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] -name = "lru" -version = "0.12.5" +name = "lua-src" +version = "547.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +checksum = "1edaf29e3517b49b8b746701e5648ccb5785cde1c119062cbabbc5d5cd115e42" dependencies = [ - "hashbrown 0.15.0", + "cc", +] + +[[package]] +name = "luajit-src" +version = "210.5.11+97813fb" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3015551c284515db7c30c559fc1080f9cb9ee990d1f6fca315451a107c7540bb" +dependencies = [ + "cc", + "which", +] + +[[package]] +name = "luau0-src" +version = "0.11.0+luau647" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca47356c40f9121e3b3d582bf7355e8b25751b2e8afdfda49cf05775dfbc6b0a" +dependencies = [ + "cc", ] [[package]] @@ -1267,43 +1143,18 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" -[[package]] -name = "measure_time" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" -dependencies = [ - "instant", - "log", -] - [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "memmap2" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" -dependencies = [ - "libc", -] - [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.8.0" @@ -1331,15 +1182,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f6ddbd668297c46be4bdea6c599dcc1f001a129586272d53170b7ac0a62961e" dependencies = [ - "bstr 1.10.0", + "bstr", "either", "erased-serde", "futures-util", + "libloading", "mlua-sys", "mlua_derive", "num-traits", "parking_lot 0.12.3", - "rustc-hash 2.0.0", + "rustc-hash", "serde", "serde-value", ] @@ -1352,6 +1204,9 @@ checksum = "e9eebac25c35a13285456c88ee2fde93d9aee8bcfdaf03f9d6d12be3391351ec" dependencies = [ "cc", "cfg-if", + "lua-src", + "luajit-src", + "luau0-src", "pkg-config", ] @@ -1361,7 +1216,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cfc5faa2e0d044b3f5f0879be2920e0a711c97744c42cf1c295cb183668933e" dependencies = [ - "itertools 0.13.0", + "itertools", "once_cell", "proc-macro-error", "proc-macro2", @@ -1370,12 +1225,6 @@ dependencies = [ "syn 2.0.85", ] -[[package]] -name = "murmurhash32" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" - [[package]] name = "native-tls" version = "0.2.12" @@ -1399,16 +1248,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "num-conv" version = "0.1.0" @@ -1422,17 +1261,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", ] [[package]] @@ -1450,12 +1278,6 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" -[[package]] -name = "oneshot" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e296cf87e61c9cfc1a61c3c63a0f7f286ed4554e0e22be84e8a38e1d264a2a29" - [[package]] name = "openssl" version = "0.10.68" @@ -1509,15 +1331,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "ownedbytes" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -1716,33 +1529,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "piccolo" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "003bf52de285e1ff1adcbc6572588db3849988ea660a2d55af3a2ffbc81f597f" -dependencies = [ - "ahash", - "allocator-api2", - "anyhow", - "gc-arena", - "hashbrown 0.14.5", - "rand", - "thiserror", -] - -[[package]] -name = "piccolo-util" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4423e61bbe92f7df753455dccf5b21b6b5c0dfd9f3222ed007cfb667ced503a9" -dependencies = [ - "gc-arena", - "piccolo", - "serde", - "thiserror", -] - [[package]] name = "pin-project-lite" version = "0.2.15" @@ -1870,36 +1656,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_distr" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" -dependencies = [ - "num-traits", - "rand", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.2.16" @@ -1926,16 +1682,10 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", + "regex-automata", "regex-syntax", ] -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - [[package]] name = "regex-automata" version = "0.4.8" @@ -2015,28 +1765,12 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.0.0" @@ -2152,24 +1886,19 @@ version = "0.1.0" dependencies = [ "axum", "axum-macros", - "crossbeam-channel", "env_logger", "log", - "lru", "mlua", "once_cell", - "piccolo", - "piccolo-util", "reqwest", "scraper", "serde", "serde_json", "serde_qs", "sled", - "tantivy", "tera", - "texting_robots", "tokio", + "toml", "url", ] @@ -2278,6 +2007,15 @@ dependencies = [ "thiserror", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2331,15 +2069,6 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" -[[package]] -name = "sketches-ddsketch" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" -dependencies = [ - "serde", -] - [[package]] name = "slab" version = "0.4.9" @@ -2397,12 +2126,6 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -[[package]] -name = "sptr" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" - [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2477,17 +2200,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.85", -] - [[package]] name = "system-configuration" version = "0.6.1" @@ -2509,147 +2221,6 @@ dependencies = [ "libc", ] -[[package]] -name = "tantivy" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8d0582f186c0a6d55655d24543f15e43607299425c5ad8352c242b914b31856" -dependencies = [ - "aho-corasick", - "arc-swap", - "base64", - "bitpacking", - "byteorder", - "census", - "crc32fast", - "crossbeam-channel", - "downcast-rs", - "fastdivide", - "fnv", - "fs4", - "htmlescape", - "itertools 0.12.1", - "levenshtein_automata", - "log", - "lru", - "measure_time", - "memmap2", - "num_cpus", - "once_cell", - "oneshot", - "rayon", - "regex", - "rust-stemmers", - "rustc-hash 1.1.0", - "serde", - "serde_json", - "sketches-ddsketch", - "smallvec", - "tantivy-bitpacker", - "tantivy-columnar", - "tantivy-common", - "tantivy-fst", - "tantivy-query-grammar", - "tantivy-stacker", - "tantivy-tokenizer-api", - "tempfile", - "thiserror", - "time", - "uuid", - "winapi", - "zstd", -] - -[[package]] -name = "tantivy-bitpacker" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" -dependencies = [ - "bitpacking", -] - -[[package]] -name = "tantivy-columnar" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" -dependencies = [ - "downcast-rs", - "fastdivide", - "itertools 0.12.1", - "serde", - "tantivy-bitpacker", - "tantivy-common", - "tantivy-sstable", - "tantivy-stacker", -] - -[[package]] -name = "tantivy-common" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" -dependencies = [ - "async-trait", - "byteorder", - "ownedbytes", - "serde", - "time", -] - -[[package]] -name = "tantivy-fst" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" -dependencies = [ - "byteorder", - "regex-syntax", - "utf8-ranges", -] - -[[package]] -name = "tantivy-query-grammar" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" -dependencies = [ - "nom", -] - -[[package]] -name = "tantivy-sstable" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" -dependencies = [ - "tantivy-bitpacker", - "tantivy-common", - "tantivy-fst", - "zstd", -] - -[[package]] -name = "tantivy-stacker" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" -dependencies = [ - "murmurhash32", - "rand_distr", - "tantivy-common", -] - -[[package]] -name = "tantivy-tokenizer-api" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" -dependencies = [ - "serde", -] - [[package]] name = "tempfile" version = "3.13.0" @@ -2696,22 +2267,6 @@ dependencies = [ "unic-segment", ] -[[package]] -name = "texting_robots" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82a718a28dda2e67ad6e0464597b58eae39e2e4d0451e03d1028d71e81bb4a" -dependencies = [ - "anyhow", - "bstr 0.2.17", - "lazy_static", - "nom", - "percent-encoding", - "regex", - "thiserror", - "url", -] - [[package]] name = "thiserror" version = "1.0.65" @@ -2840,6 +2395,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.5.1" @@ -3006,28 +2595,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf8-ranges" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" - [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" -dependencies = [ - "getrandom", - "serde", -] - [[package]] name = "vcpkg" version = "0.2.15" @@ -3142,6 +2715,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "6.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f" +dependencies = [ + "either", + "home", + "rustix", + "winsafe", +] + [[package]] name = "winapi" version = "0.3.9" @@ -3294,6 +2879,21 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + +[[package]] +name = "winsafe" +version = "0.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" + [[package]] name = "zerocopy" version = "0.7.35" @@ -3320,31 +2920,3 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" - -[[package]] -name = "zstd" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.13+zstd.1.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/Cargo.toml b/Cargo.toml index e5574c5..69920ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,19 +13,19 @@ __luau-jit = ["mlua/luau-jit"] axum = "0.7.7" env_logger = "0.11.5" log = "0.4.22" -lru = "0.12.5" +#lru = "0.12.5" once_cell = "1.20.2" reqwest = { version = "0.12.8", features = ["cookies", "gzip", "json"] } scraper = { version = "0.20.0", default-features = false, features = ["atomic"] } serde = { version = "1.0.213", features = ["derive"] } sled = "0.34.7" -tantivy = { version = "0.22.0", default-features = false, features = [ - "zstd-compression", - "mmap", - "stopwords", -] } +#tantivy = { version = "0.22.0", default-features = false, features = [ +# "zstd-compression", +# "mmap", +# "stopwords", +#] } tera = "1.20.0" -texting_robots = "0.2.2" +#texting_robots = "0.2.2" tokio = { version = "1.41.0", features = [ "rt-multi-thread", "macros", @@ -35,10 +35,10 @@ url = "2.5.2" serde_qs = "0.13.0" mlua = { version = "0.10.0", features = ["async", "macros", "serialize", "send"] } axum-macros = "0.4.2" -piccolo = "0.3.3" -piccolo-util = "0.3.3" +#piccolo = "0.3.3" +#piccolo-util = "0.3.3" serde_json = "1.0.132" -crossbeam-channel = "0.5.13" +toml = { version = "0.8.19", default-features = false, features = ["parse"] } [profile.dev.package.'*'] opt-level = 3 diff --git a/searched.toml b/searched.toml new file mode 100644 index 0000000..787e58f --- /dev/null +++ b/searched.toml @@ -0,0 +1,15 @@ +[provider.duckduckgo] +name = "DuckDuckGo" +description = "A privacy-respecting search engine" +kinds = ["sear"] + +[provider.google] +name = "Google" +description = "The most popular search engine" +kinds = ["sear"] + +[provider.wiby] +engine = "json_engine" +name = "Wiby" +description = "blah blah" +kinds = ["sear"] diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..896d58d --- /dev/null +++ b/src/config.rs @@ -0,0 +1,28 @@ +use std::{collections::HashMap, fs::File, io::Read, path::Path}; + +use crate::Kind; + +#[derive(Deserialize, Serialize, Default, Clone, Debug)] +pub struct Config { + #[serde(rename = "provider")] + pub providers: HashMap, +} +impl Config { + pub fn load(path: impl AsRef) -> Self { + let mut buf = String::new(); + File::open(path).unwrap().read_to_string(&mut buf).unwrap(); + toml::from_str(&buf).unwrap() + } +} + +#[derive(Deserialize, Serialize, Default, Clone, Debug)] +pub struct CfgProvider { + /// Underlying engine + /// + /// Uses provider name if unset + pub engine: Option, + /// Human readable name + pub name: String, + pub description: String, + pub kinds: Vec, +} diff --git a/src/lib.rs b/src/lib.rs index 252454b..5ea597e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,11 +2,14 @@ extern crate log; extern crate mlua; extern crate tokio; +#[macro_use] +extern crate serde; -pub mod crawler; +//pub mod crawler; pub mod lua_api; -pub mod page; -pub mod ranking; +//pub mod page; +//pub mod ranking; +pub mod config; use mlua::{FromLua, IntoLua, LuaSerdeExt}; use serde::{Deserialize, Serialize}; diff --git a/src/main.rs b/src/main.rs index 61252e3..7b71d40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,11 +3,11 @@ extern crate log; extern crate axum; extern crate env_logger; extern crate reqwest; -extern crate tantivy; +//extern crate tantivy; extern crate tera; #[macro_use] extern crate serde; -extern crate lru; +//extern crate lru; extern crate searched; mod web; @@ -24,7 +24,7 @@ use log::LevelFilter; //use searched::page::Page; //use reqwest::Client; //use scraper::Selector; -use searched::lua_api::PluginEnginePool; +use searched::{config::Config, lua_api::PluginEnginePool}; //use sled::Db; //use tantivy::{ // doc, @@ -45,6 +45,7 @@ pub struct AppState { //db: Db, //eng: Arc>, pool: PluginEnginePool, + config: Config, //query_tx: mpsc::Sender, //result_rx: Arc)>>, //url: Field, @@ -169,6 +170,8 @@ async fn main() { //let (engine, local) = PluginEngine::new().await.unwrap(); let (pool, joinset) = PluginEnginePool::new().await; + let config = Config::load("searched.toml"); + info!("initializing web"); let r = Router::new() .route("/", get(web::search)) @@ -184,6 +187,7 @@ async fn main() { //client, //db, //eng: Arc::new(Mutex::new(engine)), + config, pool, //url, //title,