diff --git a/Cargo.toml b/Cargo.toml index 69920ab..74d574f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ axum-macros = "0.4.2" #piccolo-util = "0.3.3" serde_json = "1.0.132" toml = { version = "0.8.19", default-features = false, features = ["parse"] } +strfmt = "0.2.4" [profile.dev.package.'*'] opt-level = 3 diff --git a/api/search.lua b/api/search.lua index df4f559..72eaeff 100644 --- a/api/search.lua +++ b/api/search.lua @@ -14,8 +14,8 @@ function add_search_provider(name, kind, callback) end --- Add a engine --- --- @param name string ---- @param callback fun(query: Query): [Result] -function add_search_provider(name, callback) end +--- @param callback fun(query: Query, url: string|nil): [Result] +function add_engine(name, callback) end --- Use a engine --- diff --git a/plugins/scrapers/duckduckgo.lua b/plugins/engines/duckduckgo.lua similarity index 95% rename from plugins/scrapers/duckduckgo.lua rename to plugins/engines/duckduckgo.lua index e193374..1868a97 100644 --- a/plugins/scrapers/duckduckgo.lua +++ b/plugins/engines/duckduckgo.lua @@ -2,7 +2,7 @@ -- Licensed MIT. -- (c) 2024 Dragynfruit -add_search_provider('duckduckgo', 'sear', function (query) +add_engine('duckduckgo', function (query, _) local offset if query.page == 2 then offset = (query.page - 1) * 20 diff --git a/plugins/scrapers/google.lua b/plugins/engines/google.lua similarity index 100% rename from plugins/scrapers/google.lua rename to plugins/engines/google.lua diff --git a/plugins/engines/json_engine.lua b/plugins/engines/json_engine.lua index 99be758..f0fe2f0 100644 --- a/plugins/engines/json_engine.lua +++ b/plugins/engines/json_engine.lua @@ -2,8 +2,9 @@ -- -- Licensed MIT. -- -- (c) 2024 Dragynfruit -add_engine("json_engine", function(url, query) - local res = get(string.format(url, query.query, query.page), {}) +add_engine("json_engine", function(_, url) + if url ~= nil then + local res = get(url, {}) local data = parse_json(res) local results = {} @@ -15,5 +16,9 @@ add_engine("json_engine", function(url, query) } end - return results -end) \ No newline at end of file + return results + else + print('wtf') + return {} + end +end) diff --git a/plugins/scrapers/stackexchange.lua b/plugins/engines/stackexchange.lua similarity index 86% rename from plugins/scrapers/stackexchange.lua rename to plugins/engines/stackexchange.lua index b664b2c..1991c25 100644 --- a/plugins/scrapers/stackexchange.lua +++ b/plugins/engines/stackexchange.lua @@ -1,4 +1,4 @@ -add_search_provider('stackoverflow', 'qans', function (query) +add_engine('stackoverflow', function (query, _) local res = get('https://api.stackexchange.com/2.3/search/advanced?q=' .. query.query .. '&page=' .. query.page .. '&site=stackoverflow', {}) print(res) diff --git a/plugins/scrapers/wikipedia.lua b/plugins/engines/wikipedia.lua similarity index 88% rename from plugins/scrapers/wikipedia.lua rename to plugins/engines/wikipedia.lua index d61d4c2..e2260d8 100644 --- a/plugins/scrapers/wikipedia.lua +++ b/plugins/engines/wikipedia.lua @@ -2,7 +2,7 @@ -- -- Licensed MIT. -- -- (c) 2024 Dragynfruit -add_search_provider('wikipedia', 'wiki', function (query) +add_engine('mediawiki', function (query, _) local res = get('https://en.wikipedia.org/w/api.php?action=opensearch&format=json&limit=10&namespace=0&search='..query.query, {}) local data = parse_json(res) diff --git a/searched.toml b/plugins/providers.toml similarity index 85% rename from searched.toml rename to plugins/providers.toml index 787e58f..a5742a5 100644 --- a/searched.toml +++ b/plugins/providers.toml @@ -10,6 +10,7 @@ kinds = ["sear"] [provider.wiby] engine = "json_engine" +url = "https://wiby.me/json?q={query}&page={page}" name = "Wiby" description = "blah blah" kinds = ["sear"] diff --git a/plugins/scrapers/wiby.lua b/plugins/scrapers/wiby.lua deleted file mode 100644 index 214efb4..0000000 --- a/plugins/scrapers/wiby.lua +++ /dev/null @@ -1,7 +0,0 @@ --- -- Wiby scraper for Searched --- -- Licensed MIT. --- -- (c) 2024 Dragynfruit - -add_search_provider('wiby', 'sear', function (query) - return use_engine('json_engine')('https://wiby.me/json?q=%s&page=%d', query) -end) \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index 896d58d..d25d15f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -21,6 +21,8 @@ pub struct CfgProvider { /// /// Uses provider name if unset pub engine: Option, + /// Formatting string for URL + pub url: Option, /// Human readable name pub name: String, pub description: String, diff --git a/src/lua_api.rs b/src/lua_api.rs index f697381..0cfd3a0 100644 --- a/src/lua_api.rs +++ b/src/lua_api.rs @@ -12,12 +12,13 @@ use std::{ use mlua::prelude::*; use reqwest::Client; use scraper::{node::Element, Html, Selector}; +use strfmt::{strfmt, Format}; use tokio::{ sync::{oneshot, watch, Mutex}, task::{spawn_local, JoinHandle, JoinSet, LocalSet}, }; -use crate::{Kind, Query}; +use crate::{config::{self, Config}, Kind, Query}; impl LuaUserData for Query { fn add_fields>(fields: &mut F) { @@ -74,11 +75,12 @@ pub struct PluginEngine { } impl PluginEngine { pub async fn new() -> Result> { + let config = Config::load("plugins/providers.toml"); let (query_tx, rx) = watch::channel(Default::default()); let (tx, results_rx) = watch::channel(Default::default()); spawn_local(async move { - Self::inner(rx, tx).await.unwrap(); + Self::inner(config.clone(), rx, tx).await.unwrap(); }); Ok(Self { @@ -87,8 +89,10 @@ impl PluginEngine { }) } + /// Actual Lua init and event loop async fn inner( + config: Config, mut rx: watch::Receiver, tx: watch::Sender>, ) -> Result<(), Box> { @@ -96,8 +100,6 @@ impl PluginEngine { // Add Lua interfaces - lua.globals() - .set("__searched_providers__", lua.create_table()?)?; lua.globals() .set("__searched_engines__", lua.create_table()?)?; lua.globals().set("Query", lua.create_proxy::()?)?; @@ -106,24 +108,6 @@ impl PluginEngine { lua.globals() .set("Element", lua.create_proxy::()?)?; - async fn add_search_provider( - lua: Lua, - (name, _kind, callback): (String, Kind, LuaFunction), - ) -> LuaResult<()> { - lua.globals() - .get::("__searched_providers__") - .unwrap() - .set(name, callback.clone()) - .unwrap(); - - Ok(()) - } - - lua.globals().set( - "add_search_provider", - lua.create_async_function(add_search_provider)?, - )?; - async fn add_engine( lua: Lua, (name, callback): (String, LuaFunction), @@ -136,27 +120,11 @@ impl PluginEngine { Ok(()) } - lua.globals().set( "add_engine", lua.create_async_function(add_engine)?, )?; - async fn use_engine( - lua: Lua, - name: String, - ) -> LuaResult { - lua.globals() - .get::("__searched_engines__") - .unwrap() - .get::(name) - } - - lua.globals().set( - "use_engine", - lua.create_async_function(use_engine)?, - )?; - async fn get(_: Lua, (url, headers): (String, LuaTable)) -> LuaResult { let mut req = Client::new().get(url); @@ -229,15 +197,15 @@ impl PluginEngine { })?, )?; - // Load scrapers - for path in read_dir("plugins/scrapers").unwrap() { - if let Ok(path) = path { - let mut buf = String::new(); - let mut f = File::open(path.path()).unwrap(); - f.read_to_string(&mut buf).unwrap(); - lua.load(&buf).exec_async().await.unwrap(); - } - } + //// Load scrapers + //for path in read_dir("plugins/scrapers").unwrap() { + // if let Ok(path) = path { + // let mut buf = String::new(); + // let mut f = File::open(path.path()).unwrap(); + // f.read_to_string(&mut buf).unwrap(); + // lua.load(&buf).exec_async().await.unwrap(); + // } + //} // Load engines for path in read_dir("plugins/engines").unwrap() { @@ -253,15 +221,33 @@ impl PluginEngine { while let Ok(()) = rx.changed().await { let query = rx.borrow_and_update().clone(); - let provider_ = lua + let provider = config.providers.get(&query.provider).unwrap(); + + let engine = provider.engine.clone().unwrap_or(query.provider.clone()); + + let engine_ = lua .globals() - .get::("__searched_providers__") + .get::("__searched_engines__") .unwrap() - .get::(query.provider.clone()) + .get::(engine) .unwrap(); - let results = provider_ - .call_async::>>(query) + let url = { + if let Some(url_fmt) = &provider.url { + let query = query.clone(); + let map = HashMap::from_iter([ + ("query".to_string(), query.query), + ("page".to_string(), query.page.to_string()), + ]); + + Some(url_fmt.format(&map).unwrap()) + } else { + None + } + }; + + let results = engine_ + .call_async::>>((query, url)) .await; match results { diff --git a/src/main.rs b/src/main.rs index 7b71d40..2944661 100644 --- a/src/main.rs +++ b/src/main.rs @@ -170,7 +170,7 @@ async fn main() { //let (engine, local) = PluginEngine::new().await.unwrap(); let (pool, joinset) = PluginEnginePool::new().await; - let config = Config::load("searched.toml"); + let config = Config::load("plugins/providers.toml"); info!("initializing web"); let r = Router::new()