Kinda works

Cargo.toml

@@ -11,15 +11,25 @@ env_logger = "0.11.3"
 log = "0.4.21"
 lru = "0.12.3"
 once_cell = "1.19.0"
-reqwest = "0.12.4"
+reqwest = { version = "0.12.4", features = ["cookies", "gzip", "json"] }
 scraper = "0.19.0"
 serde = { version = "1.0.203", features = ["derive"] }
 sled = "0.34.7"
-tantivy = { version = "0.22.0", default-features = false, features = ["zstd-compression", "mmap", "stopwords"] }
+tantivy = { version = "0.22.0", default-features = false, features = [
+  "zstd-compression",
+  "mmap",
+  "stopwords",
+] }
 tera = "1.20.0"
 texting_robots = "0.2.2"
-tokio = { version = "1.38.0", features = ["rt-multi-thread", "macros", "signal"] }
+tokio = { version = "1.38.0", features = [
+  "rt-multi-thread",
+  "macros",
+  "signal",
+] }
 url = "2.5.0"
+serde_qs = "0.13.0"
+phf = { version = "0.11", features = ["macros"] }
 
 [profile.release]
 lto = true

src/crawler.rs

@@ -1,8 +1,5 @@
 use std::{
-    collections::HashSet,
     error::Error,
-    future::IntoFuture,
-    sync::Arc,
     time::{Duration, Instant},
 };
 

src/lib.rs

@@ -0,0 +1,57 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Deserialize, Serialize)]
+pub enum Kind {
+    #[serde(rename = "imgs")]
+    Images,
+    #[serde(rename = "vids")]
+    Videos,
+    #[serde(rename = "news")]
+    News,
+    #[serde(rename = "maps")]
+    Maps,
+    #[serde(rename = "docs")]
+    Documentation,
+    #[serde(rename = "pprs")]
+    Papers,
+    #[serde(other)]
+    General,
+}
+impl Default for Kind {
+    fn default() -> Self {
+        Self::General
+    }
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct Query {
+    #[serde(rename = "q")]
+    pub query: String,
+    #[serde(rename = "k")]
+    pub kind: Kind,
+    #[serde(rename = "p")]
+    pub page: usize,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct Result {
+    pub url: String,
+    pub title: String,
+    pub general: Option<GeneralResult>,
+    pub forum: Option<ForumResult>,
+    pub image: Option<ImageResult>,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct GeneralResult {
+    pub snippet: String,
+}
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct ForumResult {
+    pub poster_image: Option<String>,
+    pub poster_username: String,
+    pub poster_url: Option<String>,
+    pub tags: Option<Vec<String>>,
+}
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct ImageResult {}