feat(transformations): add spider_transformation crate

Cargo.toml

@@ -4,6 +4,7 @@ members = [
  "spider_worker",
  "spider_cli",
  "spider_utils",
+ "spider_transformations",
  # internal
  "examples",
  "benches"]

examples/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_examples"
-version = "2.5.5"
+version = "2.6.2"
 authors = [
  "j-mendez <jeff@spider.cloud>",
 ]
@@ -205,4 +205,9 @@ required-features = ["spider/sync"]
 [[example]]
 name = "css_scrape"
 path = "css_scrape.rs"
-required-features = ["spider/sync"]
+required-features = ["spider/sync"]
+
+[[example]]
+name = "transform_markdown"
+path = "transform_markdown.rs"
+required-features = ["spider/sync", "spider_utils/transformations"]

examples/transform_markdown.rs

@@ -0,0 +1,48 @@
+//! `cargo run --example transform_markdown --features="spider/sync spider_utils/transformations"`
+extern crate spider;
+
+use spider::tokio;
+use spider::website::Website;
+use spider_utils::spider_transformations::transformation::content::{
+ transform_content, ReturnFormat, TransformConfig,
+};
+use tokio::io::AsyncWriteExt;
+
+#[tokio::main]
+async fn main() {
+ let mut website: Website = Website::new("https://rsseau.fr");
+ let mut rx2: tokio::sync::broadcast::Receiver<spider::page::Page> =
+ website.subscribe(0).unwrap();
+ let mut stdout = tokio::io::stdout();
+
+ let mut conf = TransformConfig::default();
+ conf.return_format = ReturnFormat::Markdown;
+
+ let join_handle = tokio::spawn(async move {
+ while let Ok(res) = rx2.recv().await {
+ let markup = transform_content(&res, &conf, &None, &None);
+
+ let _ = stdout
+ .write_all(format!("- {}\n {}\n", res.get_url(), markup).as_bytes())
+ .await;
+ }
+ stdout
+ });
+
+ let start = std::time::Instant::now();
+ website.crawl().await;
+ website.unsubscribe();
+ let duration = start.elapsed();
+ let mut stdout = join_handle.await.unwrap();
+
+ let _ = stdout
+ .write_all(
+ format!(
+ "Time elapsed in website.crawl() is: {:?} for total pages: {:?}",
+ duration,
+ website.get_links().len()
+ )
+ .as_bytes(),
+ )
+ .await;
+}

spider/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.5.5"
+version = "2.6.2"
 authors = [
  "j-mendez <jeff@spider.cloud>"
 ]

spider/src/utils/header_utils.rs

@@ -1,5 +1,5 @@
 use crate::configuration::Configuration;
-use reqwest::header::REFERER;
+use reqwest::header::{HOST, REFERER};
 use reqwest::{
  header::{HeaderMap, HeaderValue},
  ClientBuilder,
@@ -10,6 +10,7 @@ pub fn setup_default_headers(
  client_builder: ClientBuilder,
  configuration: &Configuration,
  header_map: HeaderMap,
+ url: &Option<Box<url::Url>>,
 ) -> ClientBuilder {
  let mut headers = match configuration.headers.clone() {
  Some(h) => *h,
@@ -26,6 +27,21 @@ pub fn setup_default_headers(
  }
  }
 
+ if !headers.contains_key(HOST) {
+ match url {
+ Some(u) => {
+ if let Some(host) = u.host_str() {
+ if let Ok(ref_value) = HeaderValue::from_str(&host) {
+ if !ref_value.is_empty() {
+ headers.insert(HOST, ref_value);
+ }
+ }
+ }
+ }
+ _ => (),
+ }
+ }
+
  headers.extend(header_map);
 
  client_builder.default_headers(headers)

spider/src/website.rs

@@ -786,8 +786,12 @@ impl Website {
  client
  };
 
- let client =
- crate::utils::header_utils::setup_default_headers(client, &self.configuration, headers);
+ let client = crate::utils::header_utils::setup_default_headers(
+ client,
+ &self.configuration,
+ headers,
+ self.get_url_parsed(),
+ );
 
  let mut client = match &self.configuration.request_timeout {
  Some(t) => client.timeout(**t),
@@ -841,8 +845,12 @@ impl Website {
  client
  };
 
- let client =
- crate::utils::header_utils::setup_default_headers(client, &self.configuration, headers);
+ let client = crate::utils::header_utils::setup_default_headers(
+ client,
+ &self.configuration,
+ headers,
+ self.get_url_parsed(),
+ );
 
  let mut client = match &self.configuration.request_timeout {
  Some(t) => client.timeout(**t),

spider_cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.5.5"
+version = "2.6.2"
 authors = [
  "j-mendez <jeff@spider.cloud>"
 ]