chore(page): return all page content regardless of status

spider-rs · Mar 28, 2024 · 0cc921e · 0cc921e
1 parent 88688de
commit 0cc921e
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 30 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/spider/Cargo.toml b/spider/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "1.88.4"
+version = "1.88.6"
 authors = [
     "madeindjs <contact@rousseau-alexandre.fr>",
     "j-mendez <jeff@a11ywatch.com>",

diff --git a/spider/README.md b/spider/README.md
@@ -16,7 +16,7 @@ This is a basic async example crawling a web page, add spider to your `Cargo.tom
 
 ```toml
 [dependencies]
-spider = "1.88.4"
+spider = "1.88.6"
 ```
 
 And then the code:
@@ -93,7 +93,7 @@ We have the following optional feature flags.
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["regex", "ua_generator"] }
+spider = { version = "1.88.6", features = ["regex", "ua_generator"] }
 ```
 
 1. `ua_generator`: Enables auto generating a random real User-Agent.
@@ -137,7 +137,7 @@ Move processing to a worker, drastically increases performance even if worker is
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["decentralized"] }
+spider = { version = "1.88.6", features = ["decentralized"] }
 ```
 
 ```sh
@@ -168,7 +168,7 @@ Use the subscribe method to get a broadcast channel.
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["sync"] }
+spider = { version = "1.88.6", features = ["sync"] }
 ```
 
 ```rust,no_run
@@ -198,7 +198,7 @@ Allow regex for blacklisting routes
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["regex"] }
+spider = { version = "1.88.6", features = ["regex"] }
 ```
 
 ```rust,no_run
@@ -225,7 +225,7 @@ If you are performing large workloads you may need to control the crawler by ena
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["control"] }
+spider = { version = "1.88.6", features = ["control"] }
 ```
 
 ```rust
@@ -295,7 +295,7 @@ Use cron jobs to run crawls continuously at anytime.
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["sync", "cron"] }
+spider = { version = "1.88.6", features = ["sync", "cron"] }
 ```
 
 ```rust,no_run
@@ -334,7 +334,7 @@ the feature flag [`chrome_intercept`] to possibly speed up request using Network
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["chrome", "chrome_intercept"] }
+spider = { version = "1.88.6", features = ["chrome", "chrome_intercept"] }
 ```
 
 You can use `website.crawl_concurrent_raw` to perform a crawl without chromium when needed. Use the feature flag `chrome_headed` to enable headful browser usage if needed to debug.
@@ -364,7 +364,7 @@ Enabling HTTP cache can be done with the feature flag [`cache`] or [`cache_mem`]
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["cache"] }
+spider = { version = "1.88.6", features = ["cache"] }
 ```
 
 You need to set `website.cache` to true to enable as well.
@@ -395,7 +395,7 @@ Intelligently run crawls using HTTP and JavaScript Rendering when needed. The be
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["smart"] }
+spider = { version = "1.88.6", features = ["smart"] }
 ```
 
 ```rust,no_run
@@ -421,7 +421,7 @@ Use OpenAI to generate dynamic scripts to drive the browser done with the featur
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["openai"] }
+spider = { version = "1.88.6", features = ["openai"] }
 ```
 
 ```rust
@@ -446,7 +446,7 @@ Set a depth limit to prevent forwarding.
 
 ```toml
 [dependencies]
-spider = { version = "1.88.4", features = ["budget"] }
+spider = { version = "1.88.6", features = ["budget"] }
 ```
 
 ```rust,no_run

diff --git a/spider/src/utils.rs b/spider/src/utils.rs
@@ -1121,19 +1121,10 @@ pub fn clean_html_slim(html: &str) -> String {
                     el.remove();
                     Ok(())
                 }),
-                element!("svg", |el| {
+                element!("svgs", |el| {
                     el.remove();
                     Ok(())
                 }),
-                element!("a", |el| {
-                    if let Some(href) = el.get_attribute("href") {
-                        if let Ok(mut parsed_url) = url::Url::parse(&href) {
-                            parsed_url.set_query(None);
-                            el.set_attribute("href", &parsed_url.to_string()).unwrap();
-                        }
-                    }
-                    Ok(())
-                }),
                 element!("img", |el| {
                     if let Some(src) = el.get_attribute("src") {
                         if src.starts_with("data:image") {

diff --git a/spider_cli/Cargo.toml b/spider_cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "1.88.4"
+version = "1.88.6"
 authors = [
     "madeindjs <contact@rousseau-alexandre.fr>",
     "j-mendez <jeff@a11ywatch.com>",
@@ -29,7 +29,7 @@ quote = "1.0.18"
 failure_derive = "0.1.8"
 
 [dependencies.spider]
-version = "1.88.4"
+version = "1.88.6"
 path = "../spider"
 
 [[bin]]

diff --git a/spider_worker/Cargo.toml b/spider_worker/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "1.88.4"
+version = "1.88.6"
 authors = [
     "madeindjs <contact@rousseau-alexandre.fr>",
     "j-mendez <jeff@a11ywatch.com>",
@@ -25,7 +25,7 @@ lazy_static = "1.4.0"
 env_logger = "0.11.3"
 
 [dependencies.spider]
-version = "1.88.4"
+version = "1.88.6"
 path = "../spider"
 features = ["serde", "flexbuffers"]