perf(smart): fix js need determination

spider/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.11.20"
+version = "2.12.1"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]
@@ -143,7 +143,7 @@ chrome_intercept = ["chrome"]
 chrome_headless_new = ["chrome"]
 cookies = ["reqwest/cookies"]
 cron = ["dep:async_job", "dep:chrono", "dep:cron", "dep:async-trait"]
-smart = ["chrome", "dep:rand", "chrome_intercept"]
+smart = ["chrome", "dep:rand", "chrome_intercept", "dep:lol_html"]
 encoding = []
 headers = ["dep:httpdate"]
 real_browser = ["dep:statrs", "dep:rand"]

spider/src/packages/scraper/element_ref/mod.rs

@@ -59,9 +59,7 @@ impl<'a> ElementRef<'a> {
             create_missing_parent: false,
         };
         let mut buf = Vec::new();
-        match serialize(&mut buf, self, opts) {
-            _ => (),
-        };
+        let _ = serialize(&mut buf, self, opts);
         // we need to get the initial encoding of the html lang if used.
         auto_encoder::auto_encode_bytes(&buf)
     }

spider/src/packages/scraper/html/mod.rs

@@ -130,9 +130,7 @@ impl Html {
             create_missing_parent: false,
         };
         let mut buf = Vec::new();
-        match serialize(&mut buf, self, opts) {
-            _ => (),
-        };
+        let _ = serialize(&mut buf, self, opts);
         auto_encoder::auto_encode_bytes(&buf)
     }
 

spider/src/packages/scraper/html/tree_sink.rs

@@ -164,15 +164,16 @@ impl TreeSink for Html {
 
     // Detach the given node from its parent.
     fn remove_from_parent(&mut self, target: &Self::Handle) {
-        self.tree.get_mut(*target).unwrap().detach();
+        if let Some(mut p) = self.tree.get_mut(*target) {
+            p.detach();
+        }
     }
 
     // Remove all the children from node and append them to new_parent.
     fn reparent_children(&mut self, node: &Self::Handle, new_parent: &Self::Handle) {
-        self.tree
-            .get_mut(*new_parent)
-            .unwrap()
-            .reparent_from_id_append(*node);
+        if let Some(mut p) = self.tree.get_mut(*new_parent) {
+            p.reparent_from_id_append(*node);
+        }
     }
 
     // Add each attribute to the given element, if no attribute with that name already exists. The

spider/src/utils/mod.rs

@@ -980,6 +980,15 @@ pub async fn fetch_page_html_chrome_base(
                             },
                         )
                         .await;
+
+                    // perform extra navigate to trigger page actions.
+                    if let Some(u) = url_target {
+                        if u.starts_with("http") {
+                            let _ = page
+                                .evaluate(format!(r#"window.location = "{}";"#, u))
+                                .await;
+                        }
+                    }
                 }
             } else {
                 if let Err(e) = navigate(page, source, &mut chrome_http_req_res).await {

spider_chrome/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_chrome"
-version = "2.11.20"
+version = "2.12.1"
 rust-version = "1.70"
 authors = [
     "j-mendez <jeff@spider.cloud>"

spider_cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.11.20"
+version = "2.12.1"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

spider_transformations/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_transformations"
-version = "2.11.20"
+version = "2.12.1"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

spider_utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.11.20"
+version = "2.12.1"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]

spider_worker/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.11.20"
+version = "2.12.1"
 authors = [
     "j-mendez <jeff@spider.cloud>"
 ]