Don't check preconnect links (#1187)

Preconnect links are used to establish a server connection without loading a
specific resource yet. Not always do these links point to a URL that should
return a 200, and they are not user-facing, i.e. they don't show up in the
final rendered version of a page.

Therefore, we should not check them at all; not even in `--include-verbatim`
mode, as they might not point to a valid resource.

This turned out to require a significant overhaul of the html5gum extractor
to handle random attribute ordering correctly. Changes to the html5gum extractor:

* Refactor HTML link extractor for improved performance and maintainability
- Replace Vec<u8> with String for better readability and manipulation
- Introduce Element struct to encapsulate element-related data
- Use `HashMap<String, String>` for current_attributes for efficient lookups
- Add verbatim_stack to properly handle nested verbatim elements
- Remove unsafe code where possible, using String::from_utf8_lossy
- Improve attribute handling with `HashMap` entry API and prioritize `srcset`
- Simplify logic and consolidate verbatim element handling
- Enhance encapsulation in `LinkExtractor` struct
- Improve overall performance with more efficient data structures
- Increase flexibility for future feature additions or modifications

Fixes #897

lychee-lib/Cargo.toml

@@ -69,6 +69,7 @@ wiremock = "0.6.2"
 serde_json = "1.0.128"
 rstest = "0.23.0"
 toml = "0.8.19"
+pretty_assertions = "1.4.0"
 
 [features]
 
@@ -91,4 +92,4 @@ vendored-openssl = ["openssl-sys/vendored"]
 # See https://users.rust-lang.org/t/36630
 check_example_domains = []
 
-default = ["native-tls", "email-check"]
+default = ["native-tls", "email-check"]

lychee-lib/src/extract/html/html5ever.rs

@@ -6,7 +6,9 @@ use html5ever::{
     tokenizer::{Tag, TagKind, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts},
 };
 
-use super::{super::plaintext::extract_plaintext, is_email_link, is_verbatim_elem, srcset};
+use super::{
+    super::plaintext::extract_raw_uri_from_plaintext, is_email_link, is_verbatim_elem, srcset,
+};
 use crate::types::uri::raw::RawUri;
 
 #[derive(Clone, Default)]
@@ -26,7 +28,9 @@ impl TokenSink for LinkExtractor {
                 if self.current_verbatim_element_name.borrow().is_some() {
                     return TokenSinkResult::Continue;
                 }
-                self.links.borrow_mut().extend(extract_plaintext(&raw));
+                self.links
+                    .borrow_mut()
+                    .extend(extract_raw_uri_from_plaintext(&raw));
             }
             Token::TagToken(tag) => {
                 let Tag {
@@ -72,6 +76,14 @@ impl TokenSink for LinkExtractor {
                     }
                 }
 
+                // Check and exclude rel=preconnect. Other than prefetch and preload,
+                // preconnect only does DNS lookups and might not be a link to a resource
+                if let Some(rel) = attrs.iter().find(|attr| &attr.name.local == "rel") {
+                    if rel.value.contains("preconnect") {
+                        return TokenSinkResult::Continue;
+                    }
+                }
+
                 for attr in attrs {
                     let urls = LinkExtractor::extract_urls_from_elem_attr(
                         &attr.name.local,
@@ -80,7 +92,7 @@ impl TokenSink for LinkExtractor {
                     );
 
                     let new_urls = match urls {
-                        None => extract_plaintext(&attr.value),
+                        None => extract_raw_uri_from_plaintext(&attr.value),
                         Some(urls) => urls
                             .into_iter()
                             .filter(|url| {
@@ -140,6 +152,7 @@ impl LinkExtractor {
         // and https://html.spec.whatwg.org/multipage/indices.html#attributes-1
 
         match (elem_name, attr_name) {
+
             // Common element/attribute combinations for links
             (_, "href" | "src" | "cite" | "usemap")
             // Less common (but still valid!) combinations
@@ -380,4 +393,24 @@ mod tests {
         let uris = extract_html(input, false);
         assert_eq!(uris, expected);
     }
+
+    #[test]
+    fn test_skip_preconnect() {
+        let input = r#"
+            <link rel="preconnect" href="https://example.com">
+        "#;
+
+        let uris = extract_html(input, false);
+        assert!(uris.is_empty());
+    }
+
+    #[test]
+    fn test_skip_preconnect_reverse_order() {
+        let input = r#"
+            <link href="https://example.com" rel="preconnect">
+        "#;
+
+        let uris = extract_html(input, false);
+        assert!(uris.is_empty());
+    }
 }