diff --git a/Cargo.toml b/Cargo.toml index 815f558e..56c7b488 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ full = [ "gzip", "brotli", "zstd", - "deflate" + "deflate", ] websocket = ["dep:async-tungstenite", "tokio-util/compat"] @@ -39,7 +39,11 @@ cookies = ["dep:cookie_crate", "dep:cookie_store"] gzip = ["dep:async-compression", "async-compression?/gzip", "dep:tokio-util"] -brotli = ["dep:async-compression", "async-compression?/brotli", "dep:tokio-util"] +brotli = [ + "dep:async-compression", + "async-compression?/brotli", + "dep:tokio-util", +] zstd = ["dep:async-compression", "async-compression?/zstd", "dep:tokio-util"] @@ -77,7 +81,10 @@ url = "2.5" bytes = "1.0" serde = "1.0" serde_urlencoded = "0.7.1" -tower = { version = "0.5.0", default-features = false, features = ["timeout", "util"] } +tower = { version = "0.5.0", default-features = false, features = [ + "timeout", + "util", +] } tower-service = "0.3" futures-core = { version = "0.3.0", default-features = false } futures-util = { version = "0.3.0", default-features = false } @@ -99,7 +106,11 @@ hyper2 = { version = "1", features = ["http1", "http2", "client"] } log = "0.4" mime = "0.3.17" percent-encoding = "2.3" -tokio = { version = "1", default-features = false, features = ["net", "time", "rt"] } +tokio = { version = "1", default-features = false, features = [ + "net", + "time", + "rt", +] } pin-project-lite = "0.2.0" ipnet = "2.10.0" @@ -111,7 +122,7 @@ lru = { version = "0.12", default-features = false } # Optional deps... ## boring-tls -boring = { package = "boring2", version = "4", features = ["pq-experimental"]} +boring = { package = "boring2", version = "4", features = ["pq-experimental"] } boring-sys = { package = "boring-sys2", version = "4" } tokio-boring = { package = "tokio-boring2", version = "4" } foreign-types = { version = "0.5.0" } @@ -135,8 +146,13 @@ cookie_crate = { version = "0.18", package = "cookie", optional = true } cookie_store = { version = "0.21", optional = true } ## compression -async-compression = { version = "0.4.0", default-features = false, features = ["tokio"], optional = true } -tokio-util = { version = "0.7.0", default-features = false, features = ["codec", "io"], optional = true } +async-compression = { version = "0.4.0", default-features = false, features = [ + "tokio", +], optional = true } +tokio-util = { version = "0.7.0", default-features = false, features = [ + "codec", + "io", +], optional = true } ## socks tokio-socks = { version = "0.5.2", optional = true } @@ -154,8 +170,20 @@ windows-registry = "0.3.0" system-configuration = "0.6.0" [dev-dependencies] -hyper = { version = "1.1.0", default-features = false, features = ["http1", "http2", "client", "server"] } -hyper-util = { version = "0.1.10", features = ["http1", "http2", "client", "client-legacy", "server-auto", "tokio"] } +hyper = { version = "1.1.0", default-features = false, features = [ + "http1", + "http2", + "client", + "server", +] } +hyper-util = { version = "0.1.10", features = [ + "http1", + "http2", + "client", + "client-legacy", + "server-auto", + "tokio", +] } env_logger = "0.10.0" serde = { version = "1.0", features = ["derive"] } libflate = "2.0.0" @@ -259,11 +287,6 @@ name = "websocket" path = "examples/websocket.rs" required-features = ["websocket", "futures-util/std"] -[[example]] -name = "websocket_via_proxy" -path = "examples/websocket_via_proxy.rs" -required-features = ["websocket", "futures-util/std", "socks"] - [[test]] name = "cookie" path = "tests/cookie.rs" diff --git a/examples/websocket.rs b/examples/websocket.rs index bc1fb23d..49650daa 100644 --- a/examples/websocket.rs +++ b/examples/websocket.rs @@ -1,17 +1,9 @@ use futures_util::{SinkExt, StreamExt, TryStreamExt}; -use rquest::{tls::Impersonate, Client, Message}; +use rquest::Message; #[tokio::main] async fn main() -> Result<(), rquest::Error> { - // Build a client to mimic Chrome130 - let websocket = Client::builder() - .impersonate(Impersonate::Chrome130) - .build()? - .websocket("wss://echo.websocket.org") - .send() - .await? - .into_websocket() - .await?; + let websocket = rquest::websocket("wss://echo.websocket.org").await?; let (mut tx, mut rx) = websocket.split(); diff --git a/examples/websocket_via_proxy.rs b/examples/websocket_via_proxy.rs deleted file mode 100644 index ab4f40a4..00000000 --- a/examples/websocket_via_proxy.rs +++ /dev/null @@ -1,37 +0,0 @@ -use futures_util::{SinkExt, StreamExt, TryStreamExt}; -use rquest::{tls::Impersonate, Client, Message, Proxy}; - -#[tokio::main] -async fn main() -> Result<(), rquest::Error> { - env_logger::init_from_env(env_logger::Env::default().default_filter_or("debug")); - - // Build a client to mimic Chrome130 - let websocket = Client::builder() - .impersonate(Impersonate::Chrome130) - .proxy(Proxy::https("socks5h://gngpp:gngpp123@127.0.0.1:1080")?) - .build()? - .websocket("wss://echo.websocket.org") - .send() - .await? - .into_websocket() - .await?; - - let (mut tx, mut rx) = websocket.split(); - - tokio::spawn(async move { - for i in 1..11 { - tx.send(Message::Text(format!("Hello, World! #{i}"))) - .await - .unwrap(); - } - }); - - while let Some(message) = rx.try_next().await? { - match message { - Message::Text(text) => println!("received: {text}"), - _ => {} - } - } - - Ok(()) -} diff --git a/src/tls/mimic/macros.rs b/src/tls/mimic/macros.rs index f17f0374..3915d178 100644 --- a/src/tls/mimic/macros.rs +++ b/src/tls/mimic/macros.rs @@ -53,6 +53,7 @@ macro_rules! header_chrome_ua { macro_rules! header_chrome_accpet { ($headers:expr) => { $headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")); + #[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))] $headers.insert( ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br"), @@ -61,6 +62,12 @@ macro_rules! header_chrome_accpet { }; (zstd, $headers:expr) => { $headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")); + #[cfg(all( + feature = "gzip", + feature = "deflate", + feature = "brotli", + feature = "zstd" + ))] $headers.insert( ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br, zstd"), @@ -93,6 +100,7 @@ macro_rules! header_firefox_accept { "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", ), ); + #[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))] $headers.insert( header::ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br"), @@ -109,6 +117,12 @@ macro_rules! header_firefox_accept { "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", ), ); + #[cfg(all( + feature = "gzip", + feature = "deflate", + feature = "brotli", + feature = "zstd" + ))] $headers.insert( header::ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br, zstd"), diff --git a/src/tls/mimic/mod.rs b/src/tls/mimic/mod.rs index 02567a0c..2ecd5354 100644 --- a/src/tls/mimic/mod.rs +++ b/src/tls/mimic/mod.rs @@ -17,9 +17,12 @@ mod impersonate_imports { pub(crate) use crate::tls::{Http2Settings, ImpersonateSettings}; pub use crate::*; pub use http::{ - header::{ACCEPT, ACCEPT_ENCODING, ACCEPT_LANGUAGE, UPGRADE_INSECURE_REQUESTS, USER_AGENT}, + header::{ACCEPT, ACCEPT_LANGUAGE, UPGRADE_INSECURE_REQUESTS, USER_AGENT}, HeaderMap, HeaderName, HeaderValue, }; + + #[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))] + pub use http::header::ACCEPT_ENCODING; } mod tls_imports { diff --git a/src/tls/mimic/okhttp.rs b/src/tls/mimic/okhttp.rs index 4b9eab28..5ef0f747 100644 --- a/src/tls/mimic/okhttp.rs +++ b/src/tls/mimic/okhttp.rs @@ -50,6 +50,7 @@ fn header_initializer(ua: &'static str) -> HeaderMap { headers.insert(ACCEPT, HeaderValue::from_static("*/*")); headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9")); headers.insert(USER_AGENT, HeaderValue::from_static(ua)); + #[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))] headers.insert( ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br"), diff --git a/src/tls/mimic/safari.rs b/src/tls/mimic/safari.rs index dfa9c2a7..f6adc6cf 100644 --- a/src/tls/mimic/safari.rs +++ b/src/tls/mimic/safari.rs @@ -102,6 +102,7 @@ fn header_initializer_for_16_17(ua: &'static str) -> HeaderMap { HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ); headers.insert("sec-fetch-site", HeaderValue::from_static("none")); + #[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))] headers.insert( ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br"), @@ -122,6 +123,7 @@ fn header_initializer_for_15(ua: &'static str) -> HeaderMap { HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ); headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9")); + #[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))] headers.insert( ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br"), @@ -142,6 +144,7 @@ fn header_initializer_for_18(ua: &'static str) -> HeaderMap { headers.insert("sec-fetch-mode", HeaderValue::from_static("navigate")); headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9")); headers.insert("priority", HeaderValue::from_static("u=0, i")); + #[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))] headers.insert( ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br"),