Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: without compression enabled, no compression header is sent #229

Merged
merged 3 commits into from
Dec 27, 2024
Merged

feat: without compression enabled, no compression header is sent #229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cb48275
fix: Username in URL plus basic_auth() results in two Authorization h…
0x676e67 Dec 27, 2024
ad62c70
feat: without compression enabled, no compression header is sent
0x676e67 Dec 27, 2024
1a851af
Merge remote-tracking branch 'origin/main' into fix
0x676e67 Dec 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 37 additions & 14 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ full = [
"gzip",
"brotli",
"zstd",
"deflate"
"deflate",
]

websocket = ["dep:async-tungstenite", "tokio-util/compat"]
Expand All @@ -39,7 +39,11 @@ cookies = ["dep:cookie_crate", "dep:cookie_store"]

gzip = ["dep:async-compression", "async-compression?/gzip", "dep:tokio-util"]

brotli = ["dep:async-compression", "async-compression?/brotli", "dep:tokio-util"]
brotli = [
"dep:async-compression",
"async-compression?/brotli",
"dep:tokio-util",
]

zstd = ["dep:async-compression", "async-compression?/zstd", "dep:tokio-util"]

Expand Down Expand Up @@ -77,7 +81,10 @@ url = "2.5"
bytes = "1.0"
serde = "1.0"
serde_urlencoded = "0.7.1"
tower = { version = "0.5.0", default-features = false, features = ["timeout", "util"] }
tower = { version = "0.5.0", default-features = false, features = [
"timeout",
"util",
] }
tower-service = "0.3"
futures-core = { version = "0.3.0", default-features = false }
futures-util = { version = "0.3.0", default-features = false }
Expand All @@ -99,7 +106,11 @@ hyper2 = { version = "1", features = ["http1", "http2", "client"] }
log = "0.4"
mime = "0.3.17"
percent-encoding = "2.3"
tokio = { version = "1", default-features = false, features = ["net", "time", "rt"] }
tokio = { version = "1", default-features = false, features = [
"net",
"time",
"rt",
] }
pin-project-lite = "0.2.0"
ipnet = "2.10.0"

Expand All @@ -111,7 +122,7 @@ lru = { version = "0.12", default-features = false }
# Optional deps...

## boring-tls
boring = { package = "boring2", version = "4", features = ["pq-experimental"]}
boring = { package = "boring2", version = "4", features = ["pq-experimental"] }
boring-sys = { package = "boring-sys2", version = "4" }
tokio-boring = { package = "tokio-boring2", version = "4" }
foreign-types = { version = "0.5.0" }
Expand All @@ -135,8 +146,13 @@ cookie_crate = { version = "0.18", package = "cookie", optional = true }
cookie_store = { version = "0.21", optional = true }

## compression
async-compression = { version = "0.4.0", default-features = false, features = ["tokio"], optional = true }
tokio-util = { version = "0.7.0", default-features = false, features = ["codec", "io"], optional = true }
async-compression = { version = "0.4.0", default-features = false, features = [
"tokio",
], optional = true }
tokio-util = { version = "0.7.0", default-features = false, features = [
"codec",
"io",
], optional = true }

## socks
tokio-socks = { version = "0.5.2", optional = true }
Expand All @@ -154,8 +170,20 @@ windows-registry = "0.3.0"
system-configuration = "0.6.0"

[dev-dependencies]
hyper = { version = "1.1.0", default-features = false, features = ["http1", "http2", "client", "server"] }
hyper-util = { version = "0.1.10", features = ["http1", "http2", "client", "client-legacy", "server-auto", "tokio"] }
hyper = { version = "1.1.0", default-features = false, features = [
"http1",
"http2",
"client",
"server",
] }
hyper-util = { version = "0.1.10", features = [
"http1",
"http2",
"client",
"client-legacy",
"server-auto",
"tokio",
] }
env_logger = "0.10.0"
serde = { version = "1.0", features = ["derive"] }
libflate = "2.0.0"
Expand Down Expand Up @@ -259,11 +287,6 @@ name = "websocket"
path = "examples/websocket.rs"
required-features = ["websocket", "futures-util/std"]

[[example]]
name = "websocket_via_proxy"
path = "examples/websocket_via_proxy.rs"
required-features = ["websocket", "futures-util/std", "socks"]

[[test]]
name = "cookie"
path = "tests/cookie.rs"
Expand Down
12 changes: 2 additions & 10 deletions examples/websocket.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,9 @@
use futures_util::{SinkExt, StreamExt, TryStreamExt};
use rquest::{tls::Impersonate, Client, Message};
use rquest::Message;

#[tokio::main]
async fn main() -> Result<(), rquest::Error> {
// Build a client to mimic Chrome130
let websocket = Client::builder()
.impersonate(Impersonate::Chrome130)
.build()?
.websocket("wss://echo.websocket.org")
.send()
.await?
.into_websocket()
.await?;
let websocket = rquest::websocket("wss://echo.websocket.org").await?;

let (mut tx, mut rx) = websocket.split();

Expand Down
37 changes: 0 additions & 37 deletions examples/websocket_via_proxy.rs
View file
Open in desktop

This file was deleted.

14 changes: 14 additions & 0 deletions src/tls/mimic/macros.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ macro_rules! header_chrome_ua {
macro_rules! header_chrome_accpet {
($headers:expr) => {
$headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
#[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))]
$headers.insert(
ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br"),
Expand All @@ -61,6 +62,12 @@ macro_rules! header_chrome_accpet {
};
(zstd, $headers:expr) => {
$headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
#[cfg(all(
feature = "gzip",
feature = "deflate",
feature = "brotli",
feature = "zstd"
))]
$headers.insert(
ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br, zstd"),
Expand Down Expand Up @@ -93,6 +100,7 @@ macro_rules! header_firefox_accept {
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
),
);
#[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))]
$headers.insert(
header::ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br"),
Expand All @@ -109,6 +117,12 @@ macro_rules! header_firefox_accept {
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
),
);
#[cfg(all(
feature = "gzip",
feature = "deflate",
feature = "brotli",
feature = "zstd"
))]
$headers.insert(
header::ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br, zstd"),
Expand Down
5 changes: 4 additions & 1 deletion src/tls/mimic/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,12 @@ mod impersonate_imports {
pub(crate) use crate::tls::{Http2Settings, ImpersonateSettings};
pub use crate::*;
pub use http::{
header::{ACCEPT, ACCEPT_ENCODING, ACCEPT_LANGUAGE, UPGRADE_INSECURE_REQUESTS, USER_AGENT},
header::{ACCEPT, ACCEPT_LANGUAGE, UPGRADE_INSECURE_REQUESTS, USER_AGENT},
HeaderMap, HeaderName, HeaderValue,
};

#[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))]
pub use http::header::ACCEPT_ENCODING;
}

mod tls_imports {
Expand Down
1 change: 1 addition & 0 deletions src/tls/mimic/okhttp.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ fn header_initializer(ua: &'static str) -> HeaderMap {
headers.insert(ACCEPT, HeaderValue::from_static("*/*"));
headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
headers.insert(USER_AGENT, HeaderValue::from_static(ua));
#[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))]
headers.insert(
ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br"),
Expand Down
3 changes: 3 additions & 0 deletions src/tls/mimic/safari.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ fn header_initializer_for_16_17(ua: &'static str) -> HeaderMap {
HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
);
headers.insert("sec-fetch-site", HeaderValue::from_static("none"));
#[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))]
headers.insert(
ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br"),
Expand All @@ -122,6 +123,7 @@ fn header_initializer_for_15(ua: &'static str) -> HeaderMap {
HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
);
headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
#[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))]
headers.insert(
ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br"),
Expand All @@ -142,6 +144,7 @@ fn header_initializer_for_18(ua: &'static str) -> HeaderMap {
headers.insert("sec-fetch-mode", HeaderValue::from_static("navigate"));
headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
headers.insert("priority", HeaderValue::from_static("u=0, i"));
#[cfg(all(feature = "gzip", feature = "deflate", feature = "brotli"))]
headers.insert(
ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br"),
Expand Down