Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit dfffb5d

Browse files
dis-archivedis-archive
committedFeb 17, 2025·
Allow recording the raw HTTP/1 headers sent and received
This allows feeding them into a WARC file (https://en.wikipedia.org/wiki/WARC_(file_format)). The rest of the request and response is already available as it's either the explicitly set body or the received response body. This includes the final `\r\n\r\n` between the headers and the response (so that it can be distinguished from bare `\n\n`). Needed for a similar request in reqwest: seanmonstar/reqwest#1229.
1 parent 621d8e4 commit dfffb5d

File tree

6 files changed

+144
-4
lines changed

6 files changed

+144
-4
lines changed
 

‎src/client/conn/http1.rs

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ pub struct Builder {
112112
h1_parser_config: ParserConfig,
113113
h1_writev: Option<bool>,
114114
h1_title_case_headers: bool,
115+
h1_record_raw_request_headers: bool,
116+
h1_record_raw_response_headers: bool,
115117
h1_preserve_header_case: bool,
116118
h1_max_headers: Option<usize>,
117119
#[cfg(feature = "ffi")]
@@ -312,6 +314,8 @@ impl Builder {
312314
h1_read_buf_exact_size: None,
313315
h1_parser_config: Default::default(),
314316
h1_title_case_headers: false,
317+
h1_record_raw_request_headers: false,
318+
h1_record_raw_response_headers: false,
315319
h1_preserve_header_case: false,
316320
h1_max_headers: None,
317321
#[cfg(feature = "ffi")]
@@ -428,6 +432,22 @@ impl Builder {
428432
self
429433
}
430434

435+
/// Set whether to record the raw headers sent.
436+
///
437+
/// Default is false.
438+
pub fn record_raw_request_headers(&mut self, enabled: bool) -> &mut Builder {
439+
self.h1_record_raw_request_headers = enabled;
440+
self
441+
}
442+
443+
/// Set whether to record the raw headers received.
444+
///
445+
/// Default is false.
446+
pub fn record_raw_response_headers(&mut self, enabled: bool) -> &mut Builder {
447+
self.h1_record_raw_response_headers = enabled;
448+
self
449+
}
450+
431451
/// Set whether to support preserving original header cases.
432452
///
433453
/// Currently, this will record the original cases received, and store them
@@ -539,6 +559,12 @@ impl Builder {
539559
if opts.h1_title_case_headers {
540560
conn.set_title_case_headers();
541561
}
562+
if opts.h1_record_raw_request_headers {
563+
conn.set_record_raw_request_headers();
564+
}
565+
if opts.h1_record_raw_response_headers {
566+
conn.set_record_raw_response_headers();
567+
}
542568
if opts.h1_preserve_header_case {
543569
conn.set_preserve_header_case();
544570
}

‎src/ext/mod.rs

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,38 @@ impl fmt::Debug for Protocol {
8686
}
8787
}
8888

89+
/// Raw request headers as sent over the TLS or TCP connection.
90+
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
91+
#[derive(Clone, Debug)]
92+
pub struct RawRequestHeaders(Bytes);
93+
94+
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
95+
impl RawRequestHeaders {
96+
/// Returns the raw bytes sent for the header of the request.
97+
pub fn as_bytes(&self) -> &[u8] {
98+
&self.0
99+
}
100+
pub(crate) fn from(bytes: Bytes) -> Self {
101+
Self(bytes)
102+
}
103+
}
104+
105+
/// Raw response headers as sent over the TLS or TCP connection.
106+
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
107+
#[derive(Clone, Debug)]
108+
pub struct RawResponseHeaders(Bytes);
109+
110+
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
111+
impl RawResponseHeaders {
112+
/// Returns the raw bytes received for the header of the response.
113+
pub fn as_bytes(&self) -> &[u8] {
114+
&self.0
115+
}
116+
pub(crate) fn from(bytes: Bytes) -> Self {
117+
Self(bytes)
118+
}
119+
}
120+
89121
/// A map from header names to their original casing as received in an HTTP message.
90122
///
91123
/// If an HTTP/1 response `res` is parsed on a connection whose option

‎src/proto/h1/conn.rs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ where
6868
date_header: true,
6969
#[cfg(feature = "server")]
7070
timer: Time::Empty,
71+
raw_request_headers: None,
72+
record_raw_request_headers: false,
73+
record_raw_response_headers: false,
7174
preserve_header_case: false,
7275
#[cfg(feature = "ffi")]
7376
preserve_header_order: false,
@@ -123,6 +126,14 @@ where
123126
self.state.title_case_headers = true;
124127
}
125128

129+
pub(crate) fn set_record_raw_request_headers(&mut self) {
130+
self.state.record_raw_request_headers = true;
131+
}
132+
133+
pub(crate) fn set_record_raw_response_headers(&mut self) {
134+
self.state.record_raw_response_headers = true;
135+
}
136+
126137
pub(crate) fn set_preserve_header_case(&mut self) {
127138
self.state.preserve_header_case = true;
128139
}
@@ -241,6 +252,8 @@ where
241252
req_method: &mut self.state.method,
242253
h1_parser_config: self.state.h1_parser_config.clone(),
243254
h1_max_headers: self.state.h1_max_headers,
255+
raw_request_headers: self.state.raw_request_headers.as_ref(),
256+
record_raw_headers: self.state.record_raw_response_headers,
244257
preserve_header_case: self.state.preserve_header_case,
245258
#[cfg(feature = "ffi")]
246259
preserve_header_order: self.state.preserve_header_order,
@@ -617,6 +630,7 @@ where
617630
self.enforce_version(&mut head);
618631

619632
let buf = self.io.headers_buf();
633+
let headers_start = buf.len();
620634
match super::role::encode_headers::<T>(
621635
Encode {
622636
head: &mut head,
@@ -633,6 +647,11 @@ where
633647
Ok(encoder) => {
634648
debug_assert!(self.state.cached_headers.is_none());
635649
debug_assert!(head.headers.is_empty());
650+
if self.state.record_raw_request_headers {
651+
self.state.raw_request_headers = Some(crate::ext::RawRequestHeaders::from(Bytes::copy_from_slice(&buf[headers_start..])));
652+
} else {
653+
self.state.raw_request_headers = None;
654+
}
636655
self.state.cached_headers = Some(head.headers);
637656

638657
#[cfg(feature = "client")]
@@ -934,6 +953,9 @@ struct State {
934953
date_header: bool,
935954
#[cfg(feature = "server")]
936955
timer: Time,
956+
raw_request_headers: Option<crate::ext::RawRequestHeaders>,
957+
record_raw_request_headers: bool,
958+
record_raw_response_headers: bool,
937959
preserve_header_case: bool,
938960
#[cfg(feature = "ffi")]
939961
preserve_header_order: bool,

‎src/proto/h1/io.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,8 @@ where
184184
req_method: parse_ctx.req_method,
185185
h1_parser_config: parse_ctx.h1_parser_config.clone(),
186186
h1_max_headers: parse_ctx.h1_max_headers,
187+
raw_request_headers: parse_ctx.raw_request_headers,
188+
record_raw_headers: parse_ctx.record_raw_headers,
187189
preserve_header_case: parse_ctx.preserve_header_case,
188190
#[cfg(feature = "ffi")]
189191
preserve_header_order: parse_ctx.preserve_header_order,
@@ -706,6 +708,8 @@ mod tests {
706708
req_method: &mut None,
707709
h1_parser_config: Default::default(),
708710
h1_max_headers: None,
711+
raw_request_headers: None,
712+
record_raw_headers: false,
709713
preserve_header_case: false,
710714
#[cfg(feature = "ffi")]
711715
preserve_header_order: false,

‎src/proto/h1/mod.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ pub(crate) struct ParseContext<'a> {
7373
req_method: &'a mut Option<Method>,
7474
h1_parser_config: ParserConfig,
7575
h1_max_headers: Option<usize>,
76+
record_raw_headers: bool,
77+
raw_request_headers: Option<&'a crate::ext::RawRequestHeaders>,
7678
preserve_header_case: bool,
7779
#[cfg(feature = "ffi")]
7880
preserve_header_order: bool,

‎src/proto/h1/role.rs

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use crate::body::DecodedLength;
1818
use crate::common::date;
1919
use crate::error::Parse;
2020
use crate::ext::HeaderCaseMap;
21+
use crate::ext::RawResponseHeaders;
2122
#[cfg(feature = "ffi")]
2223
use crate::ext::OriginalHeaderOrder;
2324
use crate::headers;
@@ -1056,19 +1057,32 @@ impl Http1Transaction for Client {
10561057
};
10571058

10581059
let mut slice = buf.split_to(len);
1060+
let raw_headers;
10591061

1060-
if ctx
1062+
let slice = if ctx
10611063
.h1_parser_config
10621064
.obsolete_multiline_headers_in_responses_are_allowed()
10631065
{
1066+
raw_headers = if ctx.record_raw_headers {
1067+
Some(RawResponseHeaders::from(slice.clone().freeze()))
1068+
} else {
1069+
None
1070+
};
10641071
for header in &mut headers_indices[..headers_len] {
10651072
// SAFETY: array is valid up to `headers_len`
10661073
let header = unsafe { header.assume_init_mut() };
10671074
Client::obs_fold_line(&mut slice, header);
10681075
}
1069-
}
1070-
1071-
let slice = slice.freeze();
1076+
slice.freeze()
1077+
} else {
1078+
let slice = slice.freeze();
1079+
raw_headers = if ctx.record_raw_headers {
1080+
Some(RawResponseHeaders::from(slice.clone()))
1081+
} else {
1082+
None
1083+
};
1084+
slice
1085+
};
10721086

10731087
let mut headers = ctx.cached_headers.take().unwrap_or_default();
10741088

@@ -1119,6 +1133,14 @@ impl Http1Transaction for Client {
11191133

11201134
let mut extensions = http::Extensions::default();
11211135

1136+
if let Some(raw_request_headers) = ctx.raw_request_headers {
1137+
extensions.insert(raw_request_headers.clone());
1138+
}
1139+
1140+
if let Some(raw_headers) = raw_headers {
1141+
extensions.insert(raw_headers);
1142+
}
1143+
11221144
if let Some(header_case_map) = header_case_map {
11231145
extensions.insert(header_case_map);
11241146
}
@@ -1656,6 +1678,8 @@ mod tests {
16561678
req_method: &mut method,
16571679
h1_parser_config: Default::default(),
16581680
h1_max_headers: None,
1681+
raw_request_headers: None,
1682+
record_raw_headers: false,
16591683
preserve_header_case: false,
16601684
#[cfg(feature = "ffi")]
16611685
preserve_header_order: false,
@@ -1684,6 +1708,8 @@ mod tests {
16841708
req_method: &mut Some(crate::Method::GET),
16851709
h1_parser_config: Default::default(),
16861710
h1_max_headers: None,
1711+
raw_request_headers: None,
1712+
record_raw_headers: false,
16871713
preserve_header_case: false,
16881714
#[cfg(feature = "ffi")]
16891715
preserve_header_order: false,
@@ -1708,6 +1734,8 @@ mod tests {
17081734
req_method: &mut None,
17091735
h1_parser_config: Default::default(),
17101736
h1_max_headers: None,
1737+
raw_request_headers: None,
1738+
record_raw_headers: false,
17111739
preserve_header_case: false,
17121740
#[cfg(feature = "ffi")]
17131741
preserve_header_order: false,
@@ -1729,6 +1757,8 @@ mod tests {
17291757
req_method: &mut Some(crate::Method::GET),
17301758
h1_parser_config: Default::default(),
17311759
h1_max_headers: None,
1760+
raw_request_headers: None,
1761+
record_raw_headers: false,
17321762
preserve_header_case: false,
17331763
#[cfg(feature = "ffi")]
17341764
preserve_header_order: false,
@@ -1752,6 +1782,8 @@ mod tests {
17521782
req_method: &mut Some(crate::Method::GET),
17531783
h1_parser_config: Default::default(),
17541784
h1_max_headers: None,
1785+
raw_request_headers: None,
1786+
record_raw_headers: false,
17551787
preserve_header_case: false,
17561788
#[cfg(feature = "ffi")]
17571789
preserve_header_order: false,
@@ -1779,6 +1811,8 @@ mod tests {
17791811
req_method: &mut Some(crate::Method::GET),
17801812
h1_parser_config,
17811813
h1_max_headers: None,
1814+
raw_request_headers: None,
1815+
record_raw_headers: false,
17821816
preserve_header_case: false,
17831817
#[cfg(feature = "ffi")]
17841818
preserve_header_order: false,
@@ -1803,6 +1837,8 @@ mod tests {
18031837
req_method: &mut Some(crate::Method::GET),
18041838
h1_parser_config: Default::default(),
18051839
h1_max_headers: None,
1840+
raw_request_headers: None,
1841+
record_raw_headers: false,
18061842
preserve_header_case: false,
18071843
#[cfg(feature = "ffi")]
18081844
preserve_header_order: false,
@@ -1823,6 +1859,8 @@ mod tests {
18231859
req_method: &mut None,
18241860
h1_parser_config: Default::default(),
18251861
h1_max_headers: None,
1862+
raw_request_headers: None,
1863+
record_raw_headers: false,
18261864
preserve_header_case: true,
18271865
#[cfg(feature = "ffi")]
18281866
preserve_header_order: false,
@@ -1862,6 +1900,8 @@ mod tests {
18621900
req_method: &mut None,
18631901
h1_parser_config: Default::default(),
18641902
h1_max_headers: None,
1903+
raw_request_headers: None,
1904+
record_raw_headers: false,
18651905
preserve_header_case: false,
18661906
#[cfg(feature = "ffi")]
18671907
preserve_header_order: false,
@@ -1883,6 +1923,8 @@ mod tests {
18831923
req_method: &mut None,
18841924
h1_parser_config: Default::default(),
18851925
h1_max_headers: None,
1926+
raw_request_headers: None,
1927+
record_raw_headers: false,
18861928
preserve_header_case: false,
18871929
#[cfg(feature = "ffi")]
18881930
preserve_header_order: false,
@@ -2113,6 +2155,8 @@ mod tests {
21132155
req_method: &mut Some(Method::GET),
21142156
h1_parser_config: Default::default(),
21152157
h1_max_headers: None,
2158+
raw_request_headers: None,
2159+
record_raw_headers: false,
21162160
preserve_header_case: false,
21172161
#[cfg(feature = "ffi")]
21182162
preserve_header_order: false,
@@ -2134,6 +2178,8 @@ mod tests {
21342178
req_method: &mut Some(m),
21352179
h1_parser_config: Default::default(),
21362180
h1_max_headers: None,
2181+
raw_request_headers: None,
2182+
record_raw_headers: false,
21372183
preserve_header_case: false,
21382184
#[cfg(feature = "ffi")]
21392185
preserve_header_order: false,
@@ -2155,6 +2201,8 @@ mod tests {
21552201
req_method: &mut Some(Method::GET),
21562202
h1_parser_config: Default::default(),
21572203
h1_max_headers: None,
2204+
raw_request_headers: None,
2205+
record_raw_headers: false,
21582206
preserve_header_case: false,
21592207
#[cfg(feature = "ffi")]
21602208
preserve_header_order: false,
@@ -2725,6 +2773,8 @@ mod tests {
27252773
req_method: &mut Some(Method::GET),
27262774
h1_parser_config: Default::default(),
27272775
h1_max_headers: None,
2776+
raw_request_headers: None,
2777+
record_raw_headers: false,
27282778
preserve_header_case: false,
27292779
#[cfg(feature = "ffi")]
27302780
preserve_header_order: false,
@@ -2769,6 +2819,8 @@ mod tests {
27692819
req_method: &mut None,
27702820
h1_parser_config: Default::default(),
27712821
h1_max_headers: max_headers,
2822+
raw_request_headers: None,
2823+
record_raw_headers: false,
27722824
preserve_header_case: false,
27732825
#[cfg(feature = "ffi")]
27742826
preserve_header_order: false,
@@ -2793,6 +2845,8 @@ mod tests {
27932845
req_method: &mut None,
27942846
h1_parser_config: Default::default(),
27952847
h1_max_headers: max_headers,
2848+
raw_request_headers: None,
2849+
record_raw_headers: false,
27962850
preserve_header_case: false,
27972851
#[cfg(feature = "ffi")]
27982852
preserve_header_order: false,

0 commit comments

Comments
 (0)
Please sign in to comment.