diff --git a/Cargo.toml b/Cargo.toml index d3b55b5..958530a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,4 +16,5 @@ categories = ["network-programming", "web-programming::http-client"] [dependencies] rustls = "0.18.1" webpki-roots = "0.20.0" -webpki = "0.21.3" \ No newline at end of file +webpki = "0.21.3" +chunked_transfer = "1.2.0" \ No newline at end of file diff --git a/src/structs.rs b/src/structs.rs index 07148df..1205bf5 100644 --- a/src/structs.rs +++ b/src/structs.rs @@ -43,7 +43,6 @@ pub struct Response { pub cookies: HashMap, pub cookie_count: usize, pub body: Option, - pub chunk_size: Option, } #[derive(Debug, Clone)] @@ -71,8 +70,8 @@ pub struct WarpConfig { } impl Response { - pub fn new(raw: String, head_line: String /*config: WarpConfig*/) -> Response { - utils::new_response(raw, head_line) + pub fn new(body: String, head: Vec) -> Response { + utils::new_response(body, head) } } @@ -101,7 +100,7 @@ impl Request { } - pub fn send(&self) -> Result<(), Box> { + pub fn send(&self) -> Result> { return match self.protocol { HTTPVersion::HTTPS => { println!("HTTPS is experimental, we recommend switching to HTTP"); diff --git a/src/tcp.rs b/src/tcp.rs index 21bf19a..cd65171 100644 --- a/src/tcp.rs +++ b/src/tcp.rs @@ -1,9 +1,9 @@ use std::net::TcpStream; use crate::structs::Response; use std::io::{Write, Read, BufReader, BufRead}; +use chunked_transfer::Decoder; - -pub fn get>(domain: S, path: S) { +pub fn get>(domain: S, path: S) -> Response { let host = domain.into(); let location = path.into(); @@ -15,43 +15,46 @@ pub fn get>(domain: S, path: S) { stream.flush().unwrap(); let mut reader = BufReader::new(&mut stream); - let mut received: Vec = reader.fill_buf().unwrap().to_vec(); - - reader.consume(received.len()); - let mut response = String::from_utf8(received.clone()).unwrap(); - let mut lines = response.split("\r\n").collect::>().clone(); - let head_line = lines.first().unwrap().clone().to_string(); + let mut head_line = String::new(); + let mut lines: Vec = Vec::new(); - let mut parsed_response: Response = Response::new(lines.join("\r\n").clone(), head_line.clone()); - println!("{:#?}", parsed_response); + reader.read_line(&mut head_line); + lines.push(head_line.clone()); - let mut passes = 0; + while lines.last().unwrap() != &String::from("\r\n") { + let mut buf_str = String::new(); + reader.read_line(&mut buf_str); + lines.push(buf_str.clone()) + } - if parsed_response.chunk_size != None { - while parsed_response.chunk_size.clone().unwrap() > 0 && passes < 5{ - println!("Chunk Size: {}", parsed_response.chunk_size.clone().unwrap()); - let mut temp_buf = reader.fill_buf().unwrap().to_vec(); - if temp_buf == received { - break; - } - // while temp_buf.len() < reader.buffer().len() { - // temp_buf.extend(reader.fill_buf().unwrap().to_vec()); - // } - reader.consume(temp_buf.len()); + lines.pop(); - received.extend(temp_buf); + let head = lines; - response = String::from_utf8(received.clone()).unwrap(); + lines = Vec::new(); + lines.push("FIRST".to_string()); - parsed_response = Response::new(response.clone(), head_line.clone()); + while lines.last().unwrap() != &String::from("\r\n") { + let mut buf_str = String::new(); - passes += 1; - } + reader.read_line(&mut buf_str); + lines.push(buf_str.clone()) } - return (); - // return parsed_response; + + let encoded = lines.join(""); + + let mut decoder = chunked_transfer::Decoder::new(encoded.as_bytes()); + + let mut response = String::new(); + decoder.read_to_string(&mut response); + + let mut parsed_response: Response = Response::new(response, head); + + println!("{:#?}", parsed_response); + + return parsed_response; } // last run got to id 43 \ No newline at end of file diff --git a/src/tests.rs b/src/tests.rs index 6496506..0d516b0 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -3,7 +3,6 @@ use std::time::Instant; const LIMIT: usize = 10000; -const TEST_STR: &str = "HTTP/1.1 301 TLS Redirect\r\nDate: Fri, 21 Aug 2020 17:42:29 GMT\r\nContent-Type: application/json; charset=utf-8\r\nConnection: keep-alive\r\nSet-Cookie: __cfduid=d1cd636ec4303be8a4ac9d8d01f93e1e71598031749; expires=Sun, 20-Sep-20 17:42:29 GMT; path=/; domain=.typicode.com; HttpOnly; SameSite=Lax\r\nX-Powered-By: Express\r\nX-Ratelimit-Limit: 1000\r\nX-Ratelimit-Remaining: 999\r\nX-Ratelimit-Reset: 1597842544\r\nVary: Origin, Accept-Encoding\r\nAccess-Control-Allow-Credentials: true\r\nCache-Control: max-age=43200\r\nPragma: no-cache\r\nExpires: -1\r\nX-Content-Type-Options: nosniff\r\nEtag: W/\"5ef7-4Ad6/n39KWY9q6Ykm/ULNQ2F5IM\"\r\nVia: 1.1 vegur\r\nCF-Cache-Status: HIT\r\nAge: 10212\r\ncf-request-id: 04b3b67aed0000e608b91e0200000001\r\nServer: cloudflare\r\nCF-RAY: 5c6626a4ad9ae608-LHR"; #[test] fn test_get() { @@ -31,8 +30,32 @@ fn test_request_builder_get() { #[test] fn bench_response_parsing() { bench("parse response", LIMIT, || { - let header_line = "HTTP/1.1 301 TLS Redirect\r\n".to_string(); - let response = crate::structs::Response::new(String::from(TEST_STR), header_line); + let header_line: Vec = vec!["HTTP/1.1 301 TLS Redirect\r\n".to_string()]; + let test_body: Vec = vec![ + "HTTP/1.1 301 TLS Redirect\r\n".to_string(), + "Date: Fri, 21 Aug 2020 17:42:29 GMT\r\n".to_string(), + "Content-Type: application/json; charset=utf-8\r\n".to_string(), + "Connection: keep-alive\r\n".to_string(), + "Set-Cookie: __cfduid=d1cd636ec4303be8a4ac9d8d01f93e1e71598031749; expires=Sun, 20-Sep-20 17:42:29 GMT; path=/; domain=.typicode.com; HttpOnly; SameSite=Lax\r\n".to_string(), + "X-Powered-By: Express\r\n".to_string(), + "X-Ratelimit-Limit: 1000\r\n".to_string(), + "X-Ratelimit-Remaining: 999\r\n".to_string(), + "X-Ratelimit-Reset: 1597842544\r\n".to_string(), + "Vary: Origin, Accept-Encoding\r\n".to_string(), + "Access-Control-Allow-Credentials: true\r\n".to_string(), + "Cache-Control: max-age=43200\r\n".to_string(), + "Pragma: no-cache\r\n".to_string(), + "Expires: -1\r\n".to_string(), + "X-Content-Type-Options: nosniff\r\n".to_string(), + "Etag: W/\"5ef7-4Ad6/n39KWY9q6Ykm/ULNQ2F5IM\"\r\n".to_string(), + "Via: 1.1 vegur\r\n".to_string(), + "CF-Cache-Status: HIT\r\n".to_string(), + "Age: 10212\r\n".to_string(), + "cf-request-id: 04b3b67aed0000e608b91e0200000001\r\n".to_string(), + "Server: cloudflare\r\n".to_string(), + "CF-RAY: 5c6626a4ad9ae608-LHR".to_string() + ]; + let response = crate::structs::Response::new(test_body.join(""), header_line); }) } @@ -40,7 +63,7 @@ fn bench_response_parsing() { fn bench_cookie_parsing() { let cookie = "Set-Cookie: has_recent_activity=1; path=/; expires=Fri, 21 Aug 2020 21:11:53 GMT; secure; HttpOnly; SameSite=Lax"; bench("parse cookie", LIMIT, || { - let cookie = crate::utils::parse_cookie(cookie); + let cookie = crate::utils::parse_cookie(cookie.to_string()); }) } @@ -48,7 +71,7 @@ fn bench_cookie_parsing() { fn bench_header_parsing() { let header = "Date: Fri, 21 Aug 2020 17:42:29 GMT"; bench("parse header", LIMIT, || { - let header = crate::utils::parse_header(header); + let header = crate::utils::parse_header(header.to_string()); }) } diff --git a/src/tls.rs b/src/tls.rs index 1254216..9261769 100644 --- a/src/tls.rs +++ b/src/tls.rs @@ -4,8 +4,9 @@ use webpki_roots::TLS_SERVER_ROOTS; use webpki::*; use std::sync::Arc; use std::io::Write; +use crate::structs::Response; -pub fn get>(domain: S, path: S) { +pub fn get>(domain: S, path: S) -> Response { let formatted_domain = domain.into(); let formatted_path = path.into(); let config = Arc::new(build_tls_config()); @@ -13,6 +14,18 @@ pub fn get>(domain: S, path: S) { let mut client = ClientSession::new(&config, domain_ref); let request = format!("GET {} HTTP/1.1\r\nUser-Agent: Warp/1.0\r\nHost: {}\r\nConnection: Keep-Alive\r\n\r\n", formatted_path, formatted_domain); client.write_all(request.as_bytes()).unwrap(); + + return Response { + raw: "".to_string(), + protocol: None, + status: None, + status_text: None, + headers: Default::default(), + header_count: 0, + cookies: Default::default(), + cookie_count: 0, + body: None, + } } fn build_tls_config() -> ClientConfig { diff --git a/src/utils.rs b/src/utils.rs index 192d6a7..2701d54 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -2,7 +2,7 @@ use std::fmt::Write; use std::collections::HashMap; use crate::structs::{Cookie, Header, Response}; -pub fn parse_cookie(line: &str) -> Cookie { +pub fn parse_cookie(line: String) -> Cookie { let mut formatted = line.split("Set-Cookie:").collect::>(); let args = formatted.last().unwrap().split(';').collect::>(); let mut parsed_args = HashMap::::new(); @@ -85,7 +85,7 @@ pub fn parse_cookie(line: &str) -> Cookie { }; } -pub fn parse_header(line: &str) -> Header { +pub fn parse_header(line: String) -> Header { let mut parsed_args = HashMap::::new(); let mut keypair = line.split(": ").collect::>(); keypair.reverse(); @@ -99,14 +99,18 @@ pub fn parse_header(line: &str) -> Header { if key_name.len() == 1 as usize { key = key_name.last().unwrap().to_string(); } - let value = keypair.join("="); + let mut value = keypair.join("="); + value = value.split("\r\n").collect::>().join(""); return Header { name: Some(key), value: Some(value), }; } -pub fn new_response(raw: String, head_line: String) -> Response { +pub fn new_response(mut body_text: String, mut head: Vec) -> Response { + head.reverse(); + let head_line = head.pop().unwrap(); + head.reverse(); let mut head_content: Vec<&str> = head_line.split_ascii_whitespace().collect(); head_content.reverse(); let protocol = head_content.pop().unwrap().to_owned(); @@ -114,51 +118,31 @@ pub fn new_response(raw: String, head_line: String) -> Response { head_content.reverse(); let status_text = head_content.join(" "); - let lines = raw.split("\r\n").collect::>(); + let mut cookies = HashMap::::new(); let mut headers = HashMap::::new(); - let mut is_body = false; - let mut body_lines = Vec::<&str>::new(); - for line in lines { - if !line.starts_with("HTTP") { - if line.starts_with("Set-Cookie:") { - let cookie = parse_cookie(line); - cookies.insert(cookie.name.clone().unwrap(), cookie); - } else { - if line == "" && !is_body || line == "\n" && !is_body { - is_body = true; - } else if is_body { - body_lines.push(line); - } else { - let header = parse_header(line); - headers.insert(header.name.unwrap(), header.value.unwrap()); - } - } + + for line in head { + if line.starts_with("Set-Cookie:") { + let cookie = parse_cookie(line); + cookies.insert(cookie.name.clone().unwrap(), cookie); + } else { + let header = parse_header(line); + headers.insert(header.name.unwrap(), header.value.unwrap()); } } - let header_count = headers.len(); let cookie_count = cookies.len(); - let mut chunk_size: Option = None; - let encoding = headers.get("Transfer-Encoding"); - if encoding != None { - if encoding.unwrap() == &String::from("chunked") { - body_lines.reverse(); - chunk_size = Some(i64::from_str_radix(body_lines.pop().unwrap(), 16).unwrap()); - body_lines.reverse(); - } - } - let mut body = None; - if body_lines.len() > 0 { - body = Some(body_lines.join("\n")) + if body_text.len() > 0 { + body = Some(body_text) } Response { - raw: raw.escape_default().to_string(), + raw: body.clone().unwrap_or(String::new()).escape_default().to_string(), protocol: Some(protocol), status: Some(status.parse::().unwrap()), status_text: Some(status_text), @@ -167,6 +151,5 @@ pub fn new_response(raw: String, head_line: String) -> Response { headers, header_count, body, - chunk_size, } } \ No newline at end of file