oh wow, it actually works???? what is this fuckery

Cargo.toml

@@ -16,4 +16,5 @@ categories = ["network-programming", "web-programming::http-client"]
 [dependencies]
 rustls = "0.18.1"
 webpki-roots = "0.20.0"
-webpki = "0.21.3"
+webpki = "0.21.3"
+chunked_transfer = "1.2.0"

src/structs.rs

@@ -43,7 +43,6 @@ pub struct Response {
  pub cookies: HashMap<String, Cookie>,
  pub cookie_count: usize,
  pub body: Option<String>,
- pub chunk_size: Option<i64>,
 }
 
 #[derive(Debug, Clone)]
@@ -71,8 +70,8 @@ pub struct WarpConfig {
 }
 
 impl Response {
- pub fn new(raw: String, head_line: String /*config: WarpConfig*/) -> Response {
- utils::new_response(raw, head_line)
+ pub fn new(body: String, head: Vec<String>) -> Response {
+ utils::new_response(body, head)
  }
 }
 
@@ -101,7 +100,7 @@ impl Request {
  }
 
 
- pub fn send(&self) -> Result<(), Box<dyn Error>> {
+ pub fn send(&self) -> Result<Response, Box<dyn Error>> {
  return match self.protocol {
  HTTPVersion::HTTPS => {
  println!("HTTPS is experimental, we recommend switching to HTTP");

src/tcp.rs

@@ -1,9 +1,9 @@
 use std::net::TcpStream;
 use crate::structs::Response;
 use std::io::{Write, Read, BufReader, BufRead};
+use chunked_transfer::Decoder;
 
-
-pub fn get<S: Into<String>>(domain: S, path: S) {
+pub fn get<S: Into<String>>(domain: S, path: S) -> Response {
  let host = domain.into();
  let location = path.into();
 
@@ -15,43 +15,46 @@ pub fn get<S: Into<String>>(domain: S, path: S) {
  stream.flush().unwrap();
 
  let mut reader = BufReader::new(&mut stream);
- let mut received: Vec<u8> = reader.fill_buf().unwrap().to_vec();
-
- reader.consume(received.len());
- let mut response = String::from_utf8(received.clone()).unwrap();
- let mut lines = response.split("\r\n").collect::<Vec<&str>>().clone();
 
- let head_line = lines.first().unwrap().clone().to_string();
+ let mut head_line = String::new();
+ let mut lines: Vec<String> = Vec::new();
 
- let mut parsed_response: Response = Response::new(lines.join("\r\n").clone(), head_line.clone());
- println!("{:#?}", parsed_response);
+ reader.read_line(&mut head_line);
+ lines.push(head_line.clone());
 
- let mut passes = 0;
+ while lines.last().unwrap() != &String::from("\r\n") {
+ let mut buf_str = String::new();
+ reader.read_line(&mut buf_str);
+ lines.push(buf_str.clone())
+ }
 
- if parsed_response.chunk_size != None {
- while parsed_response.chunk_size.clone().unwrap() > 0 && passes < 5{
- println!("Chunk Size: {}", parsed_response.chunk_size.clone().unwrap());
- let mut temp_buf = reader.fill_buf().unwrap().to_vec();
- if temp_buf == received {
- break;
- }
- // while temp_buf.len() < reader.buffer().len() {
- // temp_buf.extend(reader.fill_buf().unwrap().to_vec());
- // }
- reader.consume(temp_buf.len());
+ lines.pop();
 
-  received.extend(temp_buf);
+ let head = lines;
 
- response = String::from_utf8(received.clone()).unwrap();
+ lines = Vec::new();
+ lines.push("FIRST".to_string());
 
- parsed_response = Response::new(response.clone(), head_line.clone());
+ while lines.last().unwrap() != &String::from("\r\n") {
+ let mut buf_str = String::new();
 
-  passes += 1;
- }
+ reader.read_line(&mut buf_str);
+ lines.push(buf_str.clone())
  }
 
- return ();
- // return parsed_response;
+
+ let encoded = lines.join("");
+
+ let mut decoder = chunked_transfer::Decoder::new(encoded.as_bytes());
+
+ let mut response = String::new();
+ decoder.read_to_string(&mut response);
+
+ let mut parsed_response: Response = Response::new(response, head);
+
+ println!("{:#?}", parsed_response);
+
+ return parsed_response;
 }
 
 // last run got to id 43

src/tests.rs

@@ -3,7 +3,6 @@ use std::time::Instant;
 
 
 const LIMIT: usize = 10000;
-const TEST_STR: &str = "HTTP/1.1 301 TLS Redirect\r\nDate: Fri, 21 Aug 2020 17:42:29 GMT\r\nContent-Type: application/json; charset=utf-8\r\nConnection: keep-alive\r\nSet-Cookie: __cfduid=d1cd636ec4303be8a4ac9d8d01f93e1e71598031749; expires=Sun, 20-Sep-20 17:42:29 GMT; path=/; domain=.typicode.com; HttpOnly; SameSite=Lax\r\nX-Powered-By: Express\r\nX-Ratelimit-Limit: 1000\r\nX-Ratelimit-Remaining: 999\r\nX-Ratelimit-Reset: 1597842544\r\nVary: Origin, Accept-Encoding\r\nAccess-Control-Allow-Credentials: true\r\nCache-Control: max-age=43200\r\nPragma: no-cache\r\nExpires: -1\r\nX-Content-Type-Options: nosniff\r\nEtag: W/\"5ef7-4Ad6/n39KWY9q6Ykm/ULNQ2F5IM\"\r\nVia: 1.1 vegur\r\nCF-Cache-Status: HIT\r\nAge: 10212\r\ncf-request-id: 04b3b67aed0000e608b91e0200000001\r\nServer: cloudflare\r\nCF-RAY: 5c6626a4ad9ae608-LHR";
 
 #[test]
 fn test_get() {
@@ -31,24 +30,48 @@ fn test_request_builder_get() {
 #[test]
 fn bench_response_parsing() {
  bench("parse response", LIMIT, || {
- let header_line = "HTTP/1.1 301 TLS Redirect\r\n".to_string();
- let response = crate::structs::Response::new(String::from(TEST_STR), header_line);
+ let header_line: Vec<String> = vec!["HTTP/1.1 301 TLS Redirect\r\n".to_string()];
+ let test_body: Vec<String> = vec![
+ "HTTP/1.1 301 TLS Redirect\r\n".to_string(),
+ "Date: Fri, 21 Aug 2020 17:42:29 GMT\r\n".to_string(),
+ "Content-Type: application/json; charset=utf-8\r\n".to_string(),
+ "Connection: keep-alive\r\n".to_string(),
+ "Set-Cookie: __cfduid=d1cd636ec4303be8a4ac9d8d01f93e1e71598031749; expires=Sun, 20-Sep-20 17:42:29 GMT; path=/; domain=.typicode.com; HttpOnly; SameSite=Lax\r\n".to_string(),
+ "X-Powered-By: Express\r\n".to_string(),
+ "X-Ratelimit-Limit: 1000\r\n".to_string(),
+ "X-Ratelimit-Remaining: 999\r\n".to_string(),
+ "X-Ratelimit-Reset: 1597842544\r\n".to_string(),
+ "Vary: Origin, Accept-Encoding\r\n".to_string(),
+ "Access-Control-Allow-Credentials: true\r\n".to_string(),
+ "Cache-Control: max-age=43200\r\n".to_string(),
+ "Pragma: no-cache\r\n".to_string(),
+ "Expires: -1\r\n".to_string(),
+ "X-Content-Type-Options: nosniff\r\n".to_string(),
+ "Etag: W/\"5ef7-4Ad6/n39KWY9q6Ykm/ULNQ2F5IM\"\r\n".to_string(),
+ "Via: 1.1 vegur\r\n".to_string(),
+ "CF-Cache-Status: HIT\r\n".to_string(),
+ "Age: 10212\r\n".to_string(),
+ "cf-request-id: 04b3b67aed0000e608b91e0200000001\r\n".to_string(),
+ "Server: cloudflare\r\n".to_string(),
+ "CF-RAY: 5c6626a4ad9ae608-LHR".to_string()
+ ];
+ let response = crate::structs::Response::new(test_body.join(""), header_line);
  })
 }
 
 #[test]
 fn bench_cookie_parsing() {
  let cookie = "Set-Cookie: has_recent_activity=1; path=/; expires=Fri, 21 Aug 2020 21:11:53 GMT; secure; HttpOnly; SameSite=Lax";
  bench("parse cookie", LIMIT, || {
- let cookie = crate::utils::parse_cookie(cookie);
+ let cookie = crate::utils::parse_cookie(cookie.to_string());
  })
 }
 
 #[test]
 fn bench_header_parsing() {
  let header = "Date: Fri, 21 Aug 2020 17:42:29 GMT";
  bench("parse header", LIMIT, || {
- let header = crate::utils::parse_header(header);
+ let header = crate::utils::parse_header(header.to_string());
  })
 }
 

src/tls.rs

@@ -4,15 +4,28 @@ use webpki_roots::TLS_SERVER_ROOTS;
 use webpki::*;
 use std::sync::Arc;
 use std::io::Write;
+use crate::structs::Response;
 
-pub fn get<S: Into<String>>(domain: S, path: S) {
+pub fn get<S: Into<String>>(domain: S, path: S) -> Response {
  let formatted_domain = domain.into();
  let formatted_path = path.into();
  let config = Arc::new(build_tls_config());
  let domain_ref = DNSNameRef::try_from_ascii_str(formatted_domain.as_str()).unwrap();
  let mut client = ClientSession::new(&config, domain_ref);
  let request = format!("GET {} HTTP/1.1\r\nUser-Agent: Warp/1.0\r\nHost: {}\r\nConnection: Keep-Alive\r\n\r\n", formatted_path, formatted_domain);
  client.write_all(request.as_bytes()).unwrap();
+
+ return Response {
+ raw: "".to_string(),
+ protocol: None,
+ status: None,
+ status_text: None,
+ headers: Default::default(),
+ header_count: 0,
+ cookies: Default::default(),
+ cookie_count: 0,
+ body: None,
+ }
 }
 
 fn build_tls_config() -> ClientConfig {

src/utils.rs

@@ -2,7 +2,7 @@ use std::fmt::Write;
 use std::collections::HashMap;
 use crate::structs::{Cookie, Header, Response};
 
-pub fn parse_cookie(line: &str) -> Cookie {
+pub fn parse_cookie(line: String) -> Cookie {
  let mut formatted = line.split("Set-Cookie:").collect::<Vec<&str>>();
  let args = formatted.last().unwrap().split(';').collect::<Vec<&str>>();
  let mut parsed_args = HashMap::<String, String>::new();
@@ -85,7 +85,7 @@ pub fn parse_cookie(line: &str) -> Cookie {
  };
 }
 
-pub fn parse_header(line: &str) -> Header {
+pub fn parse_header(line: String) -> Header {
  let mut parsed_args = HashMap::<String, String>::new();
  let mut keypair = line.split(": ").collect::<Vec<&str>>();
  keypair.reverse();
@@ -99,66 +99,50 @@ pub fn parse_header(line: &str) -> Header {
  if key_name.len() == 1 as usize {
  key = key_name.last().unwrap().to_string();
  }
- let value = keypair.join("=");
+ let mut value = keypair.join("=");
+ value = value.split("\r\n").collect::<Vec<&str>>().join("");
  return Header {
  name: Some(key),
  value: Some(value),
  };
 }
 
-pub fn new_response(raw: String, head_line: String) -> Response {
+pub fn new_response(mut body_text: String, mut head: Vec<String>) -> Response {
+ head.reverse();
+ let head_line = head.pop().unwrap();
+ head.reverse();
  let mut head_content: Vec<&str> = head_line.split_ascii_whitespace().collect();
  head_content.reverse();
  let protocol = head_content.pop().unwrap().to_owned();
  let status = head_content.pop().unwrap().to_owned();
  head_content.reverse();
  let status_text = head_content.join(" ");
 
- let lines = raw.split("\r\n").collect::<Vec<&str>>();
+
  let mut cookies = HashMap::<String, Cookie>::new();
  let mut headers = HashMap::<String, String>::new();
- let mut is_body = false;
- let mut body_lines = Vec::<&str>::new();
- for line in lines {
- if !line.starts_with("HTTP") {
- if line.starts_with("Set-Cookie:") {
- let cookie = parse_cookie(line);
- cookies.insert(cookie.name.clone().unwrap(), cookie);
- } else {
- if line == "" && !is_body || line == "\n" && !is_body {
- is_body = true;
- } else if is_body {
- body_lines.push(line);
- } else {
- let header = parse_header(line);
- headers.insert(header.name.unwrap(), header.value.unwrap());
- }
- }
+
+ for line in head {
+ if line.starts_with("Set-Cookie:") {
+ let cookie = parse_cookie(line);
+ cookies.insert(cookie.name.clone().unwrap(), cookie);
+ } else {
+ let header = parse_header(line);
+ headers.insert(header.name.unwrap(), header.value.unwrap());
  }
  }
 
-
  let header_count = headers.len();
  let cookie_count = cookies.len();
 
- let mut chunk_size: Option<i64> = None;
- let encoding = headers.get("Transfer-Encoding");
- if encoding != None {
- if encoding.unwrap() == &String::from("chunked") {
- body_lines.reverse();
- chunk_size = Some(i64::from_str_radix(body_lines.pop().unwrap(), 16).unwrap());
- body_lines.reverse();
- }
- }
-
  let mut body = None;
 
- if body_lines.len() > 0 {
- body = Some(body_lines.join("\n"))
+ if body_text.len() > 0 {
+ body = Some(body_text)
  }
 
  Response {
- raw: raw.escape_default().to_string(),
+ raw: body.clone().unwrap_or(String::new()).escape_default().to_string(),
  protocol: Some(protocol),
  status: Some(status.parse::<isize>().unwrap()),
  status_text: Some(status_text),
@@ -167,6 +151,5 @@ pub fn new_response(raw: String, head_line: String) -> Response {
  headers,
  header_count,
  body,
- chunk_size,
  }
 }