From 6083c6cc7060441be70f1688b11a10b298cc2684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 00:08:27 +0800 Subject: [PATCH 01/44] feat(http): wip client --- Cargo.toml | 3 + compio-http/Cargo.toml | 30 ++++++ compio-http/examples/client.rs | 18 ++++ compio-http/src/client.rs | 34 +++++++ compio-http/src/lib.rs | 5 + compio-http/src/stream.rs | 175 +++++++++++++++++++++++++++++++++ 6 files changed, 265 insertions(+) create mode 100644 compio-http/Cargo.toml create mode 100644 compio-http/examples/client.rs create mode 100644 compio-http/src/client.rs create mode 100644 compio-http/src/lib.rs create mode 100644 compio-http/src/stream.rs diff --git a/Cargo.toml b/Cargo.toml index 2b6608a3..3cf1a419 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "compio-dispatcher", "compio-io", "compio-tls", + "compio-http", ] resolver = "2" @@ -31,3 +32,5 @@ compio-io = { path = "./compio-io", version = "0.1.0" } compio-net = { path = "./compio-net", version = "0.1.0" } compio-signal = { path = "./compio-signal", version = "0.1.0" } compio-dispatcher = { path = "./compio-dispatcher", version = "0.1.0" } + +compio-tls = { path = "./compio-tls", version = "0.1.0" } diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml new file mode 100644 index 00000000..58cda129 --- /dev/null +++ b/compio-http/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "compio-http" +version = "0.1.0" +categories = ["asynchronous", "network-programming"] +keywords = ["async", "net"] +edition = { workspace = true } +authors = { workspace = true } +readme = { workspace = true } +license = { workspace = true } +repository = { workspace = true } + +[dependencies] +compio-buf = { workspace = true } +compio-io = { workspace = true } +compio-net = { workspace = true, features = ["runtime"] } +compio-tls = { workspace = true } + +native-tls = { version = "0.2", optional = true } + +http = "0.2" +httparse = "1" + +[dev-dependencies] +compio-runtime = { workspace = true } +compio-macros = { workspace = true } + +[features] +default = ["native-tls"] +native-tls = ["compio-tls/native-tls", "dep:native-tls"] +vendored = ["native-tls?/vendored"] diff --git a/compio-http/examples/client.rs b/compio-http/examples/client.rs new file mode 100644 index 00000000..86c26921 --- /dev/null +++ b/compio-http/examples/client.rs @@ -0,0 +1,18 @@ +use compio_http::Client; +use http::{HeaderValue, Method, Request, Version}; + +#[compio_macros::main] +async fn main() { + let client = Client::new(); + let mut request = Request::new(vec![]); + *request.method_mut() = Method::GET; + *request.uri_mut() = "https://www.example.com/".parse().unwrap(); + *request.version_mut() = Version::HTTP_11; + let headers = request.headers_mut(); + headers.append("Host", HeaderValue::from_str("www.example.com").unwrap()); + headers.append("Connection", HeaderValue::from_str("close").unwrap()); + let response = client.execute(request).await.unwrap(); + let (parts, body) = response.into_parts(); + println!("{:?}", parts); + println!("{}", String::from_utf8(body).unwrap()); +} diff --git a/compio-http/src/client.rs b/compio-http/src/client.rs new file mode 100644 index 00000000..61d82cc5 --- /dev/null +++ b/compio-http/src/client.rs @@ -0,0 +1,34 @@ +use std::io; + +use compio_io::{AsyncWrite, AsyncWriteExt}; +use http::{uri::Scheme, Request, Response}; + +use crate::HttpStream; + +#[derive(Debug, Clone)] +pub struct Client {} + +impl Client { + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + Self {} + } + + pub async fn execute(&self, request: Request>) -> io::Result>> { + let uri = request.uri(); + let scheme = uri.scheme().unwrap_or(&Scheme::HTTP); + let host = uri + .host() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "must specify host"))?; + let port = uri.port_u16(); + let mut stream = HttpStream::new(scheme, host, port).await?; + + let (parts, body) = request.into_parts(); + stream.write_request_parts(&parts).await?; + stream.write_all(body).await.0?; + stream.flush().await?; + + let response = stream.read_response().await?; + Ok(response) + } +} diff --git a/compio-http/src/lib.rs b/compio-http/src/lib.rs new file mode 100644 index 00000000..afc21c08 --- /dev/null +++ b/compio-http/src/lib.rs @@ -0,0 +1,5 @@ +mod client; +pub use client::*; + +mod stream; +pub(crate) use stream::*; diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs new file mode 100644 index 00000000..a389b655 --- /dev/null +++ b/compio-http/src/stream.rs @@ -0,0 +1,175 @@ +use std::io; + +use compio_buf::{BufResult, IntoInner, IoBuf, IoBufMut, IoVectoredBuf, IoVectoredBufMut}; +use compio_io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use compio_net::TcpStream; +use compio_tls::{TlsConnector, TlsStream}; +use http::{ + header::CONTENT_LENGTH, request, uri::Scheme, HeaderName, HeaderValue, Response, StatusCode, + Version, +}; + +pub enum HttpStream { + Tcp(TcpStream), + Tls(TlsStream), +} + +impl HttpStream { + pub async fn new(scheme: &Scheme, host: &str, port: Option) -> io::Result { + match scheme.as_str() { + "http" => Ok(Self::Tcp( + TcpStream::connect((host, port.unwrap_or(80))).await?, + )), + "https" => { + let stream = TcpStream::connect((host, port.unwrap_or(443))).await?; + let connector = TlsConnector::from( + native_tls::TlsConnector::new() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?, + ); + Ok(Self::Tls(connector.connect(host, stream).await?)) + } + _ => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "unsupported scheme", + )), + } + } + + pub async fn write_request_parts(&mut self, parts: &request::Parts) -> io::Result<()> { + self.write_all(parts.method.to_string()).await.0?; + self.write_all(" ").await.0?; + self.write_all( + parts + .uri + .path_and_query() + .map(|s| s.to_string()) + .unwrap_or_default(), + ) + .await + .0?; + self.write_all(" ").await.0?; + self.write_all(format!("{:?}", parts.version)).await.0?; + self.write_all("\r\n").await.0?; + for (header_name, header_value) in &parts.headers { + self.write_all(header_name.to_string()).await.0?; + self.write_all(": ").await.0?; + self.write_all(header_value.as_bytes().to_vec()).await.0?; + self.write_all("\r\n").await.0?; + } + self.write_all("\r\n").await.0?; + Ok(()) + } + + pub async fn read_response(&mut self) -> io::Result>> { + let mut buffer = Vec::with_capacity(1024); + 'read_loop: loop { + let len = buffer.len(); + let BufResult(res, slice) = self.read(buffer.slice(len..)).await; + res?; + buffer = slice.into_inner(); + + let mut header_buffer = vec![httparse::EMPTY_HEADER; 16]; + 'parse_loop: loop { + let mut response = httparse::Response::new(&mut header_buffer); + match response.parse(&buffer) { + Ok(status) => match status { + httparse::Status::Complete(len) => { + let mut resp = Response::new(vec![]); + *resp.version_mut() = match response.version.unwrap_or(1) { + 0 => Version::HTTP_10, + 1 => Version::HTTP_11, + _ => Version::HTTP_09, + }; + *resp.status_mut() = StatusCode::from_u16(response.code.unwrap_or(200)) + .expect("server should return a valid status code"); + let headers = resp.headers_mut(); + for header in response.headers { + if !header.name.is_empty() { + let name = HeaderName::from_bytes(header.name.as_bytes()) + .expect("server should return a valid header name"); + let value = HeaderValue::from_bytes(header.value) + .expect("server should return a valid header value"); + headers.append(name, value); + } + } + let full_len = headers + .get(CONTENT_LENGTH) + .map(|v| { + std::str::from_utf8(v.as_bytes()) + .expect("content length should be valid utf8") + .parse::() + .expect("content length should be a integer") + }) + .expect("should contain content length"); + let mut buffer = buffer[len..].to_vec(); + let curr_len = buffer.len(); + if curr_len < full_len { + buffer.reserve(full_len - curr_len); + let BufResult(res, slice) = + self.read_exact(buffer.slice(curr_len..full_len)).await; + res?; + buffer = slice.into_inner(); + } + *resp.body_mut() = buffer; + return Ok(resp); + } + httparse::Status::Partial => continue 'read_loop, + }, + Err(e) => match e { + httparse::Error::TooManyHeaders => { + header_buffer.resize(16, httparse::EMPTY_HEADER); + continue 'parse_loop; + } + _ => return Err(io::Error::new(io::ErrorKind::InvalidData, e)), + }, + } + } + } + } +} + +impl AsyncRead for HttpStream { + async fn read(&mut self, buf: B) -> BufResult { + match self { + Self::Tcp(s) => s.read(buf).await, + Self::Tls(s) => s.read(buf).await, + } + } + + async fn read_vectored(&mut self, buf: V) -> BufResult { + match self { + Self::Tcp(s) => s.read_vectored(buf).await, + Self::Tls(s) => s.read_vectored(buf).await, + } + } +} + +impl AsyncWrite for HttpStream { + async fn write(&mut self, buf: T) -> BufResult { + match self { + Self::Tcp(s) => s.write(buf).await, + Self::Tls(s) => s.write(buf).await, + } + } + + async fn write_vectored(&mut self, buf: T) -> BufResult { + match self { + Self::Tcp(s) => s.write_vectored(buf).await, + Self::Tls(s) => s.write_vectored(buf).await, + } + } + + async fn flush(&mut self) -> io::Result<()> { + match self { + Self::Tcp(s) => s.flush().await, + Self::Tls(s) => s.flush().await, + } + } + + async fn shutdown(&mut self) -> io::Result<()> { + match self { + Self::Tcp(s) => s.shutdown().await, + Self::Tls(s) => s.shutdown().await, + } + } +} From 604e1df2b91dba889c0e03abde9605b854dcde92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 00:36:45 +0800 Subject: [PATCH 02/44] fix: reserve enough space to read --- compio-http/examples/client.rs | 1 - compio-http/src/stream.rs | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/compio-http/examples/client.rs b/compio-http/examples/client.rs index 86c26921..a82b00d0 100644 --- a/compio-http/examples/client.rs +++ b/compio-http/examples/client.rs @@ -10,7 +10,6 @@ async fn main() { *request.version_mut() = Version::HTTP_11; let headers = request.headers_mut(); headers.append("Host", HeaderValue::from_str("www.example.com").unwrap()); - headers.append("Connection", HeaderValue::from_str("close").unwrap()); let response = client.execute(request).await.unwrap(); let (parts, body) = response.into_parts(); println!("{:?}", parts); diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index a389b655..b6c50a78 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -113,11 +113,14 @@ impl HttpStream { *resp.body_mut() = buffer; return Ok(resp); } - httparse::Status::Partial => continue 'read_loop, + httparse::Status::Partial => { + buffer.reserve(1024); + continue 'read_loop; + } }, Err(e) => match e { httparse::Error::TooManyHeaders => { - header_buffer.resize(16, httparse::EMPTY_HEADER); + header_buffer.resize(header_buffer.len() + 16, httparse::EMPTY_HEADER); continue 'parse_loop; } _ => return Err(io::Error::new(io::ErrorKind::InvalidData, e)), From d8eabec04a0bf82c7b00dc41374759c64f3ce9ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 15:45:53 +0800 Subject: [PATCH 03/44] feat: use hyper instead of handwriting --- compio-http/Cargo.toml | 7 +- compio-http/examples/client.rs | 20 +-- compio-http/src/client.rs | 41 +++--- compio-http/src/connector.rs | 28 ++++ compio-http/src/executor.rs | 12 ++ compio-http/src/lib.rs | 6 + compio-http/src/stream.rs | 226 +++++++++++++++++---------------- 7 files changed, 195 insertions(+), 145 deletions(-) create mode 100644 compio-http/src/connector.rs create mode 100644 compio-http/src/executor.rs diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 58cda129..71d3a16a 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -12,16 +12,17 @@ repository = { workspace = true } [dependencies] compio-buf = { workspace = true } compio-io = { workspace = true } +compio-runtime = { workspace = true } compio-net = { workspace = true, features = ["runtime"] } compio-tls = { workspace = true } native-tls = { version = "0.2", optional = true } -http = "0.2" -httparse = "1" +hyper = { version = "0.14", features = ["client", "http1"] } +send_wrapper = { version = "0.6", features = ["futures"] } +tokio = { version = "1", default-features = false } [dev-dependencies] -compio-runtime = { workspace = true } compio-macros = { workspace = true } [features] diff --git a/compio-http/examples/client.rs b/compio-http/examples/client.rs index a82b00d0..44aad522 100644 --- a/compio-http/examples/client.rs +++ b/compio-http/examples/client.rs @@ -1,17 +1,17 @@ use compio_http::Client; -use http::{HeaderValue, Method, Request, Version}; +use hyper::{body::HttpBody, Method}; #[compio_macros::main] async fn main() { let client = Client::new(); - let mut request = Request::new(vec![]); - *request.method_mut() = Method::GET; - *request.uri_mut() = "https://www.example.com/".parse().unwrap(); - *request.version_mut() = Version::HTTP_11; - let headers = request.headers_mut(); - headers.append("Host", HeaderValue::from_str("www.example.com").unwrap()); - let response = client.execute(request).await.unwrap(); - let (parts, body) = response.into_parts(); + let response = client + .request(Method::GET, "https://www.example.com/".parse().unwrap()) + .await + .unwrap(); + let (parts, mut body) = response.into_parts(); println!("{:?}", parts); - println!("{}", String::from_utf8(body).unwrap()); + println!( + "{}", + std::str::from_utf8(&body.data().await.unwrap().unwrap()).unwrap() + ); } diff --git a/compio-http/src/client.rs b/compio-http/src/client.rs index 61d82cc5..3126e94f 100644 --- a/compio-http/src/client.rs +++ b/compio-http/src/client.rs @@ -1,34 +1,31 @@ -use std::io; +use hyper::{Body, Method, Request, Response, Uri}; -use compio_io::{AsyncWrite, AsyncWriteExt}; -use http::{uri::Scheme, Request, Response}; - -use crate::HttpStream; +use crate::{CompioExecutor, Connector}; #[derive(Debug, Clone)] -pub struct Client {} +pub struct Client { + client: hyper::Client, +} impl Client { #[allow(clippy::new_without_default)] pub fn new() -> Self { - Self {} + Self { + client: hyper::Client::builder() + .executor(CompioExecutor) + .set_host(true) + .build(Connector), + } } - pub async fn execute(&self, request: Request>) -> io::Result>> { - let uri = request.uri(); - let scheme = uri.scheme().unwrap_or(&Scheme::HTTP); - let host = uri - .host() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "must specify host"))?; - let port = uri.port_u16(); - let mut stream = HttpStream::new(scheme, host, port).await?; - - let (parts, body) = request.into_parts(); - stream.write_request_parts(&parts).await?; - stream.write_all(body).await.0?; - stream.flush().await?; + pub async fn execute(&self, request: Request) -> hyper::Result> { + self.client.request(request).await + } - let response = stream.read_response().await?; - Ok(response) + pub async fn request(&self, method: Method, uri: Uri) -> hyper::Result> { + let mut request = Request::new(Body::empty()); + *request.method_mut() = method; + *request.uri_mut() = uri; + self.execute(request).await } } diff --git a/compio-http/src/connector.rs b/compio-http/src/connector.rs new file mode 100644 index 00000000..aedf8c51 --- /dev/null +++ b/compio-http/src/connector.rs @@ -0,0 +1,28 @@ +use std::{ + future::Future, + io, + pin::Pin, + task::{Context, Poll}, +}; + +use hyper::{service::Service, Uri}; +use send_wrapper::SendWrapper; + +use crate::HttpStream; + +#[derive(Debug, Clone)] +pub struct Connector; + +impl Service for Connector { + type Error = io::Error; + type Future = Pin> + Send>>; + type Response = HttpStream; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, req: Uri) -> Self::Future { + Box::pin(SendWrapper::new(HttpStream::new(req))) + } +} diff --git a/compio-http/src/executor.rs b/compio-http/src/executor.rs new file mode 100644 index 00000000..a038be30 --- /dev/null +++ b/compio-http/src/executor.rs @@ -0,0 +1,12 @@ +use std::{future::Future, pin::Pin}; + +use hyper::rt::Executor; + +#[derive(Debug, Clone)] +pub struct CompioExecutor; + +impl Executor + Send>>> for CompioExecutor { + fn execute(&self, fut: Pin + Send>>) { + compio_runtime::spawn(fut).detach() + } +} diff --git a/compio-http/src/lib.rs b/compio-http/src/lib.rs index afc21c08..6e13fd49 100644 --- a/compio-http/src/lib.rs +++ b/compio-http/src/lib.rs @@ -3,3 +3,9 @@ pub use client::*; mod stream; pub(crate) use stream::*; + +mod connector; +pub(crate) use connector::*; + +mod executor; +pub(crate) use executor::*; diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index b6c50a78..766bf44f 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -1,25 +1,37 @@ -use std::io; +use std::{ + future::Future, + io, + pin::Pin, + task::{Context, Poll}, +}; -use compio_buf::{BufResult, IntoInner, IoBuf, IoBufMut, IoVectoredBuf, IoVectoredBufMut}; -use compio_io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use compio_buf::{BufResult, IoBuf, IoBufMut, IoVectoredBuf, IoVectoredBufMut}; +use compio_io::{AsyncRead, AsyncWrite}; use compio_net::TcpStream; use compio_tls::{TlsConnector, TlsStream}; -use http::{ - header::CONTENT_LENGTH, request, uri::Scheme, HeaderName, HeaderValue, Response, StatusCode, - Version, +use hyper::{ + client::connect::{Connected, Connection}, + Uri, }; +use send_wrapper::SendWrapper; -pub enum HttpStream { +enum HttpStreamInner { Tcp(TcpStream), Tls(TlsStream), } -impl HttpStream { - pub async fn new(scheme: &Scheme, host: &str, port: Option) -> io::Result { - match scheme.as_str() { - "http" => Ok(Self::Tcp( - TcpStream::connect((host, port.unwrap_or(80))).await?, - )), +impl HttpStreamInner { + pub async fn new(uri: Uri) -> io::Result { + let scheme = uri.scheme_str().unwrap_or("http"); + let host = uri + .host() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "should specify host"))?; + let port = uri.port_u16(); + match scheme { + "http" => { + let stream = TcpStream::connect((host, port.unwrap_or(80))).await?; + Ok(Self::Tcp(stream)) + } "https" => { let stream = TcpStream::connect((host, port.unwrap_or(443))).await?; let connector = TlsConnector::from( @@ -34,104 +46,9 @@ impl HttpStream { )), } } - - pub async fn write_request_parts(&mut self, parts: &request::Parts) -> io::Result<()> { - self.write_all(parts.method.to_string()).await.0?; - self.write_all(" ").await.0?; - self.write_all( - parts - .uri - .path_and_query() - .map(|s| s.to_string()) - .unwrap_or_default(), - ) - .await - .0?; - self.write_all(" ").await.0?; - self.write_all(format!("{:?}", parts.version)).await.0?; - self.write_all("\r\n").await.0?; - for (header_name, header_value) in &parts.headers { - self.write_all(header_name.to_string()).await.0?; - self.write_all(": ").await.0?; - self.write_all(header_value.as_bytes().to_vec()).await.0?; - self.write_all("\r\n").await.0?; - } - self.write_all("\r\n").await.0?; - Ok(()) - } - - pub async fn read_response(&mut self) -> io::Result>> { - let mut buffer = Vec::with_capacity(1024); - 'read_loop: loop { - let len = buffer.len(); - let BufResult(res, slice) = self.read(buffer.slice(len..)).await; - res?; - buffer = slice.into_inner(); - - let mut header_buffer = vec![httparse::EMPTY_HEADER; 16]; - 'parse_loop: loop { - let mut response = httparse::Response::new(&mut header_buffer); - match response.parse(&buffer) { - Ok(status) => match status { - httparse::Status::Complete(len) => { - let mut resp = Response::new(vec![]); - *resp.version_mut() = match response.version.unwrap_or(1) { - 0 => Version::HTTP_10, - 1 => Version::HTTP_11, - _ => Version::HTTP_09, - }; - *resp.status_mut() = StatusCode::from_u16(response.code.unwrap_or(200)) - .expect("server should return a valid status code"); - let headers = resp.headers_mut(); - for header in response.headers { - if !header.name.is_empty() { - let name = HeaderName::from_bytes(header.name.as_bytes()) - .expect("server should return a valid header name"); - let value = HeaderValue::from_bytes(header.value) - .expect("server should return a valid header value"); - headers.append(name, value); - } - } - let full_len = headers - .get(CONTENT_LENGTH) - .map(|v| { - std::str::from_utf8(v.as_bytes()) - .expect("content length should be valid utf8") - .parse::() - .expect("content length should be a integer") - }) - .expect("should contain content length"); - let mut buffer = buffer[len..].to_vec(); - let curr_len = buffer.len(); - if curr_len < full_len { - buffer.reserve(full_len - curr_len); - let BufResult(res, slice) = - self.read_exact(buffer.slice(curr_len..full_len)).await; - res?; - buffer = slice.into_inner(); - } - *resp.body_mut() = buffer; - return Ok(resp); - } - httparse::Status::Partial => { - buffer.reserve(1024); - continue 'read_loop; - } - }, - Err(e) => match e { - httparse::Error::TooManyHeaders => { - header_buffer.resize(header_buffer.len() + 16, httparse::EMPTY_HEADER); - continue 'parse_loop; - } - _ => return Err(io::Error::new(io::ErrorKind::InvalidData, e)), - }, - } - } - } - } } -impl AsyncRead for HttpStream { +impl AsyncRead for HttpStreamInner { async fn read(&mut self, buf: B) -> BufResult { match self { Self::Tcp(s) => s.read(buf).await, @@ -147,7 +64,7 @@ impl AsyncRead for HttpStream { } } -impl AsyncWrite for HttpStream { +impl AsyncWrite for HttpStreamInner { async fn write(&mut self, buf: T) -> BufResult { match self { Self::Tcp(s) => s.write(buf).await, @@ -176,3 +93,92 @@ impl AsyncWrite for HttpStream { } } } + +type PinBoxFuture = Pin>>; + +pub struct HttpStream { + inner: HttpStreamInner, + read_future: Option>>>, + write_future: Option>>>, + flush_future: Option>>, + shutdown_future: Option>>, +} + +impl HttpStream { + pub async fn new(uri: Uri) -> io::Result { + Ok(Self { + inner: HttpStreamInner::new(uri).await?, + read_future: None, + write_future: None, + flush_future: None, + shutdown_future: None, + }) + } +} + +macro_rules! poll_future { + ($f:expr, $cx:expr, $e:expr) => {{ + let mut future = match $f.take() { + Some(f) => f, + None => Box::pin(SendWrapper::new($e)), + }; + let f = future.as_mut(); + match f.poll($cx) { + Poll::Pending => { + $f = Some(future); + return Poll::Pending; + } + Poll::Ready(res) => res, + } + }}; +} + +impl tokio::io::AsyncRead for HttpStream { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + let inner: &'static mut HttpStreamInner = unsafe { &mut *(&mut self.inner as *mut _) }; + let BufResult(res, inner_buf) = poll_future!( + self.read_future, + cx, + inner.read(Vec::with_capacity(unsafe { buf.unfilled_mut() }.len())) + ); + res?; + buf.put_slice(&inner_buf); + Poll::Ready(Ok(())) + } +} + +impl tokio::io::AsyncWrite for HttpStream { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let inner: &'static mut HttpStreamInner = unsafe { &mut *(&mut self.inner as *mut _) }; + let BufResult(res, _) = poll_future!(self.write_future, cx, inner.write(buf.to_vec())); + Poll::Ready(res) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let inner: &'static mut HttpStreamInner = unsafe { &mut *(&mut self.inner as *mut _) }; + let res = poll_future!(self.flush_future, cx, inner.flush()); + Poll::Ready(res) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let inner: &'static mut HttpStreamInner = unsafe { &mut *(&mut self.inner as *mut _) }; + let res = poll_future!(self.shutdown_future, cx, inner.shutdown()); + Poll::Ready(res) + } +} + +impl Connection for HttpStream { + fn connected(&self) -> Connected { + Connected::new() + } +} + +unsafe impl Send for HttpStream {} From 68eef0023856712b77e13dcbf2231840d65df725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 15:49:38 +0800 Subject: [PATCH 04/44] fix: use hyper body to_bytes --- compio-http/examples/client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compio-http/examples/client.rs b/compio-http/examples/client.rs index 44aad522..4dbc1fb1 100644 --- a/compio-http/examples/client.rs +++ b/compio-http/examples/client.rs @@ -1,5 +1,5 @@ use compio_http::Client; -use hyper::{body::HttpBody, Method}; +use hyper::Method; #[compio_macros::main] async fn main() { @@ -8,10 +8,10 @@ async fn main() { .request(Method::GET, "https://www.example.com/".parse().unwrap()) .await .unwrap(); - let (parts, mut body) = response.into_parts(); + let (parts, body) = response.into_parts(); println!("{:?}", parts); println!( "{}", - std::str::from_utf8(&body.data().await.unwrap().unwrap()).unwrap() + std::str::from_utf8(&hyper::body::to_bytes(body).await.unwrap()).unwrap() ); } From 1db7b7e7839bdb170db3fe65cc178392755d5f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 17:32:56 +0800 Subject: [PATCH 05/44] fix: use SendWrapper on stream inner --- compio-http/src/stream.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index 766bf44f..b51fa7e6 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -1,6 +1,7 @@ use std::{ future::Future, io, + ops::DerefMut, pin::Pin, task::{Context, Poll}, }; @@ -94,10 +95,10 @@ impl AsyncWrite for HttpStreamInner { } } -type PinBoxFuture = Pin>>; +type PinBoxFuture = Pin + Send>>; pub struct HttpStream { - inner: HttpStreamInner, + inner: SendWrapper, read_future: Option>>>, write_future: Option>>>, flush_future: Option>>, @@ -107,7 +108,7 @@ pub struct HttpStream { impl HttpStream { pub async fn new(uri: Uri) -> io::Result { Ok(Self { - inner: HttpStreamInner::new(uri).await?, + inner: SendWrapper::new(HttpStreamInner::new(uri).await?), read_future: None, write_future: None, flush_future: None, @@ -139,7 +140,8 @@ impl tokio::io::AsyncRead for HttpStream { cx: &mut Context<'_>, buf: &mut tokio::io::ReadBuf<'_>, ) -> Poll> { - let inner: &'static mut HttpStreamInner = unsafe { &mut *(&mut self.inner as *mut _) }; + let inner: &'static mut HttpStreamInner = + unsafe { &mut *(self.inner.deref_mut() as *mut _) }; let BufResult(res, inner_buf) = poll_future!( self.read_future, cx, @@ -157,19 +159,22 @@ impl tokio::io::AsyncWrite for HttpStream { cx: &mut Context<'_>, buf: &[u8], ) -> Poll> { - let inner: &'static mut HttpStreamInner = unsafe { &mut *(&mut self.inner as *mut _) }; + let inner: &'static mut HttpStreamInner = + unsafe { &mut *(self.inner.deref_mut() as *mut _) }; let BufResult(res, _) = poll_future!(self.write_future, cx, inner.write(buf.to_vec())); Poll::Ready(res) } fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let inner: &'static mut HttpStreamInner = unsafe { &mut *(&mut self.inner as *mut _) }; + let inner: &'static mut HttpStreamInner = + unsafe { &mut *(self.inner.deref_mut() as *mut _) }; let res = poll_future!(self.flush_future, cx, inner.flush()); Poll::Ready(res) } fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let inner: &'static mut HttpStreamInner = unsafe { &mut *(&mut self.inner as *mut _) }; + let inner: &'static mut HttpStreamInner = + unsafe { &mut *(self.inner.deref_mut() as *mut _) }; let res = poll_future!(self.shutdown_future, cx, inner.shutdown()); Poll::Ready(res) } @@ -180,5 +185,3 @@ impl Connection for HttpStream { Connected::new() } } - -unsafe impl Send for HttpStream {} From faf4ed86cf2ec5edff9edd8d35d0ffaf351ede5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 19:38:32 +0800 Subject: [PATCH 06/44] fix: use buffer instead of tmp alloc --- compio-http/src/stream.rs | 117 +++++++++++++++++++++++++++++++------- compio-tls/src/wrapper.rs | 1 + 2 files changed, 99 insertions(+), 19 deletions(-) diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index b51fa7e6..26989f3f 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -6,8 +6,10 @@ use std::{ task::{Context, Poll}, }; -use compio_buf::{BufResult, IoBuf, IoBufMut, IoVectoredBuf, IoVectoredBufMut}; -use compio_io::{AsyncRead, AsyncWrite}; +use compio_buf::{ + BufResult, IntoInner, IoBuf, IoBufMut, IoVectoredBuf, IoVectoredBufMut, SetBufInit, +}; +use compio_io::{AsyncRead, AsyncWrite, AsyncWriteExt, Buffer}; use compio_net::TcpStream; use compio_tls::{TlsConnector, TlsStream}; use hyper::{ @@ -95,12 +97,90 @@ impl AsyncWrite for HttpStreamInner { } } +const DEFAULT_BUF_SIZE: usize = 8 * 1024; + +struct HttpStreamBufInner { + inner: HttpStreamInner, + read_buffer: Buffer, + write_buffer: Buffer, +} + +impl HttpStreamBufInner { + pub async fn new(uri: Uri) -> io::Result { + Ok(Self { + inner: HttpStreamInner::new(uri).await?, + read_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), + write_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), + }) + } + + pub async fn fill_read_buf(&mut self) -> io::Result<()> { + if self.read_buffer.all_done() { + self.read_buffer.reset(); + } + if self.read_buffer.slice().is_empty() { + self.read_buffer + .with(|b| async { + let len = b.buf_len(); + let slice = b.slice(len..); + self.inner.read(slice).await.into_inner() + }) + .await?; + } + + Ok(()) + } + + pub fn read_buf_slice(&self) -> &[u8] { + self.read_buffer.slice() + } + + pub fn consume_read_buf(&mut self, amt: usize) { + self.read_buffer.advance(amt); + } + + pub async fn flush_write_buf_if_needed(&mut self) -> io::Result<()> { + if self.write_buffer.need_flush() { + self.flush_write_buf().await?; + } + Ok(()) + } + + pub fn write_slice(&mut self, buf: &[u8]) -> io::Result { + self.write_buffer.with_sync(|mut inner| { + let len = buf.len().min(inner.buf_capacity() - inner.buf_len()); + unsafe { + std::ptr::copy_nonoverlapping( + buf.as_ptr(), + inner.as_buf_mut_ptr().add(inner.buf_len()), + len, + ); + inner.set_buf_init(inner.buf_len() + len); + } + BufResult(Ok(len), inner) + }) + } + + pub async fn flush_write_buf(&mut self) -> io::Result<()> { + if !self.write_buffer.is_empty() { + self.write_buffer.with(|b| self.inner.write_all(b)).await?; + self.write_buffer.reset(); + } + self.inner.flush().await?; + Ok(()) + } + + pub async fn shutdown(&mut self) -> io::Result<()> { + self.inner.shutdown().await + } +} + type PinBoxFuture = Pin + Send>>; pub struct HttpStream { - inner: SendWrapper, - read_future: Option>>>, - write_future: Option>>>, + inner: SendWrapper, + read_future: Option>>, + write_future: Option>>, flush_future: Option>>, shutdown_future: Option>>, } @@ -108,7 +188,7 @@ pub struct HttpStream { impl HttpStream { pub async fn new(uri: Uri) -> io::Result { Ok(Self { - inner: SendWrapper::new(HttpStreamInner::new(uri).await?), + inner: SendWrapper::new(HttpStreamBufInner::new(uri).await?), read_future: None, write_future: None, flush_future: None, @@ -140,15 +220,13 @@ impl tokio::io::AsyncRead for HttpStream { cx: &mut Context<'_>, buf: &mut tokio::io::ReadBuf<'_>, ) -> Poll> { - let inner: &'static mut HttpStreamInner = + let inner: &'static mut HttpStreamBufInner = unsafe { &mut *(self.inner.deref_mut() as *mut _) }; - let BufResult(res, inner_buf) = poll_future!( - self.read_future, - cx, - inner.read(Vec::with_capacity(unsafe { buf.unfilled_mut() }.len())) - ); - res?; - buf.put_slice(&inner_buf); + poll_future!(self.read_future, cx, inner.fill_read_buf())?; + let slice = self.inner.read_buf_slice(); + let len = slice.len().min(buf.remaining()); + buf.put_slice(&slice[..len]); + self.inner.consume_read_buf(len); Poll::Ready(Ok(())) } } @@ -159,21 +237,22 @@ impl tokio::io::AsyncWrite for HttpStream { cx: &mut Context<'_>, buf: &[u8], ) -> Poll> { - let inner: &'static mut HttpStreamInner = + let inner: &'static mut HttpStreamBufInner = unsafe { &mut *(self.inner.deref_mut() as *mut _) }; - let BufResult(res, _) = poll_future!(self.write_future, cx, inner.write(buf.to_vec())); + poll_future!(self.write_future, cx, inner.flush_write_buf_if_needed())?; + let res = self.inner.write_slice(buf); Poll::Ready(res) } fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let inner: &'static mut HttpStreamInner = + let inner: &'static mut HttpStreamBufInner = unsafe { &mut *(self.inner.deref_mut() as *mut _) }; - let res = poll_future!(self.flush_future, cx, inner.flush()); + let res = poll_future!(self.flush_future, cx, inner.flush_write_buf()); Poll::Ready(res) } fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let inner: &'static mut HttpStreamInner = + let inner: &'static mut HttpStreamBufInner = unsafe { &mut *(self.inner.deref_mut() as *mut _) }; let res = poll_future!(self.shutdown_future, cx, inner.shutdown()); Poll::Ready(res) diff --git a/compio-tls/src/wrapper.rs b/compio-tls/src/wrapper.rs index 0e9e2c3f..0f6d59ad 100644 --- a/compio-tls/src/wrapper.rs +++ b/compio-tls/src/wrapper.rs @@ -129,6 +129,7 @@ impl StreamWrapper { let stream = &mut self.stream; let len = self.write_buffer.with(|b| stream.write_all(b)).await?; self.write_buffer.reset(); + stream.flush().await?; Ok(len) } } From 038816de4c4f4728f7239706ca7f79614582243c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 21:27:05 +0800 Subject: [PATCH 07/44] feat: add request builder --- compio-http/Cargo.toml | 9 + compio-http/src/client.rs | 54 ++- compio-http/src/executor.rs | 12 - compio-http/src/lib.rs | 36 +- compio-http/src/request.rs | 400 +++++++++++++++++++ compio-http/src/{connector.rs => service.rs} | 11 +- compio-http/src/util.rs | 57 +++ 7 files changed, 549 insertions(+), 30 deletions(-) delete mode 100644 compio-http/src/executor.rs create mode 100644 compio-http/src/request.rs rename compio-http/src/{connector.rs => service.rs} (66%) create mode 100644 compio-http/src/util.rs diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 71d3a16a..6d92ccd6 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -18,10 +18,18 @@ compio-tls = { workspace = true } native-tls = { version = "0.2", optional = true } +http = "0.2" hyper = { version = "0.14", features = ["client", "http1"] } send_wrapper = { version = "0.6", features = ["futures"] } tokio = { version = "1", default-features = false } +base64 = "0.21" +serde = "1" +serde_json = { version = "1", optional = true } +serde_urlencoded = "0.7" +thiserror = "1" +url = "2" + [dev-dependencies] compio-macros = { workspace = true } @@ -29,3 +37,4 @@ compio-macros = { workspace = true } default = ["native-tls"] native-tls = ["compio-tls/native-tls", "dep:native-tls"] vendored = ["native-tls?/vendored"] +json = ["dep:serde_json"] diff --git a/compio-http/src/client.rs b/compio-http/src/client.rs index 3126e94f..888938dd 100644 --- a/compio-http/src/client.rs +++ b/compio-http/src/client.rs @@ -1,31 +1,59 @@ -use hyper::{Body, Method, Request, Response, Uri}; +use std::rc::Rc; -use crate::{CompioExecutor, Connector}; +use hyper::{Body, Method, Response, Uri}; +use url::Url; +use crate::{CompioExecutor, Connector, Request, Result}; + +/// An asynchronous `Client` to make Requests with. #[derive(Debug, Clone)] pub struct Client { - client: hyper::Client, + client: Rc, } impl Client { #[allow(clippy::new_without_default)] pub fn new() -> Self { Self { - client: hyper::Client::builder() - .executor(CompioExecutor) - .set_host(true) - .build(Connector), + client: Rc::new(ClientRef { + client: hyper::Client::builder() + .executor(CompioExecutor) + .set_host(true) + .build(Connector), + }), } } - pub async fn execute(&self, request: Request) -> hyper::Result> { - self.client.request(request).await + pub async fn execute(&self, request: Request) -> Result> { + let (method, url, headers, body, timeout, version) = request.pieces(); + let mut request = hyper::Request::builder() + .method(method) + .uri( + url.as_str() + .parse::() + .expect("a parsed Url should always be a valid Uri"), + ) + .version(version) + .body(body.unwrap_or_else(Body::empty))?; + *request.headers_mut() = headers; + let future = self.client.client.request(request); + let res = if let Some(timeout) = timeout { + compio_runtime::time::timeout(timeout, future) + .await + .map_err(|_| crate::Error::Timeout)?? + } else { + future.await? + }; + Ok(res) } - pub async fn request(&self, method: Method, uri: Uri) -> hyper::Result> { - let mut request = Request::new(Body::empty()); - *request.method_mut() = method; - *request.uri_mut() = uri; + pub async fn request(&self, method: Method, url: Url) -> Result> { + let request = Request::new(method, url); self.execute(request).await } } + +#[derive(Debug)] +struct ClientRef { + client: hyper::Client, +} diff --git a/compio-http/src/executor.rs b/compio-http/src/executor.rs deleted file mode 100644 index a038be30..00000000 --- a/compio-http/src/executor.rs +++ /dev/null @@ -1,12 +0,0 @@ -use std::{future::Future, pin::Pin}; - -use hyper::rt::Executor; - -#[derive(Debug, Clone)] -pub struct CompioExecutor; - -impl Executor + Send>>> for CompioExecutor { - fn execute(&self, fut: Pin + Send>>) { - compio_runtime::spawn(fut).detach() - } -} diff --git a/compio-http/src/lib.rs b/compio-http/src/lib.rs index 6e13fd49..bd64776c 100644 --- a/compio-http/src/lib.rs +++ b/compio-http/src/lib.rs @@ -1,11 +1,39 @@ +//! A high level HTTP client library based on compio. + +#![warn(missing_docs)] + mod client; pub use client::*; +mod request; +pub use request::*; + mod stream; pub(crate) use stream::*; -mod connector; -pub(crate) use connector::*; +mod service; +pub(crate) use service::*; + +mod util; + +use thiserror::Error; + +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum Error { + #[error("request timeout")] + Timeout, + #[error("system error: {0}")] + System(#[from] std::io::Error), + #[error("`http` error: {0}")] + Http(#[from] http::Error), + #[error("`hyper` error: {0}")] + Hyper(#[from] hyper::Error), + #[error("url encode error: {0}")] + UrlEncoded(#[from] serde_urlencoded::ser::Error), + #[cfg(feature = "json")] + #[error("json error: {0}")] + Json(#[from] serde_json::Error), +} -mod executor; -pub(crate) use executor::*; +pub type Result = std::result::Result; diff --git a/compio-http/src/request.rs b/compio-http/src/request.rs new file mode 100644 index 00000000..16b2dfc7 --- /dev/null +++ b/compio-http/src/request.rs @@ -0,0 +1,400 @@ +use std::{fmt::Display, time::Duration}; + +use hyper::{ + header::{HeaderName, HeaderValue, AUTHORIZATION, CONTENT_TYPE}, + Body, HeaderMap, Method, Response, Version, +}; +use serde::Serialize; +use url::Url; + +use crate::{Client, Result}; + +/// A request which can be executed with `Client::execute()`. +pub struct Request { + method: Method, + url: Url, + headers: HeaderMap, + body: Option, + timeout: Option, + version: Version, +} + +impl Request { + /// Constructs a new request. + #[inline] + pub fn new(method: Method, url: Url) -> Self { + Request { + method, + url, + headers: HeaderMap::new(), + body: None, + timeout: None, + version: Version::default(), + } + } + + /// Get the method. + #[inline] + pub fn method(&self) -> &Method { + &self.method + } + + /// Get a mutable reference to the method. + #[inline] + pub fn method_mut(&mut self) -> &mut Method { + &mut self.method + } + + /// Get the url. + #[inline] + pub fn url(&self) -> &Url { + &self.url + } + + /// Get a mutable reference to the url. + #[inline] + pub fn url_mut(&mut self) -> &mut Url { + &mut self.url + } + + /// Get the headers. + #[inline] + pub fn headers(&self) -> &HeaderMap { + &self.headers + } + + /// Get a mutable reference to the headers. + #[inline] + pub fn headers_mut(&mut self) -> &mut HeaderMap { + &mut self.headers + } + + /// Get the body. + #[inline] + pub fn body(&self) -> Option<&Body> { + self.body.as_ref() + } + + /// Get a mutable reference to the body. + #[inline] + pub fn body_mut(&mut self) -> &mut Option { + &mut self.body + } + + /// Get the timeout. + #[inline] + pub fn timeout(&self) -> Option<&Duration> { + self.timeout.as_ref() + } + + /// Get a mutable reference to the timeout. + #[inline] + pub fn timeout_mut(&mut self) -> &mut Option { + &mut self.timeout + } + + /// Get the http version. + #[inline] + pub fn version(&self) -> Version { + self.version + } + + /// Get a mutable reference to the http version. + #[inline] + pub fn version_mut(&mut self) -> &mut Version { + &mut self.version + } + + pub(super) fn pieces( + self, + ) -> ( + Method, + Url, + HeaderMap, + Option, + Option, + Version, + ) { + ( + self.method, + self.url, + self.headers, + self.body, + self.timeout, + self.version, + ) + } +} + +/// A builder to construct the properties of a `Request`. +pub struct RequestBuilder { + client: Client, + request: Result, +} + +impl RequestBuilder { + /// Assemble a builder starting from an existing `Client` and a `Request`. + pub fn from_parts(client: Client, request: Request) -> RequestBuilder { + RequestBuilder { + client, + request: Ok(request), + } + } + + /// Add a `Header` to this Request. + pub fn header, V: TryInto>( + self, + key: K, + value: V, + ) -> RequestBuilder + where + K::Error: Into, + V::Error: Into, + { + self.header_sensitive(key, value, false) + } + + /// Add a `Header` to this Request with ability to define if `header_value` + /// is sensitive. + fn header_sensitive, V: TryInto>( + mut self, + key: K, + value: V, + sensitive: bool, + ) -> RequestBuilder + where + K::Error: Into, + V::Error: Into, + { + let mut error = None; + if let Ok(ref mut req) = self.request { + match key.try_into() { + Ok(key) => match value.try_into() { + Ok(mut value) => { + // We want to potentially make an unsensitive header + // to be sensitive, not the reverse. So, don't turn off + // a previously sensitive header. + if sensitive { + value.set_sensitive(true); + } + req.headers_mut().append(key, value); + } + Err(e) => error = Some(e.into()), + }, + Err(e) => error = Some(e.into()), + }; + } + if let Some(err) = error { + self.request = Err(err.into()); + } + self + } + + /// Add a set of Headers to the existing ones on this Request. + /// + /// The headers will be merged in to any already set. + pub fn headers(mut self, headers: HeaderMap) -> RequestBuilder { + if let Ok(ref mut req) = self.request { + crate::util::replace_headers(req.headers_mut(), headers); + } + self + } + + /// Enable HTTP basic authentication. + /// + /// ```rust + /// # use reqwest::Error; + /// + /// # async fn run() -> Result<(), Error> { + /// let client = reqwest::Client::new(); + /// let resp = client + /// .delete("http://httpbin.org/delete") + /// .basic_auth("admin", Some("good password")) + /// .send() + /// .await?; + /// # Ok(()) + /// # } + /// ``` + pub fn basic_auth( + self, + username: U, + password: Option

, + ) -> RequestBuilder { + let header_value = crate::util::basic_auth(username, password); + self.header_sensitive(AUTHORIZATION, header_value, true) + } + + /// Enable HTTP bearer authentication. + pub fn bearer_auth(self, token: T) -> RequestBuilder { + let header_value = format!("Bearer {}", token); + self.header_sensitive(AUTHORIZATION, header_value, true) + } + + /// Set the request body. + pub fn body>(mut self, body: T) -> RequestBuilder { + if let Ok(ref mut req) = self.request { + *req.body_mut() = Some(body.into()); + } + self + } + + /// Enables a request timeout. + /// + /// The timeout is applied from when the request starts connecting until the + /// response body has finished. It affects only this request and overrides + /// the timeout configured using `ClientBuilder::timeout()`. + pub fn timeout(mut self, timeout: Duration) -> RequestBuilder { + if let Ok(ref mut req) = self.request { + *req.timeout_mut() = Some(timeout); + } + self + } + + /// Modify the query string of the URL. + /// + /// Modifies the URL of this request, adding the parameters provided. + /// This method appends and does not overwrite. This means that it can + /// be called multiple times and that existing query parameters are not + /// overwritten if the same key is used. The key will simply show up + /// twice in the query string. + /// Calling `.query(&[("foo", "a"), ("foo", "b")])` gives `"foo=a&foo=b"`. + /// + /// # Note + /// This method does not support serializing a single key-value + /// pair. Instead of using `.query(("key", "val"))`, use a sequence, such + /// as `.query(&[("key", "val")])`. It's also possible to serialize structs + /// and maps into a key-value pair. + /// + /// # Errors + /// This method will fail if the object you provide cannot be serialized + /// into a query string. + pub fn query(mut self, query: &T) -> RequestBuilder { + let mut error = None; + if let Ok(ref mut req) = self.request { + let url = req.url_mut(); + let mut pairs = url.query_pairs_mut(); + let serializer = serde_urlencoded::Serializer::new(&mut pairs); + + if let Err(err) = query.serialize(serializer) { + error = Some(err.into()); + } + } + if let Ok(ref mut req) = self.request { + if let Some("") = req.url().query() { + req.url_mut().set_query(None); + } + } + if let Some(err) = error { + self.request = Err(err); + } + self + } + + /// Set HTTP version + pub fn version(mut self, version: Version) -> RequestBuilder { + if let Ok(ref mut req) = self.request { + req.version = version; + } + self + } + + /// Send a form body. + /// + /// Sets the body to the url encoded serialization of the passed value, + /// and also sets the `Content-Type: application/x-www-form-urlencoded` + /// header. + /// + /// ```rust + /// # use reqwest::Error; + /// # use std::collections::HashMap; + /// # + /// # async fn run() -> Result<(), Error> { + /// let mut params = HashMap::new(); + /// params.insert("lang", "rust"); + /// + /// let client = reqwest::Client::new(); + /// let res = client + /// .post("http://httpbin.org") + /// .form(¶ms) + /// .send() + /// .await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// # Errors + /// + /// This method fails if the passed value cannot be serialized into + /// url encoded format + pub fn form(mut self, form: &T) -> RequestBuilder { + let mut error = None; + if let Ok(ref mut req) = self.request { + match serde_urlencoded::to_string(form) { + Ok(body) => { + req.headers_mut().insert( + CONTENT_TYPE, + HeaderValue::from_static("application/x-www-form-urlencoded"), + ); + *req.body_mut() = Some(body.into()); + } + Err(err) => error = Some(err.into()), + } + } + if let Some(err) = error { + self.request = Err(err); + } + self + } + + /// Send a JSON body. + /// + /// # Optional + /// + /// This requires the optional `json` feature enabled. + /// + /// # Errors + /// + /// Serialization can fail if `T`'s implementation of `Serialize` decides to + /// fail, or if `T` contains a map with non-string keys. + #[cfg(feature = "json")] + pub fn json(mut self, json: &T) -> RequestBuilder { + let mut error = None; + if let Ok(ref mut req) = self.request { + match serde_json::to_vec(json) { + Ok(body) => { + if !req.headers().contains_key(CONTENT_TYPE) { + req.headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + } + *req.body_mut() = Some(body.into()); + } + Err(err) => error = Some(err.into()), + } + } + if let Some(err) = error { + self.request = Err(err); + } + self + } + + /// Build a `Request`, which can be inspected, modified and executed with + /// `Client::execute()`. + pub fn build(self) -> Result { + self.request + } + + /// Build a `Request`, which can be inspected, modified and executed with + /// `Client::execute()`. + /// + /// This is similar to [`RequestBuilder::build()`], but also returns the + /// embedded `Client`. + pub fn build_split(self) -> (Client, Result) { + (self.client, self.request) + } + + /// Constructs the Request and sends it to the target URL, returning a + /// future Response. + pub async fn send(self) -> Result> { + self.client.execute(self.request?).await + } +} diff --git a/compio-http/src/connector.rs b/compio-http/src/service.rs similarity index 66% rename from compio-http/src/connector.rs rename to compio-http/src/service.rs index aedf8c51..9cbce9df 100644 --- a/compio-http/src/connector.rs +++ b/compio-http/src/service.rs @@ -5,11 +5,20 @@ use std::{ task::{Context, Poll}, }; -use hyper::{service::Service, Uri}; +use hyper::{rt::Executor, service::Service, Uri}; use send_wrapper::SendWrapper; use crate::HttpStream; +#[derive(Debug, Clone)] +pub struct CompioExecutor; + +impl Executor + Send>>> for CompioExecutor { + fn execute(&self, fut: Pin + Send>>) { + compio_runtime::spawn(fut).detach() + } +} + #[derive(Debug, Clone)] pub struct Connector; diff --git a/compio-http/src/util.rs b/compio-http/src/util.rs new file mode 100644 index 00000000..a14a57d6 --- /dev/null +++ b/compio-http/src/util.rs @@ -0,0 +1,57 @@ +//! Code from reqwest. + +use hyper::{ + header::{Entry, HeaderValue, OccupiedEntry}, + HeaderMap, +}; + +pub fn basic_auth(username: U, password: Option

) -> HeaderValue +where + U: std::fmt::Display, + P: std::fmt::Display, +{ + use std::io::Write; + + use base64::{prelude::BASE64_STANDARD, write::EncoderWriter}; + + let mut buf = b"Basic ".to_vec(); + { + let mut encoder = EncoderWriter::new(&mut buf, &BASE64_STANDARD); + let _ = write!(encoder, "{}:", username); + if let Some(password) = password { + let _ = write!(encoder, "{}", password); + } + } + let mut header = HeaderValue::from_bytes(&buf).expect("base64 is always valid HeaderValue"); + header.set_sensitive(true); + header +} + +pub(crate) fn replace_headers(dst: &mut HeaderMap, src: HeaderMap) { + // IntoIter of HeaderMap yields (Option, HeaderValue). + // The first time a name is yielded, it will be Some(name), and if + // there are more values with the same name, the next yield will be + // None. + + let mut prev_entry: Option> = None; + for (key, value) in src { + match key { + Some(key) => match dst.entry(key) { + Entry::Occupied(mut e) => { + e.insert(value); + prev_entry = Some(e); + } + Entry::Vacant(e) => { + let e = e.insert_entry(value); + prev_entry = Some(e); + } + }, + None => match prev_entry { + Some(ref mut entry) => { + entry.append(value); + } + None => unreachable!("HeaderMap::into_iter yielded None first"), + }, + } + } +} From d7d93974d4e1525f85bae8008cd510faaf4607ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 21:43:38 +0800 Subject: [PATCH 08/44] feat: add response wrapper --- compio-http/Cargo.toml | 4 +- compio-http/examples/client.rs | 8 +- compio-http/src/client.rs | 16 +-- compio-http/src/lib.rs | 11 ++ compio-http/src/request.rs | 12 +- compio-http/src/response.rs | 210 +++++++++++++++++++++++++++++++++ 6 files changed, 240 insertions(+), 21 deletions(-) create mode 100644 compio-http/src/response.rs diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 6d92ccd6..4efffbf1 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -10,7 +10,7 @@ license = { workspace = true } repository = { workspace = true } [dependencies] -compio-buf = { workspace = true } +compio-buf = { workspace = true, features = ["bytes"] } compio-io = { workspace = true } compio-runtime = { workspace = true } compio-net = { workspace = true, features = ["runtime"] } @@ -24,6 +24,8 @@ send_wrapper = { version = "0.6", features = ["futures"] } tokio = { version = "1", default-features = false } base64 = "0.21" +encoding_rs = "0.8" +mime = "0.3" serde = "1" serde_json = { version = "1", optional = true } serde_urlencoded = "0.7" diff --git a/compio-http/examples/client.rs b/compio-http/examples/client.rs index 4dbc1fb1..b0408534 100644 --- a/compio-http/examples/client.rs +++ b/compio-http/examples/client.rs @@ -6,12 +6,8 @@ async fn main() { let client = Client::new(); let response = client .request(Method::GET, "https://www.example.com/".parse().unwrap()) + .send() .await .unwrap(); - let (parts, body) = response.into_parts(); - println!("{:?}", parts); - println!( - "{}", - std::str::from_utf8(&hyper::body::to_bytes(body).await.unwrap()).unwrap() - ); + println!("{}", response.text().await.unwrap()); } diff --git a/compio-http/src/client.rs b/compio-http/src/client.rs index 888938dd..fe879d50 100644 --- a/compio-http/src/client.rs +++ b/compio-http/src/client.rs @@ -1,9 +1,9 @@ use std::rc::Rc; -use hyper::{Body, Method, Response, Uri}; +use hyper::{Body, Method, Uri}; use url::Url; -use crate::{CompioExecutor, Connector, Request, Result}; +use crate::{CompioExecutor, Connector, Request, RequestBuilder, Response, Result}; /// An asynchronous `Client` to make Requests with. #[derive(Debug, Clone)] @@ -12,6 +12,7 @@ pub struct Client { } impl Client { + /// Create a client with default config. #[allow(clippy::new_without_default)] pub fn new() -> Self { Self { @@ -24,7 +25,8 @@ impl Client { } } - pub async fn execute(&self, request: Request) -> Result> { + /// Send a request and wait for a response. + pub async fn execute(&self, request: Request) -> Result { let (method, url, headers, body, timeout, version) = request.pieces(); let mut request = hyper::Request::builder() .method(method) @@ -44,12 +46,12 @@ impl Client { } else { future.await? }; - Ok(res) + Ok(Response::new(res, url)) } - pub async fn request(&self, method: Method, url: Url) -> Result> { - let request = Request::new(method, url); - self.execute(request).await + /// Send a request with method and url. + pub fn request(&self, method: Method, url: Url) -> RequestBuilder { + RequestBuilder::from_parts(self.clone(), Request::new(method, url)) } } diff --git a/compio-http/src/lib.rs b/compio-http/src/lib.rs index bd64776c..b2a1ee1c 100644 --- a/compio-http/src/lib.rs +++ b/compio-http/src/lib.rs @@ -8,6 +8,9 @@ pub use client::*; mod request; pub use request::*; +mod response; +pub use response::*; + mod stream; pub(crate) use stream::*; @@ -18,22 +21,30 @@ mod util; use thiserror::Error; +/// The error type used in `compio-http`. #[derive(Debug, Error)] #[non_exhaustive] pub enum Error { + /// The request is timeout. #[error("request timeout")] Timeout, + /// An IO error occurs. #[error("system error: {0}")] System(#[from] std::io::Error), + /// An HTTP related parse error. #[error("`http` error: {0}")] Http(#[from] http::Error), + /// A hyper error. #[error("`hyper` error: {0}")] Hyper(#[from] hyper::Error), + /// A URL encoding error. #[error("url encode error: {0}")] UrlEncoded(#[from] serde_urlencoded::ser::Error), + /// A JSON serialization error. #[cfg(feature = "json")] #[error("json error: {0}")] Json(#[from] serde_json::Error), } +/// The result type used in `compio-http`. pub type Result = std::result::Result; diff --git a/compio-http/src/request.rs b/compio-http/src/request.rs index 16b2dfc7..5691d8e6 100644 --- a/compio-http/src/request.rs +++ b/compio-http/src/request.rs @@ -2,14 +2,15 @@ use std::{fmt::Display, time::Duration}; use hyper::{ header::{HeaderName, HeaderValue, AUTHORIZATION, CONTENT_TYPE}, - Body, HeaderMap, Method, Response, Version, + Body, HeaderMap, Method, Version, }; use serde::Serialize; use url::Url; -use crate::{Client, Result}; +use crate::{Client, Response, Result}; /// A request which can be executed with `Client::execute()`. +#[derive(Debug)] pub struct Request { method: Method, url: Url, @@ -127,6 +128,7 @@ impl Request { } /// A builder to construct the properties of a `Request`. +#[derive(Debug)] pub struct RequestBuilder { client: Client, request: Result, @@ -348,10 +350,6 @@ impl RequestBuilder { /// Send a JSON body. /// - /// # Optional - /// - /// This requires the optional `json` feature enabled. - /// /// # Errors /// /// Serialization can fail if `T`'s implementation of `Serialize` decides to @@ -394,7 +392,7 @@ impl RequestBuilder { /// Constructs the Request and sends it to the target URL, returning a /// future Response. - pub async fn send(self) -> Result> { + pub async fn send(self) -> Result { self.client.execute(self.request?).await } } diff --git a/compio-http/src/response.rs b/compio-http/src/response.rs new file mode 100644 index 00000000..3374403e --- /dev/null +++ b/compio-http/src/response.rs @@ -0,0 +1,210 @@ +use compio_buf::bytes::Bytes; +use encoding_rs::{Encoding, UTF_8}; +use http::{header::CONTENT_TYPE, HeaderMap, StatusCode, Version}; +use hyper::Body; +use mime::Mime; +use url::Url; + +use crate::Result; + +/// A Response to a submitted `Request`. +#[derive(Debug)] +pub struct Response { + pub(super) res: hyper::Response, + url: Url, +} + +impl Response { + pub(super) fn new(res: hyper::Response, url: Url) -> Response { + Response { res, url } + } + + /// Get the `StatusCode` of this `Response`. + #[inline] + pub fn status(&self) -> StatusCode { + self.res.status() + } + + /// Get the HTTP `Version` of this `Response`. + #[inline] + pub fn version(&self) -> Version { + self.res.version() + } + + /// Get the `Headers` of this `Response`. + #[inline] + pub fn headers(&self) -> &HeaderMap { + self.res.headers() + } + + /// Get a mutable reference to the `Headers` of this `Response`. + #[inline] + pub fn headers_mut(&mut self) -> &mut HeaderMap { + self.res.headers_mut() + } + + /// Get the content-length of this response, if known. + /// + /// Reasons it may not be known: + /// + /// - The server didn't send a `content-length` header. + /// - The response is compressed and automatically decoded (thus changing + /// the actual decoded length). + pub fn content_length(&self) -> Option { + use hyper::body::HttpBody; + + HttpBody::size_hint(self.res.body()).exact() + } + + /// Get the final `Url` of this `Response`. + #[inline] + pub fn url(&self) -> &Url { + &self.url + } + + /// Returns a reference to the associated extensions. + pub fn extensions(&self) -> &http::Extensions { + self.res.extensions() + } + + /// Returns a mutable reference to the associated extensions. + pub fn extensions_mut(&mut self) -> &mut http::Extensions { + self.res.extensions_mut() + } + + // body methods + + /// Get the full response text. + /// + /// This method decodes the response body with BOM sniffing + /// and with malformed sequences replaced with the REPLACEMENT CHARACTER. + /// Encoding is determined from the `charset` parameter of `Content-Type` + /// header, and defaults to `utf-8` if not presented. + /// + /// Note that the BOM is stripped from the returned String. + /// + /// # Example + /// + /// ``` + /// # async fn run() -> Result<(), Box> { + /// let content = reqwest::get("http://httpbin.org/range/26") + /// .await? + /// .text() + /// .await?; + /// + /// println!("text: {:?}", content); + /// # Ok(()) + /// # } + /// ``` + pub async fn text(self) -> Result { + self.text_with_charset("utf-8").await + } + + /// Get the full response text given a specific encoding. + /// + /// This method decodes the response body with BOM sniffing + /// and with malformed sequences replaced with the REPLACEMENT CHARACTER. + /// You can provide a default encoding for decoding the raw message, while + /// the `charset` parameter of `Content-Type` header is still + /// prioritized. For more information about the possible encoding name, + /// please go to [`encoding_rs`] docs. + /// + /// Note that the BOM is stripped from the returned String. + /// + /// [`encoding_rs`]: https://docs.rs/encoding_rs/0.8/encoding_rs/#relationship-with-windows-code-pages + /// + /// # Example + /// + /// ``` + /// # async fn run() -> Result<(), Box> { + /// let content = reqwest::get("http://httpbin.org/range/26") + /// .await? + /// .text_with_charset("utf-8") + /// .await?; + /// + /// println!("text: {:?}", content); + /// # Ok(()) + /// # } + /// ``` + pub async fn text_with_charset(self, default_encoding: &str) -> Result { + let content_type = self + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + let encoding_name = content_type + .as_ref() + .and_then(|mime| mime.get_param("charset").map(|charset| charset.as_str())) + .unwrap_or(default_encoding); + let encoding = Encoding::for_label(encoding_name.as_bytes()).unwrap_or(UTF_8); + + let full = self.bytes().await?; + + let (text, ..) = encoding.decode(&full); + Ok(text.into_owned()) + } + + /// Try to deserialize the response body as JSON. + /// + /// # Optional + /// + /// This requires the optional `json` feature enabled. + /// + /// # Examples + /// + /// ``` + /// # extern crate reqwest; + /// # extern crate serde; + /// # + /// # use reqwest::Error; + /// # use serde::Deserialize; + /// # + /// // This `derive` requires the `serde` dependency. + /// #[derive(Deserialize)] + /// struct Ip { + /// origin: String, + /// } + /// + /// # async fn run() -> Result<(), Error> { + /// let ip = reqwest::get("http://httpbin.org/ip") + /// .await? + /// .json::() + /// .await?; + /// + /// println!("ip: {}", ip.origin); + /// # Ok(()) + /// # } + /// # + /// # fn main() { } + /// ``` + /// + /// # Errors + /// + /// This method fails whenever the response body is not in JSON format + /// or it cannot be properly deserialized to target type `T`. For more + /// details please see [`serde_json::from_reader`]. + /// + /// [`serde_json::from_reader`]: https://docs.serde.rs/serde_json/fn.from_reader.html + #[cfg(feature = "json")] + pub async fn json(self) -> Result { + let full = self.bytes().await?; + + Ok(serde_json::from_slice(&full)?) + } + + /// Get the full response body as `Bytes`. + /// + /// # Example + /// + /// ``` + /// # async fn run() -> Result<(), Box> { + /// let bytes = reqwest::get("http://httpbin.org/ip").await?.bytes().await?; + /// + /// println!("bytes: {:?}", bytes); + /// # Ok(()) + /// # } + /// ``` + pub async fn bytes(self) -> Result { + Ok(hyper::body::to_bytes(self.res.into_body()).await?) + } +} From c55f17d708fe2d285761e2d90d4c3f0bb86a5c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 21:55:44 +0800 Subject: [PATCH 09/44] feat: add IntoUrl --- compio-http/examples/client.rs | 2 +- compio-http/src/client.rs | 10 ++++++---- compio-http/src/into_url.rs | 36 ++++++++++++++++++++++++++++++++++ compio-http/src/lib.rs | 19 +++++++++++++----- compio-http/src/request.rs | 4 ++++ compio-http/src/stream.rs | 4 +--- 6 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 compio-http/src/into_url.rs diff --git a/compio-http/examples/client.rs b/compio-http/examples/client.rs index b0408534..383f2834 100644 --- a/compio-http/examples/client.rs +++ b/compio-http/examples/client.rs @@ -5,7 +5,7 @@ use hyper::Method; async fn main() { let client = Client::new(); let response = client - .request(Method::GET, "https://www.example.com/".parse().unwrap()) + .request(Method::GET, "https://www.example.com/") .send() .await .unwrap(); diff --git a/compio-http/src/client.rs b/compio-http/src/client.rs index fe879d50..e8b24d6c 100644 --- a/compio-http/src/client.rs +++ b/compio-http/src/client.rs @@ -1,9 +1,8 @@ use std::rc::Rc; use hyper::{Body, Method, Uri}; -use url::Url; -use crate::{CompioExecutor, Connector, Request, RequestBuilder, Response, Result}; +use crate::{CompioExecutor, Connector, IntoUrl, Request, RequestBuilder, Response, Result}; /// An asynchronous `Client` to make Requests with. #[derive(Debug, Clone)] @@ -50,8 +49,11 @@ impl Client { } /// Send a request with method and url. - pub fn request(&self, method: Method, url: Url) -> RequestBuilder { - RequestBuilder::from_parts(self.clone(), Request::new(method, url)) + pub fn request(&self, method: Method, url: U) -> RequestBuilder { + RequestBuilder::new( + self.clone(), + url.into_url().map(|url| Request::new(method, url)), + ) } } diff --git a/compio-http/src/into_url.rs b/compio-http/src/into_url.rs new file mode 100644 index 00000000..dacd3c0c --- /dev/null +++ b/compio-http/src/into_url.rs @@ -0,0 +1,36 @@ +use url::Url; + +/// A trait to try to convert some type into a [`Url`]. +pub trait IntoUrl { + /// Besides parsing as a valid [`Url`], the [`Url`] must be a valid + /// `http::Uri`, in that it makes sense to use in a network request. + fn into_url(self) -> crate::Result; +} + +impl IntoUrl for Url { + fn into_url(self) -> crate::Result { + if self.has_host() { + Ok(self) + } else { + Err(crate::Error::BadScheme(self)) + } + } +} + +impl<'a> IntoUrl for &'a str { + fn into_url(self) -> crate::Result { + Ok(Url::parse(self)?) + } +} + +impl<'a> IntoUrl for &'a String { + fn into_url(self) -> crate::Result { + (&**self).into_url() + } +} + +impl IntoUrl for String { + fn into_url(self) -> crate::Result { + (&*self).into_url() + } +} diff --git a/compio-http/src/lib.rs b/compio-http/src/lib.rs index b2a1ee1c..2c0bc9f3 100644 --- a/compio-http/src/lib.rs +++ b/compio-http/src/lib.rs @@ -11,6 +11,9 @@ pub use request::*; mod response; pub use response::*; +mod into_url; +pub use into_url::*; + mod stream; pub(crate) use stream::*; @@ -28,19 +31,25 @@ pub enum Error { /// The request is timeout. #[error("request timeout")] Timeout, - /// An IO error occurs. + /// Bad scheme. + #[error("bad scheme: {0}")] + BadScheme(url::Url), + /// IO error occurs. #[error("system error: {0}")] System(#[from] std::io::Error), - /// An HTTP related parse error. + /// HTTP related parse error. #[error("`http` error: {0}")] Http(#[from] http::Error), - /// A hyper error. + /// Hyper error. #[error("`hyper` error: {0}")] Hyper(#[from] hyper::Error), - /// A URL encoding error. + /// URL parse error. + #[error("url parse error: {0}")] + UrlParse(#[from] url::ParseError), + /// URL encoding error. #[error("url encode error: {0}")] UrlEncoded(#[from] serde_urlencoded::ser::Error), - /// A JSON serialization error. + /// JSON serialization error. #[cfg(feature = "json")] #[error("json error: {0}")] Json(#[from] serde_json::Error), diff --git a/compio-http/src/request.rs b/compio-http/src/request.rs index 5691d8e6..dc6337bc 100644 --- a/compio-http/src/request.rs +++ b/compio-http/src/request.rs @@ -135,6 +135,10 @@ pub struct RequestBuilder { } impl RequestBuilder { + pub(crate) fn new(client: Client, request: Result) -> RequestBuilder { + RequestBuilder { client, request } + } + /// Assemble a builder starting from an existing `Client` and a `Request`. pub fn from_parts(client: Client, request: Request) -> RequestBuilder { RequestBuilder { diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index 26989f3f..2fff41bf 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -26,9 +26,7 @@ enum HttpStreamInner { impl HttpStreamInner { pub async fn new(uri: Uri) -> io::Result { let scheme = uri.scheme_str().unwrap_or("http"); - let host = uri - .host() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "should specify host"))?; + let host = uri.host().expect("there should be host"); let port = uri.port_u16(); match scheme { "http" => { From 94fdd621d8d73bf017e6f5ec23c1097cbc1430f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 21:58:59 +0800 Subject: [PATCH 10/44] feat: convenience methods --- compio-http/examples/client.rs | 7 +------ compio-http/src/client.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/compio-http/examples/client.rs b/compio-http/examples/client.rs index 383f2834..bb5e17cd 100644 --- a/compio-http/examples/client.rs +++ b/compio-http/examples/client.rs @@ -1,13 +1,8 @@ use compio_http::Client; -use hyper::Method; #[compio_macros::main] async fn main() { let client = Client::new(); - let response = client - .request(Method::GET, "https://www.example.com/") - .send() - .await - .unwrap(); + let response = client.get("https://www.example.com/").send().await.unwrap(); println!("{}", response.text().await.unwrap()); } diff --git a/compio-http/src/client.rs b/compio-http/src/client.rs index e8b24d6c..a7c6af2a 100644 --- a/compio-http/src/client.rs +++ b/compio-http/src/client.rs @@ -55,6 +55,36 @@ impl Client { url.into_url().map(|url| Request::new(method, url)), ) } + + /// Convenience method to make a `GET` request to a URL. + pub fn get(&self, url: U) -> RequestBuilder { + self.request(Method::GET, url) + } + + /// Convenience method to make a `POST` request to a URL. + pub fn post(&self, url: U) -> RequestBuilder { + self.request(Method::POST, url) + } + + /// Convenience method to make a `PUT` request to a URL. + pub fn put(&self, url: U) -> RequestBuilder { + self.request(Method::PUT, url) + } + + /// Convenience method to make a `PATCH` request to a URL. + pub fn patch(&self, url: U) -> RequestBuilder { + self.request(Method::PATCH, url) + } + + /// Convenience method to make a `DELETE` request to a URL. + pub fn delete(&self, url: U) -> RequestBuilder { + self.request(Method::DELETE, url) + } + + /// Convenience method to make a `HEAD` request to a URL. + pub fn head(&self, url: U) -> RequestBuilder { + self.request(Method::HEAD, url) + } } #[derive(Debug)] From 6d95a70ddeb817b879b23c9db2ed4eeb5d24f583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 22:35:47 +0800 Subject: [PATCH 11/44] fix: http doc tests --- compio-http/src/request.rs | 8 ++++---- compio-http/src/response.rs | 27 +++++++++++++++++++++------ compio-http/src/util.rs | 2 +- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/compio-http/src/request.rs b/compio-http/src/request.rs index dc6337bc..e737d2e7 100644 --- a/compio-http/src/request.rs +++ b/compio-http/src/request.rs @@ -209,10 +209,10 @@ impl RequestBuilder { /// Enable HTTP basic authentication. /// /// ```rust - /// # use reqwest::Error; + /// # use compio_http::Error; /// /// # async fn run() -> Result<(), Error> { - /// let client = reqwest::Client::new(); + /// let client = compio_http::Client::new(); /// let resp = client /// .delete("http://httpbin.org/delete") /// .basic_auth("admin", Some("good password")) @@ -311,14 +311,14 @@ impl RequestBuilder { /// header. /// /// ```rust - /// # use reqwest::Error; + /// # use compio_http::Error; /// # use std::collections::HashMap; /// # /// # async fn run() -> Result<(), Error> { /// let mut params = HashMap::new(); /// params.insert("lang", "rust"); /// - /// let client = reqwest::Client::new(); + /// let client = compio_http::Client::new(); /// let res = client /// .post("http://httpbin.org") /// .form(¶ms) diff --git a/compio-http/src/response.rs b/compio-http/src/response.rs index 3374403e..f8aa1224 100644 --- a/compio-http/src/response.rs +++ b/compio-http/src/response.rs @@ -87,7 +87,10 @@ impl Response { /// /// ``` /// # async fn run() -> Result<(), Box> { - /// let content = reqwest::get("http://httpbin.org/range/26") + /// let client = compio_http::Client::new(); + /// let content = client + /// .get("http://httpbin.org/range/26") + /// .send() /// .await? /// .text() /// .await?; @@ -117,7 +120,10 @@ impl Response { /// /// ``` /// # async fn run() -> Result<(), Box> { - /// let content = reqwest::get("http://httpbin.org/range/26") + /// let client = compio_http::Client::new(); + /// let content = client + /// .get("http://httpbin.org/range/26") + /// .send() /// .await? /// .text_with_charset("utf-8") /// .await?; @@ -153,10 +159,10 @@ impl Response { /// # Examples /// /// ``` - /// # extern crate reqwest; + /// # extern crate compio_http; /// # extern crate serde; /// # - /// # use reqwest::Error; + /// # use compio_http::Error; /// # use serde::Deserialize; /// # /// // This `derive` requires the `serde` dependency. @@ -166,7 +172,10 @@ impl Response { /// } /// /// # async fn run() -> Result<(), Error> { - /// let ip = reqwest::get("http://httpbin.org/ip") + /// let client = compio_http::Client::new(); + /// let ip = client + /// .get("http://httpbin.org/ip") + /// .send() /// .await? /// .json::() /// .await?; @@ -198,7 +207,13 @@ impl Response { /// /// ``` /// # async fn run() -> Result<(), Box> { - /// let bytes = reqwest::get("http://httpbin.org/ip").await?.bytes().await?; + /// let client = compio_http::Client::new(); + /// let bytes = client + /// .get("http://httpbin.org/ip") + /// .send() + /// .await? + /// .bytes() + /// .await?; /// /// println!("bytes: {:?}", bytes); /// # Ok(()) diff --git a/compio-http/src/util.rs b/compio-http/src/util.rs index a14a57d6..60bc5957 100644 --- a/compio-http/src/util.rs +++ b/compio-http/src/util.rs @@ -1,4 +1,4 @@ -//! Code from reqwest. +//! Code from compio_http. use hyper::{ header::{Entry, HeaderValue, OccupiedEntry}, From 97470508794328afdb4e605f7b5a63ca092fa4e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 22:49:33 +0800 Subject: [PATCH 12/44] test: port some tests from reqwest --- compio-http/Cargo.toml | 1 + compio-http/tests/client.rs | 96 +++++++++++++++++++++++++++++++++ compio-http/tests/server/mod.rs | 91 +++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 compio-http/tests/client.rs create mode 100644 compio-http/tests/server/mod.rs diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 4efffbf1..51535f4a 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -34,6 +34,7 @@ url = "2" [dev-dependencies] compio-macros = { workspace = true } +hyper = { version = "0.14", features = ["server", "tcp"] } [features] default = ["native-tls"] diff --git a/compio-http/tests/client.rs b/compio-http/tests/client.rs new file mode 100644 index 00000000..d304eade --- /dev/null +++ b/compio-http/tests/client.rs @@ -0,0 +1,96 @@ +mod server; + +#[cfg(feature = "json")] +use std::collections::HashMap; + +use compio_http::Client; +#[cfg(feature = "json")] +use http::header::CONTENT_TYPE; + +#[compio_macros::test] +async fn response_text() { + let server = server::http(move |_req| async { http::Response::new("Hello".into()) }); + + let client = Client::new(); + + let res = client + .get(&format!("http://{}/text", server.addr())) + .send() + .await + .expect("Failed to get"); + assert_eq!(res.content_length(), Some(5)); + let text = res.text().await.expect("Failed to get text"); + assert_eq!("Hello", text); +} + +#[compio_macros::test] +async fn response_bytes() { + let server = server::http(move |_req| async { http::Response::new("Hello".into()) }); + + let client = Client::new(); + + let res = client + .get(&format!("http://{}/bytes", server.addr())) + .send() + .await + .expect("Failed to get"); + assert_eq!(res.content_length(), Some(5)); + let bytes = res.bytes().await.expect("res.bytes()"); + assert_eq!("Hello", bytes); +} + +#[compio_macros::test] +#[cfg(feature = "json")] +async fn response_json() { + let server = server::http(move |_req| async { http::Response::new("\"Hello\"".into()) }); + + let client = Client::new(); + + let res = client + .get(&format!("http://{}/json", server.addr())) + .send() + .await + .expect("Failed to get"); + let text = res.json::().await.expect("Failed to get json"); + assert_eq!("Hello", text); +} + +#[compio_macros::test] +async fn test_allowed_methods() { + let resp = compio_http::Client::new() + .get("https://www.example.com") + .send() + .await; + + assert!(resp.is_ok()); +} + +#[test] +#[cfg(feature = "json")] +fn add_json_default_content_type_if_not_set_manually() { + let mut map = HashMap::new(); + map.insert("body", "json"); + let content_type = http::HeaderValue::from_static("application/vnd.api+json"); + let req = Client::new() + .post("https://www.example.com/") + .header(CONTENT_TYPE, &content_type) + .json(&map) + .build() + .expect("request is not valid"); + + assert_eq!(content_type, req.headers().get(CONTENT_TYPE).unwrap()); +} + +#[test] +#[cfg(feature = "json")] +fn update_json_content_type_if_set_manually() { + let mut map = HashMap::new(); + map.insert("body", "json"); + let req = Client::new() + .post("https://www.example.com/") + .json(&map) + .build() + .expect("request is not valid"); + + assert_eq!("application/json", req.headers().get(CONTENT_TYPE).unwrap()); +} diff --git a/compio-http/tests/server/mod.rs b/compio-http/tests/server/mod.rs new file mode 100644 index 00000000..3609fb24 --- /dev/null +++ b/compio-http/tests/server/mod.rs @@ -0,0 +1,91 @@ +use std::{ + convert::Infallible, + future::Future, + net::{self, Ipv4Addr}, + sync::mpsc as std_mpsc, + thread, + time::Duration, +}; + +use tokio::{runtime, sync::oneshot}; + +pub struct Server { + addr: net::SocketAddr, + panic_rx: std_mpsc::Receiver<()>, + shutdown_tx: Option>, +} + +impl Server { + pub fn addr(&self) -> net::SocketAddr { + self.addr + } +} + +impl Drop for Server { + fn drop(&mut self) { + if let Some(tx) = self.shutdown_tx.take() { + let _ = tx.send(()); + } + + if !::std::thread::panicking() { + self.panic_rx + .recv_timeout(Duration::from_secs(3)) + .expect("test server should not panic"); + } + } +} + +pub fn http(func: F) -> Server +where + F: Fn(http::Request) -> Fut + Clone + Send + 'static, + Fut: Future> + Send + 'static, +{ + // Spawn new runtime in thread to prevent reactor execution context conflict + thread::spawn(move || { + let rt = runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("new rt"); + #[allow(clippy::async_yields_async)] + let srv = rt.block_on(async move { + hyper::Server::bind(&(Ipv4Addr::LOCALHOST, 0).into()).serve( + hyper::service::make_service_fn(move |_| { + let func = func.clone(); + async move { + Ok::<_, Infallible>(hyper::service::service_fn(move |req| { + let fut = func(req); + async move { Ok::<_, Infallible>(fut.await) } + })) + } + }), + ) + }); + + let addr = srv.local_addr(); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let srv = srv.with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }); + + let (panic_tx, panic_rx) = std_mpsc::channel(); + let tname = format!( + "test({})-support-server", + thread::current().name().unwrap_or("") + ); + thread::Builder::new() + .name(tname) + .spawn(move || { + rt.block_on(srv).unwrap(); + let _ = panic_tx.send(()); + }) + .expect("thread spawn"); + + Server { + addr, + panic_rx, + shutdown_tx: Some(shutdown_tx), + } + }) + .join() + .unwrap() +} From 4d09948fe75128cbe4a479371233a18261dbbc3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 26 Oct 2023 23:27:40 +0800 Subject: [PATCH 13/44] fix: add "all" feature to compio-http --- compio-http/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 51535f4a..46eaca93 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -12,7 +12,7 @@ repository = { workspace = true } [dependencies] compio-buf = { workspace = true, features = ["bytes"] } compio-io = { workspace = true } -compio-runtime = { workspace = true } +compio-runtime = { workspace = true, features = ["time"] } compio-net = { workspace = true, features = ["runtime"] } compio-tls = { workspace = true } @@ -41,3 +41,4 @@ default = ["native-tls"] native-tls = ["compio-tls/native-tls", "dep:native-tls"] vendored = ["native-tls?/vendored"] json = ["dep:serde_json"] +all = ["json"] From c83dfb67761eca3fdf94b0254a80397da7610ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Mon, 6 Nov 2023 23:33:55 +0800 Subject: [PATCH 14/44] fix: runtime doesn't live long enough --- Cargo.toml | 1 + compio-http/Cargo.toml | 4 ++-- compio-runtime/Cargo.toml | 1 + compio-runtime/src/runtime/mod.rs | 20 ++++++-------------- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b57df73a..e9e81595 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ nix = "0.27.1" once_cell = "1.18.0" os_pipe = "1.1.4" paste = "1.0.14" +send_wrapper = { version = "0.6" } slab = "0.4.9" socket2 = "0.5.5" tempfile = "3.8.1" diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 46eaca93..33514d14 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -20,8 +20,8 @@ native-tls = { version = "0.2", optional = true } http = "0.2" hyper = { version = "0.14", features = ["client", "http1"] } -send_wrapper = { version = "0.6", features = ["futures"] } -tokio = { version = "1", default-features = false } +send_wrapper = { workspace = true, features = ["futures"] } +tokio = { workspace = true } base64 = "0.21" encoding_rs = "0.8" diff --git a/compio-runtime/Cargo.toml b/compio-runtime/Cargo.toml index 87b3a617..b3a0545c 100644 --- a/compio-runtime/Cargo.toml +++ b/compio-runtime/Cargo.toml @@ -36,6 +36,7 @@ async-task = "4.5.0" cfg-if = { workspace = true, optional = true } futures-util = { workspace = true } once_cell = { workspace = true } +send_wrapper = { workspace = true } slab = { workspace = true, optional = true } smallvec = "1.11.1" diff --git a/compio-runtime/src/runtime/mod.rs b/compio-runtime/src/runtime/mod.rs index c03f3808..9b4f9960 100644 --- a/compio-runtime/src/runtime/mod.rs +++ b/compio-runtime/src/runtime/mod.rs @@ -3,13 +3,14 @@ use std::{ collections::VecDeque, future::{ready, Future}, io, + sync::Arc, task::{Context, Poll}, - thread::ThreadId, }; use async_task::{Runnable, Task}; use compio_driver::{AsRawFd, Entry, OpCode, Proactor, ProactorBuilder, PushEntry, RawFd}; use futures_util::future::Either; +use send_wrapper::SendWrapper; use smallvec::SmallVec; pub(crate) mod op; @@ -25,8 +26,7 @@ use crate::{ pub(crate) struct Runtime { driver: RefCell, - thread_id: ThreadId, - runnables: RefCell>, + runnables: Arc>>>, op_runtime: RefCell, #[cfg(feature = "time")] timer_runtime: RefCell, @@ -36,8 +36,7 @@ impl Runtime { pub fn new(builder: &ProactorBuilder) -> io::Result { Ok(Self { driver: RefCell::new(builder.build()?), - thread_id: std::thread::current().id(), - runnables: RefCell::default(), + runnables: Arc::new(SendWrapper::new(RefCell::default())), op_runtime: RefCell::default(), #[cfg(feature = "time")] timer_runtime: RefCell::new(TimerRuntime::new()), @@ -46,16 +45,9 @@ impl Runtime { // Safety: the return runnable should be scheduled. unsafe fn spawn_unchecked(&self, future: F) -> Task { + let runnables = self.runnables.clone(); let schedule = move |runnable| { - #[cold] - fn panic_send_guard() -> ! { - panic!("Cannot wake compio waker in different threads."); - } - - if self.thread_id != std::thread::current().id() { - panic_send_guard(); - } - self.runnables.borrow_mut().push_back(runnable); + runnables.borrow_mut().push_back(runnable); }; let (runnable, task) = async_task::spawn_unchecked(future, schedule); runnable.schedule(); From 5f8363a7421ff78db4ce088bbf48aa9f17b3ef0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Tue, 7 Nov 2023 00:41:11 +0800 Subject: [PATCH 15/44] feat(http): add rustls support --- Cargo.toml | 4 ++ compio-http/Cargo.toml | 6 ++- compio-http/src/backend.rs | 51 ++++++++++++++++++++ compio-http/src/client.rs | 89 ++++++++++++++++++++++++++++++----- compio-http/src/lib.rs | 3 ++ compio-http/src/service.rs | 14 ++++-- compio-http/src/stream.rs | 19 ++++---- compio-tls/Cargo.toml | 6 +-- compio-tls/src/stream/mod.rs | 9 ++++ compio-tls/src/stream/rtls.rs | 38 ++++++++++----- 10 files changed, 200 insertions(+), 39 deletions(-) create mode 100644 compio-http/src/backend.rs diff --git a/Cargo.toml b/Cargo.toml index e9e81595..3dfa2e44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,3 +50,7 @@ tempfile = "3.8.1" tokio = "1.33.0" widestring = "1.0.2" windows-sys = "0.48.0" + +native-tls = "0.2.11" +rustls = "0.21.8" +rustls-native-certs = "0.6.3" diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 33514d14..49817945 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -16,7 +16,9 @@ compio-runtime = { workspace = true, features = ["time"] } compio-net = { workspace = true, features = ["runtime"] } compio-tls = { workspace = true } -native-tls = { version = "0.2", optional = true } +native-tls = { workspace = true, optional = true } +rustls = { workspace = true, optional = true } +rustls-native-certs = { workspace = true, optional = true } http = "0.2" hyper = { version = "0.14", features = ["client", "http1"] } @@ -24,6 +26,7 @@ send_wrapper = { workspace = true, features = ["futures"] } tokio = { workspace = true } base64 = "0.21" +cfg-if = { workspace = true } encoding_rs = "0.8" mime = "0.3" serde = "1" @@ -39,6 +42,7 @@ hyper = { version = "0.14", features = ["server", "tcp"] } [features] default = ["native-tls"] native-tls = ["compio-tls/native-tls", "dep:native-tls"] +rustls = ["compio-tls/rustls", "dep:rustls", "dep:rustls-native-certs"] vendored = ["native-tls?/vendored"] json = ["dep:serde_json"] all = ["json"] diff --git a/compio-http/src/backend.rs b/compio-http/src/backend.rs new file mode 100644 index 00000000..ec52ecd6 --- /dev/null +++ b/compio-http/src/backend.rs @@ -0,0 +1,51 @@ +use std::{io, sync::Arc}; + +use compio_tls::TlsConnector; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum TlsBackend { + #[cfg(feature = "native-tls")] + NativeTls, + #[cfg(feature = "rustls")] + Rustls, +} + +impl Default for TlsBackend { + fn default() -> Self { + cfg_if::cfg_if! { + if #[cfg(feature = "native-tls")] { + Self::NativeTls + } else if #[cfg(feature = "rustls")] { + Self::Rustls + } else { + compile_error!("You must choose at least one of these features: [\"native-tls\", \"rustls\"]") + } + } + } +} + +impl TlsBackend { + pub fn create_connector(&self) -> io::Result { + match self { + #[cfg(feature = "native-tls")] + Self::NativeTls => Ok(TlsConnector::from( + native_tls::TlsConnector::new() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?, + )), + #[cfg(feature = "rustls")] + Self::Rustls => { + let mut store = rustls::RootCertStore::empty(); + for cert in rustls_native_certs::load_native_certs().unwrap() { + store.add(&rustls::Certificate(cert.0)).unwrap(); + } + + Ok(TlsConnector::from(Arc::new( + rustls::ClientConfig::builder() + .with_safe_defaults() + .with_root_certificates(store) + .with_no_client_auth(), + ))) + } + } + } +} diff --git a/compio-http/src/client.rs b/compio-http/src/client.rs index a7c6af2a..f116b9dc 100644 --- a/compio-http/src/client.rs +++ b/compio-http/src/client.rs @@ -1,8 +1,10 @@ use std::rc::Rc; -use hyper::{Body, Method, Uri}; +use hyper::{Body, HeaderMap, Method, Uri}; -use crate::{CompioExecutor, Connector, IntoUrl, Request, RequestBuilder, Response, Result}; +use crate::{ + CompioExecutor, Connector, IntoUrl, Request, RequestBuilder, Response, Result, TlsBackend, +}; /// An asynchronous `Client` to make Requests with. #[derive(Debug, Clone)] @@ -14,14 +16,14 @@ impl Client { /// Create a client with default config. #[allow(clippy::new_without_default)] pub fn new() -> Self { - Self { - client: Rc::new(ClientRef { - client: hyper::Client::builder() - .executor(CompioExecutor) - .set_host(true) - .build(Connector), - }), - } + ClientBuilder::new().build() + } + + /// Creates a `ClientBuilder` to configure a `Client`. + /// + /// This is the same as `ClientBuilder::new()`. + pub fn builder() -> ClientBuilder { + ClientBuilder::new() } /// Send a request and wait for a response. @@ -36,7 +38,9 @@ impl Client { ) .version(version) .body(body.unwrap_or_else(Body::empty))?; - *request.headers_mut() = headers; + *request.headers_mut() = self.client.headers.clone(); + crate::util::replace_headers(request.headers_mut(), headers); + let future = self.client.client.request(request); let res = if let Some(timeout) = timeout { compio_runtime::time::timeout(timeout, future) @@ -90,4 +94,67 @@ impl Client { #[derive(Debug)] struct ClientRef { client: hyper::Client, + headers: HeaderMap, +} + +/// A `ClientBuilder` can be used to create a `Client` with custom +/// configuration. +#[derive(Debug)] +#[must_use] +pub struct ClientBuilder { + headers: HeaderMap, + tls: TlsBackend, +} + +impl Default for ClientBuilder { + fn default() -> Self { + Self::new() + } +} + +impl ClientBuilder { + /// Constructs a new `ClientBuilder`. + pub fn new() -> Self { + Self { + headers: HeaderMap::new(), + tls: TlsBackend::default(), + } + } + + /// Returns a `Client` that uses this `ClientBuilder` configuration. + pub fn build(self) -> Client { + let client = hyper::Client::builder() + .executor(CompioExecutor) + .set_host(true) + .build(Connector::new(self.tls)); + let client_ref = ClientRef { + client, + headers: self.headers, + }; + Client { + client: Rc::new(client_ref), + } + } + + /// Set the default headers for every request. + pub fn default_headers(mut self, headers: HeaderMap) -> Self { + for (key, value) in headers.iter() { + self.headers.insert(key, value.clone()); + } + self + } + + /// Force using the native TLS backend. + #[cfg(feature = "native-tls")] + pub fn use_native_tls(mut self) -> Self { + self.tls = TlsBackend::NativeTls; + self + } + + /// Force using the Rustls TLS backend. + #[cfg(feature = "rustls")] + pub fn use_rustls_tls(mut self) -> Self { + self.tls = TlsBackend::Rustls; + self + } } diff --git a/compio-http/src/lib.rs b/compio-http/src/lib.rs index 2c0bc9f3..2d99c18d 100644 --- a/compio-http/src/lib.rs +++ b/compio-http/src/lib.rs @@ -20,6 +20,9 @@ pub(crate) use stream::*; mod service; pub(crate) use service::*; +mod backend; +pub(crate) use backend::*; + mod util; use thiserror::Error; diff --git a/compio-http/src/service.rs b/compio-http/src/service.rs index 9cbce9df..e91e6227 100644 --- a/compio-http/src/service.rs +++ b/compio-http/src/service.rs @@ -8,7 +8,7 @@ use std::{ use hyper::{rt::Executor, service::Service, Uri}; use send_wrapper::SendWrapper; -use crate::HttpStream; +use crate::{HttpStream, TlsBackend}; #[derive(Debug, Clone)] pub struct CompioExecutor; @@ -20,7 +20,15 @@ impl Executor + Send>>> for CompioExecutor { } #[derive(Debug, Clone)] -pub struct Connector; +pub struct Connector { + tls: TlsBackend, +} + +impl Connector { + pub fn new(tls: TlsBackend) -> Self { + Self { tls } + } +} impl Service for Connector { type Error = io::Error; @@ -32,6 +40,6 @@ impl Service for Connector { } fn call(&mut self, req: Uri) -> Self::Future { - Box::pin(SendWrapper::new(HttpStream::new(req))) + Box::pin(SendWrapper::new(HttpStream::new(req, self.tls))) } } diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index 2fff41bf..d25b2c58 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -11,20 +11,22 @@ use compio_buf::{ }; use compio_io::{AsyncRead, AsyncWrite, AsyncWriteExt, Buffer}; use compio_net::TcpStream; -use compio_tls::{TlsConnector, TlsStream}; +use compio_tls::TlsStream; use hyper::{ client::connect::{Connected, Connection}, Uri, }; use send_wrapper::SendWrapper; +use crate::TlsBackend; + enum HttpStreamInner { Tcp(TcpStream), Tls(TlsStream), } impl HttpStreamInner { - pub async fn new(uri: Uri) -> io::Result { + pub async fn new(uri: Uri, tls: TlsBackend) -> io::Result { let scheme = uri.scheme_str().unwrap_or("http"); let host = uri.host().expect("there should be host"); let port = uri.port_u16(); @@ -35,10 +37,7 @@ impl HttpStreamInner { } "https" => { let stream = TcpStream::connect((host, port.unwrap_or(443))).await?; - let connector = TlsConnector::from( - native_tls::TlsConnector::new() - .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?, - ); + let connector = tls.create_connector()?; Ok(Self::Tls(connector.connect(host, stream).await?)) } _ => Err(io::Error::new( @@ -104,9 +103,9 @@ struct HttpStreamBufInner { } impl HttpStreamBufInner { - pub async fn new(uri: Uri) -> io::Result { + pub async fn new(uri: Uri, tls: TlsBackend) -> io::Result { Ok(Self { - inner: HttpStreamInner::new(uri).await?, + inner: HttpStreamInner::new(uri, tls).await?, read_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), write_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), }) @@ -184,9 +183,9 @@ pub struct HttpStream { } impl HttpStream { - pub async fn new(uri: Uri) -> io::Result { + pub async fn new(uri: Uri, tls: TlsBackend) -> io::Result { Ok(Self { - inner: SendWrapper::new(HttpStreamBufInner::new(uri).await?), + inner: SendWrapper::new(HttpStreamBufInner::new(uri, tls).await?), read_future: None, write_future: None, flush_future: None, diff --git a/compio-tls/Cargo.toml b/compio-tls/Cargo.toml index 851ccd2d..c781f0d7 100644 --- a/compio-tls/Cargo.toml +++ b/compio-tls/Cargo.toml @@ -13,15 +13,15 @@ repository = { workspace = true } compio-buf = { workspace = true } compio-io = { workspace = true } -native-tls = { version = "0.2.11", optional = true } -rustls = { version = "0.21.8", optional = true } +native-tls = { workspace = true, optional = true } +rustls = { workspace = true, optional = true } [dev-dependencies] compio-net = { workspace = true } compio-runtime = { workspace = true } compio-macros = { workspace = true } -rustls-native-certs = "0.6.3" +rustls-native-certs = { workspace = true } [features] default = ["native-tls"] diff --git a/compio-tls/src/stream/mod.rs b/compio-tls/src/stream/mod.rs index db8afa92..97afa04c 100644 --- a/compio-tls/src/stream/mod.rs +++ b/compio-tls/src/stream/mod.rs @@ -134,6 +134,15 @@ impl AsyncWrite for TlsStream { } async fn flush(&mut self) -> io::Result<()> { + loop { + match io::Write::flush(&mut self.0) { + Ok(()) => break, + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + self.0.get_mut().flush_write_buf().await?; + } + Err(e) => return Err(e), + } + } self.0.get_mut().flush_write_buf().await?; Ok(()) } diff --git a/compio-tls/src/stream/rtls.rs b/compio-tls/src/stream/rtls.rs index 6b3c2cec..6a958a20 100644 --- a/compio-tls/src/stream/rtls.rs +++ b/compio-tls/src/stream/rtls.rs @@ -37,12 +37,26 @@ impl TlsConnection { } } + pub fn wants_read(&self) -> bool { + match self { + Self::Client(c) => c.wants_read(), + Self::Server(c) => c.wants_read(), + } + } + pub fn write_tls(&mut self, wr: &mut dyn io::Write) -> io::Result { match self { Self::Client(c) => c.write_tls(wr), Self::Server(c) => c.write_tls(wr), } } + + pub fn wants_write(&self) -> bool { + match self { + Self::Client(c) => c.wants_write(), + Self::Server(c) => c.wants_write(), + } + } } #[derive(Debug)] @@ -74,20 +88,20 @@ impl TlsStream { impl io::Read for TlsStream { fn read(&mut self, buf: &mut [u8]) -> io::Result { loop { + io::Write::flush(self)?; + + while self.conn.wants_read() { + self.conn.read_tls(&mut self.inner)?; + self.conn + .process_new_packets() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + } + match self.conn.reader().read(buf) { Ok(len) => { return Ok(len); } - Err(e) if e.kind() == io::ErrorKind::WouldBlock => { - self.conn.read_tls(&mut self.inner)?; - let state = self - .conn - .process_new_packets() - .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; - if state.tls_bytes_to_write() > 0 { - io::Write::flush(self)?; - } - } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, Err(e) => return Err(e), } } @@ -101,7 +115,9 @@ impl io::Write for TlsStream { } fn flush(&mut self) -> io::Result<()> { - self.conn.write_tls(&mut self.inner)?; + while self.conn.wants_write() { + self.conn.write_tls(&mut self.inner)?; + } self.inner.flush()?; Ok(()) } From c06c251be25eab2ace6da2d008b127c763237f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Tue, 7 Nov 2023 00:48:59 +0800 Subject: [PATCH 16/44] fix: clippy warnings --- compio-http/Cargo.toml | 2 +- compio-http/src/backend.rs | 4 ++-- compio-http/src/stream.rs | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 49817945..94662010 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -45,4 +45,4 @@ native-tls = ["compio-tls/native-tls", "dep:native-tls"] rustls = ["compio-tls/rustls", "dep:rustls", "dep:rustls-native-certs"] vendored = ["native-tls?/vendored"] json = ["dep:serde_json"] -all = ["json"] +all = ["json", "native-tls", "rustls"] diff --git a/compio-http/src/backend.rs b/compio-http/src/backend.rs index ec52ecd6..fa5ac793 100644 --- a/compio-http/src/backend.rs +++ b/compio-http/src/backend.rs @@ -1,4 +1,4 @@ -use std::{io, sync::Arc}; +use std::io; use compio_tls::TlsConnector; @@ -39,7 +39,7 @@ impl TlsBackend { store.add(&rustls::Certificate(cert.0)).unwrap(); } - Ok(TlsConnector::from(Arc::new( + Ok(TlsConnector::from(std::sync::Arc::new( rustls::ClientConfig::builder() .with_safe_defaults() .with_root_certificates(store) diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index d25b2c58..6f52ed79 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -20,6 +20,7 @@ use send_wrapper::SendWrapper; use crate::TlsBackend; +#[allow(clippy::large_enum_variant)] enum HttpStreamInner { Tcp(TcpStream), Tls(TlsStream), From ecc6e35b86abd0b027fc48b407da6070274b4bed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Tue, 7 Nov 2023 19:23:36 +0800 Subject: [PATCH 17/44] fix(runtime): use manually drop --- compio-runtime/src/lib.rs | 12 +++++++++--- compio-runtime/src/runtime/mod.rs | 10 ++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/compio-runtime/src/lib.rs b/compio-runtime/src/lib.rs index 61a2b4f4..761d459b 100644 --- a/compio-runtime/src/lib.rs +++ b/compio-runtime/src/lib.rs @@ -24,7 +24,7 @@ pub mod event; #[cfg(feature = "time")] pub mod time; -use std::{cell::RefCell, future::Future, io}; +use std::{cell::RefCell, future::Future, io, mem::ManuallyDrop}; use async_task::Task; pub use attacher::*; @@ -37,8 +37,14 @@ use runtime::Runtime; thread_local! { pub(crate) static PROACTOR_BUILDER: RefCell = RefCell::new(ProactorBuilder::new()); - pub(crate) static RUNTIME: LazyCell = LazyCell::new(|| { - PROACTOR_BUILDER.with(|builder| Runtime::new(&builder.borrow())).expect("cannot create compio runtime") + // Use ManuallyDrop here to avoid misc bugs on some platforms when + // trying to get current thread id after thread local resources dropped. + pub(crate) static RUNTIME: LazyCell> = LazyCell::new(|| { + ManuallyDrop::new( + PROACTOR_BUILDER + .with(|builder| Runtime::new(&builder.borrow())) + .expect("cannot create compio runtime") + ) }); } diff --git a/compio-runtime/src/runtime/mod.rs b/compio-runtime/src/runtime/mod.rs index 9b4f9960..d46732a1 100644 --- a/compio-runtime/src/runtime/mod.rs +++ b/compio-runtime/src/runtime/mod.rs @@ -3,7 +3,7 @@ use std::{ collections::VecDeque, future::{ready, Future}, io, - sync::Arc, + rc::Rc, task::{Context, Poll}, }; @@ -26,7 +26,7 @@ use crate::{ pub(crate) struct Runtime { driver: RefCell, - runnables: Arc>>>, + runnables: Rc>>, op_runtime: RefCell, #[cfg(feature = "time")] timer_runtime: RefCell, @@ -36,7 +36,7 @@ impl Runtime { pub fn new(builder: &ProactorBuilder) -> io::Result { Ok(Self { driver: RefCell::new(builder.build()?), - runnables: Arc::new(SendWrapper::new(RefCell::default())), + runnables: Rc::new(RefCell::default()), op_runtime: RefCell::default(), #[cfg(feature = "time")] timer_runtime: RefCell::new(TimerRuntime::new()), @@ -45,7 +45,9 @@ impl Runtime { // Safety: the return runnable should be scheduled. unsafe fn spawn_unchecked(&self, future: F) -> Task { - let runnables = self.runnables.clone(); + // clone is cheap because it is Rc; + // SendWrapper is used to avoid cross-thread scheduling. + let runnables = SendWrapper::new(self.runnables.clone()); let schedule = move |runnable| { runnables.borrow_mut().push_back(runnable); }; From d3d17d73aa666ef4462bf676d3fd7245700cf6ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Tue, 7 Nov 2023 19:28:18 +0800 Subject: [PATCH 18/44] test(http): add different tls backend test --- compio-http/src/client.rs | 2 +- compio-http/tests/client.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/compio-http/src/client.rs b/compio-http/src/client.rs index f116b9dc..d3540b79 100644 --- a/compio-http/src/client.rs +++ b/compio-http/src/client.rs @@ -153,7 +153,7 @@ impl ClientBuilder { /// Force using the Rustls TLS backend. #[cfg(feature = "rustls")] - pub fn use_rustls_tls(mut self) -> Self { + pub fn use_rustls(mut self) -> Self { self.tls = TlsBackend::Rustls; self } diff --git a/compio-http/tests/client.rs b/compio-http/tests/client.rs index d304eade..86d9b179 100644 --- a/compio-http/tests/client.rs +++ b/compio-http/tests/client.rs @@ -65,6 +65,32 @@ async fn test_allowed_methods() { assert!(resp.is_ok()); } +#[compio_macros::test] +#[cfg(feature = "native-tls")] +async fn test_native_tls() { + let resp = compio_http::ClientBuilder::new() + .use_native_tls() + .build() + .get("https://www.example.com") + .send() + .await + .unwrap(); + resp.text().await.unwrap(); +} + +#[compio_macros::test] +#[cfg(feature = "rustls")] +async fn test_rustls() { + let resp = compio_http::ClientBuilder::new() + .use_rustls() + .build() + .get("https://www.example.com") + .send() + .await + .unwrap(); + resp.text().await.unwrap(); +} + #[test] #[cfg(feature = "json")] fn add_json_default_content_type_if_not_set_manually() { From 3acf0c70ca242ca4bb2db575a09c41415b6fc101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Tue, 7 Nov 2023 19:42:13 +0800 Subject: [PATCH 19/44] fix(tls): don't write in read --- compio-tls/src/stream/mod.rs | 6 +----- compio-tls/src/stream/rtls.rs | 4 +--- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/compio-tls/src/stream/mod.rs b/compio-tls/src/stream/mod.rs index 97afa04c..746d21ce 100644 --- a/compio-tls/src/stream/mod.rs +++ b/compio-tls/src/stream/mod.rs @@ -89,7 +89,7 @@ impl From>> for TlsStream { } } -impl AsyncRead for TlsStream { +impl AsyncRead for TlsStream { async fn read(&mut self, mut buf: B) -> BufResult { let slice: &mut [MaybeUninit] = buf.as_mut_slice(); slice.fill(MaybeUninit::new(0)); @@ -103,10 +103,6 @@ impl AsyncRead for TlsStream { return BufResult(Ok(res), buf); } Err(e) if e.kind() == io::ErrorKind::WouldBlock => { - match self.flush().await { - Ok(()) => {} - Err(e) => return BufResult(Err(e), buf), - } match self.0.get_mut().fill_read_buf().await { Ok(_) => continue, Err(e) => return BufResult(Err(e), buf), diff --git a/compio-tls/src/stream/rtls.rs b/compio-tls/src/stream/rtls.rs index 6a958a20..f669fa98 100644 --- a/compio-tls/src/stream/rtls.rs +++ b/compio-tls/src/stream/rtls.rs @@ -85,11 +85,9 @@ impl TlsStream { } } -impl io::Read for TlsStream { +impl io::Read for TlsStream { fn read(&mut self, buf: &mut [u8]) -> io::Result { loop { - io::Write::flush(self)?; - while self.conn.wants_read() { self.conn.read_tls(&mut self.inner)?; self.conn From e367a3c198a1734c874f18308f3b63c34608f9fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Tue, 7 Nov 2023 20:24:08 +0800 Subject: [PATCH 20/44] feat: split http & http-client --- Cargo.toml | 4 ++ compio-http-client/Cargo.toml | 42 +++++++++++++ .../examples/client.rs | 2 +- .../src/client.rs | 5 +- .../src/into_url.rs | 0 compio-http-client/src/lib.rs | 52 ++++++++++++++++ .../src/request.rs | 0 .../src/response.rs | 0 .../src/util.rs | 0 .../tests/client.rs | 11 ++-- .../tests/server/mod.rs | 0 compio-http/Cargo.toml | 25 ++------ compio-http/src/backend.rs | 7 ++- compio-http/src/lib.rs | 61 ++----------------- compio-http/src/service.rs | 9 ++- compio-http/src/stream.rs | 3 + compio-tls/src/lib.rs | 1 + 17 files changed, 133 insertions(+), 89 deletions(-) create mode 100644 compio-http-client/Cargo.toml rename {compio-http => compio-http-client}/examples/client.rs (86%) rename {compio-http => compio-http-client}/src/client.rs (97%) rename {compio-http => compio-http-client}/src/into_url.rs (100%) create mode 100644 compio-http-client/src/lib.rs rename {compio-http => compio-http-client}/src/request.rs (100%) rename {compio-http => compio-http-client}/src/response.rs (100%) rename {compio-http => compio-http-client}/src/util.rs (100%) rename {compio-http => compio-http-client}/tests/client.rs (92%) rename {compio-http => compio-http-client}/tests/server/mod.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 3dfa2e44..7e18d012 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "compio-io", "compio-tls", "compio-http", + "compio-http-client", ] resolver = "2" @@ -33,6 +34,7 @@ compio-net = { path = "./compio-net", version = "0.2.0" } compio-signal = { path = "./compio-signal", version = "0.1.1" } compio-dispatcher = { path = "./compio-dispatcher", version = "0.1.0" } compio-tls = { path = "./compio-tls", version = "0.1.0" } +compio-http = { path = "./compio-http", version = "0.1.0" } cfg-if = "1.0.0" crossbeam-channel = "0.5.8" @@ -54,3 +56,5 @@ windows-sys = "0.48.0" native-tls = "0.2.11" rustls = "0.21.8" rustls-native-certs = "0.6.3" + +hyper = "0.14" diff --git a/compio-http-client/Cargo.toml b/compio-http-client/Cargo.toml new file mode 100644 index 00000000..7d6a354d --- /dev/null +++ b/compio-http-client/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "compio-http-client" +version = "0.1.0" +categories = ["asynchronous", "network-programming"] +keywords = ["async", "net"] +edition = { workspace = true } +authors = { workspace = true } +readme = { workspace = true } +license = { workspace = true } +repository = { workspace = true } + +[dependencies] +compio-buf = { workspace = true, features = ["bytes"] } +compio-runtime = { workspace = true, features = ["time"] } +compio-tls = { workspace = true } +compio-http = { workspace = true } + +http = "0.2" +hyper = { workspace = true, features = ["client", "http1"] } + +base64 = "0.21" +encoding_rs = "0.8" +mime = "0.3" +serde = "1" +serde_json = { version = "1", optional = true } +serde_urlencoded = "0.7" +thiserror = "1" +url = "2" + +[dev-dependencies] +compio-macros = { workspace = true } + +hyper = { version = "0.14", features = ["server", "tcp"] } +tokio = { workspace = true, features = ["rt", "sync"] } + +[features] +default = ["native-tls"] +native-tls = ["compio-http/native-tls"] +rustls = ["compio-http/rustls"] +vendored = ["compio-http/vendored"] +json = ["dep:serde_json"] +all = ["json", "native-tls", "rustls", "compio-http/all"] diff --git a/compio-http/examples/client.rs b/compio-http-client/examples/client.rs similarity index 86% rename from compio-http/examples/client.rs rename to compio-http-client/examples/client.rs index bb5e17cd..e3d6ee25 100644 --- a/compio-http/examples/client.rs +++ b/compio-http-client/examples/client.rs @@ -1,4 +1,4 @@ -use compio_http::Client; +use compio_http_client::Client; #[compio_macros::main] async fn main() { diff --git a/compio-http/src/client.rs b/compio-http-client/src/client.rs similarity index 97% rename from compio-http/src/client.rs rename to compio-http-client/src/client.rs index d3540b79..3ce1f0c9 100644 --- a/compio-http/src/client.rs +++ b/compio-http-client/src/client.rs @@ -1,10 +1,9 @@ use std::rc::Rc; +use compio_http::{CompioExecutor, Connector, TlsBackend}; use hyper::{Body, HeaderMap, Method, Uri}; -use crate::{ - CompioExecutor, Connector, IntoUrl, Request, RequestBuilder, Response, Result, TlsBackend, -}; +use crate::{IntoUrl, Request, RequestBuilder, Response, Result}; /// An asynchronous `Client` to make Requests with. #[derive(Debug, Clone)] diff --git a/compio-http/src/into_url.rs b/compio-http-client/src/into_url.rs similarity index 100% rename from compio-http/src/into_url.rs rename to compio-http-client/src/into_url.rs diff --git a/compio-http-client/src/lib.rs b/compio-http-client/src/lib.rs new file mode 100644 index 00000000..c835d91f --- /dev/null +++ b/compio-http-client/src/lib.rs @@ -0,0 +1,52 @@ +//! A high level HTTP client based on compio. + +#![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] + +mod client; +pub use client::*; + +mod request; +pub use request::*; + +mod response; +pub use response::*; + +mod into_url; +pub use into_url::*; + +mod util; + +/// The error type used in `compio-http`. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The request is timeout. + #[error("request timeout")] + Timeout, + /// Bad scheme. + #[error("bad scheme: {0}")] + BadScheme(url::Url), + /// IO error occurs. + #[error("system error: {0}")] + System(#[from] std::io::Error), + /// HTTP related parse error. + #[error("`http` error: {0}")] + Http(#[from] http::Error), + /// Hyper error. + #[error("`hyper` error: {0}")] + Hyper(#[from] hyper::Error), + /// URL parse error. + #[error("url parse error: {0}")] + UrlParse(#[from] url::ParseError), + /// URL encoding error. + #[error("url encode error: {0}")] + UrlEncoded(#[from] serde_urlencoded::ser::Error), + /// JSON serialization error. + #[cfg(feature = "json")] + #[error("json error: {0}")] + Json(#[from] serde_json::Error), +} + +/// The result type used in `compio-http`. +pub type Result = std::result::Result; diff --git a/compio-http/src/request.rs b/compio-http-client/src/request.rs similarity index 100% rename from compio-http/src/request.rs rename to compio-http-client/src/request.rs diff --git a/compio-http/src/response.rs b/compio-http-client/src/response.rs similarity index 100% rename from compio-http/src/response.rs rename to compio-http-client/src/response.rs diff --git a/compio-http/src/util.rs b/compio-http-client/src/util.rs similarity index 100% rename from compio-http/src/util.rs rename to compio-http-client/src/util.rs diff --git a/compio-http/tests/client.rs b/compio-http-client/tests/client.rs similarity index 92% rename from compio-http/tests/client.rs rename to compio-http-client/tests/client.rs index 86d9b179..731e3d3e 100644 --- a/compio-http/tests/client.rs +++ b/compio-http-client/tests/client.rs @@ -3,7 +3,7 @@ mod server; #[cfg(feature = "json")] use std::collections::HashMap; -use compio_http::Client; +use compio_http_client::Client; #[cfg(feature = "json")] use http::header::CONTENT_TYPE; @@ -57,10 +57,7 @@ async fn response_json() { #[compio_macros::test] async fn test_allowed_methods() { - let resp = compio_http::Client::new() - .get("https://www.example.com") - .send() - .await; + let resp = Client::new().get("https://www.example.com").send().await; assert!(resp.is_ok()); } @@ -68,7 +65,7 @@ async fn test_allowed_methods() { #[compio_macros::test] #[cfg(feature = "native-tls")] async fn test_native_tls() { - let resp = compio_http::ClientBuilder::new() + let resp = Client::builder() .use_native_tls() .build() .get("https://www.example.com") @@ -81,7 +78,7 @@ async fn test_native_tls() { #[compio_macros::test] #[cfg(feature = "rustls")] async fn test_rustls() { - let resp = compio_http::ClientBuilder::new() + let resp = Client::builder() .use_rustls() .build() .get("https://www.example.com") diff --git a/compio-http/tests/server/mod.rs b/compio-http-client/tests/server/mod.rs similarity index 100% rename from compio-http/tests/server/mod.rs rename to compio-http-client/tests/server/mod.rs diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 94662010..7c785d6e 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -10,9 +10,9 @@ license = { workspace = true } repository = { workspace = true } [dependencies] -compio-buf = { workspace = true, features = ["bytes"] } +compio-buf = { workspace = true } compio-io = { workspace = true } -compio-runtime = { workspace = true, features = ["time"] } +compio-runtime = { workspace = true } compio-net = { workspace = true, features = ["runtime"] } compio-tls = { workspace = true } @@ -20,29 +20,14 @@ native-tls = { workspace = true, optional = true } rustls = { workspace = true, optional = true } rustls-native-certs = { workspace = true, optional = true } -http = "0.2" -hyper = { version = "0.14", features = ["client", "http1"] } +cfg-if = { workspace = true } +hyper = { workspace = true } send_wrapper = { workspace = true, features = ["futures"] } tokio = { workspace = true } -base64 = "0.21" -cfg-if = { workspace = true } -encoding_rs = "0.8" -mime = "0.3" -serde = "1" -serde_json = { version = "1", optional = true } -serde_urlencoded = "0.7" -thiserror = "1" -url = "2" - -[dev-dependencies] -compio-macros = { workspace = true } -hyper = { version = "0.14", features = ["server", "tcp"] } - [features] default = ["native-tls"] native-tls = ["compio-tls/native-tls", "dep:native-tls"] rustls = ["compio-tls/rustls", "dep:rustls", "dep:rustls-native-certs"] vendored = ["native-tls?/vendored"] -json = ["dep:serde_json"] -all = ["json", "native-tls", "rustls"] +all = ["native-tls", "rustls"] diff --git a/compio-http/src/backend.rs b/compio-http/src/backend.rs index fa5ac793..2bd62f71 100644 --- a/compio-http/src/backend.rs +++ b/compio-http/src/backend.rs @@ -2,10 +2,13 @@ use std::io; use compio_tls::TlsConnector; +/// Represents TLS backend options #[derive(Debug, Clone, Copy)] -pub(crate) enum TlsBackend { +pub enum TlsBackend { + /// Use [`native_tls`] as TLS backend. #[cfg(feature = "native-tls")] NativeTls, + /// Use [`rustls`] as TLS backend. #[cfg(feature = "rustls")] Rustls, } @@ -25,7 +28,7 @@ impl Default for TlsBackend { } impl TlsBackend { - pub fn create_connector(&self) -> io::Result { + pub(crate) fn create_connector(&self) -> io::Result { match self { #[cfg(feature = "native-tls")] Self::NativeTls => Ok(TlsConnector::from( diff --git a/compio-http/src/lib.rs b/compio-http/src/lib.rs index 2d99c18d..b501b104 100644 --- a/compio-http/src/lib.rs +++ b/compio-http/src/lib.rs @@ -1,62 +1,13 @@ -//! A high level HTTP client library based on compio. +//! A mid level HTTP services for [`hyper`]. #![warn(missing_docs)] - -mod client; -pub use client::*; - -mod request; -pub use request::*; - -mod response; -pub use response::*; - -mod into_url; -pub use into_url::*; - -mod stream; -pub(crate) use stream::*; +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] mod service; -pub(crate) use service::*; +pub use service::*; mod backend; -pub(crate) use backend::*; - -mod util; - -use thiserror::Error; +pub use backend::*; -/// The error type used in `compio-http`. -#[derive(Debug, Error)] -#[non_exhaustive] -pub enum Error { - /// The request is timeout. - #[error("request timeout")] - Timeout, - /// Bad scheme. - #[error("bad scheme: {0}")] - BadScheme(url::Url), - /// IO error occurs. - #[error("system error: {0}")] - System(#[from] std::io::Error), - /// HTTP related parse error. - #[error("`http` error: {0}")] - Http(#[from] http::Error), - /// Hyper error. - #[error("`hyper` error: {0}")] - Hyper(#[from] hyper::Error), - /// URL parse error. - #[error("url parse error: {0}")] - UrlParse(#[from] url::ParseError), - /// URL encoding error. - #[error("url encode error: {0}")] - UrlEncoded(#[from] serde_urlencoded::ser::Error), - /// JSON serialization error. - #[cfg(feature = "json")] - #[error("json error: {0}")] - Json(#[from] serde_json::Error), -} - -/// The result type used in `compio-http`. -pub type Result = std::result::Result; +mod stream; +pub use stream::*; diff --git a/compio-http/src/service.rs b/compio-http/src/service.rs index e91e6227..23f3bbc2 100644 --- a/compio-http/src/service.rs +++ b/compio-http/src/service.rs @@ -10,7 +10,9 @@ use send_wrapper::SendWrapper; use crate::{HttpStream, TlsBackend}; -#[derive(Debug, Clone)] +/// An executor service based on [`compio_runtime`]. It uses +/// [`compio_runtime::spawn`] interally. +#[derive(Debug, Default, Clone)] pub struct CompioExecutor; impl Executor + Send>>> for CompioExecutor { @@ -19,12 +21,17 @@ impl Executor + Send>>> for CompioExecutor { } } +/// An HTTP connector service. +/// +/// It panics when called in a different thread other than the thread creates +/// it. #[derive(Debug, Clone)] pub struct Connector { tls: TlsBackend, } impl Connector { + /// Creates the connector with specific TLS backend. pub fn new(tls: TlsBackend) -> Self { Self { tls } } diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index 6f52ed79..ca7a33bf 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -175,6 +175,8 @@ impl HttpStreamBufInner { type PinBoxFuture = Pin + Send>>; +/// A HTTP stream wrapper, based on compio, and exposes [`tokio::io`] +/// interfaces. pub struct HttpStream { inner: SendWrapper, read_future: Option>>, @@ -184,6 +186,7 @@ pub struct HttpStream { } impl HttpStream { + /// Create [`HttpStream`] with target uri and TLS backend. pub async fn new(uri: Uri, tls: TlsBackend) -> io::Result { Ok(Self { inner: SendWrapper::new(HttpStreamBufInner::new(uri, tls).await?), diff --git a/compio-tls/src/lib.rs b/compio-tls/src/lib.rs index 6622724b..6df4a662 100644 --- a/compio-tls/src/lib.rs +++ b/compio-tls/src/lib.rs @@ -1,6 +1,7 @@ //! Async TLS streams. #![warn(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] mod adapter; mod stream; From 1c574f3df5c0e7e11b25e2fe78b82fdbb68db526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Tue, 7 Nov 2023 21:04:32 +0800 Subject: [PATCH 21/44] feat(http): add server services --- compio-http/Cargo.toml | 5 +- compio-http/src/lib.rs | 5 ++ compio-http/src/server.rs | 120 +++++++++++++++++++++++++++++++++++++ compio-http/src/service.rs | 2 +- compio-http/src/stream.rs | 54 ++++++++++++++--- 5 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 compio-http/src/server.rs diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 7c785d6e..2ce0e429 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -30,4 +30,7 @@ default = ["native-tls"] native-tls = ["compio-tls/native-tls", "dep:native-tls"] rustls = ["compio-tls/rustls", "dep:rustls", "dep:rustls-native-certs"] vendored = ["native-tls?/vendored"] -all = ["native-tls", "rustls"] + +server = ["hyper/server"] + +all = ["native-tls", "rustls", "server"] diff --git a/compio-http/src/lib.rs b/compio-http/src/lib.rs index b501b104..e254ae47 100644 --- a/compio-http/src/lib.rs +++ b/compio-http/src/lib.rs @@ -9,5 +9,10 @@ pub use service::*; mod backend; pub use backend::*; +#[cfg(feature = "server")] +mod server; +#[cfg(feature = "server")] +pub use server::*; + mod stream; pub use stream::*; diff --git a/compio-http/src/server.rs b/compio-http/src/server.rs new file mode 100644 index 00000000..e6b7247f --- /dev/null +++ b/compio-http/src/server.rs @@ -0,0 +1,120 @@ +use std::{ + future::Future, + io, + net::SocketAddr, + pin::Pin, + task::{ready, Context, Poll}, +}; + +use compio_net::{TcpListener, TcpStream}; +use compio_tls::TlsStream; +use hyper::server::accept::Accept; + +use crate::HttpStream; + +type LocalPinBoxFuture = Pin>>; + +/// A TCP acceptor. +pub struct Acceptor { + listener: TcpListener, + fut: Option>>, +} + +impl Acceptor { + /// Create [`Acceptor`] binding to provided socket address. + pub async fn bind(addr: &SocketAddr) -> io::Result { + let listener = TcpListener::bind(addr).await?; + Ok(Self::from_listener(listener)) + } + + /// Create [`Acceptor`] from an existing [`compio_net::TcpListener`]. + pub fn from_listener(listener: TcpListener) -> Self { + Self { + listener, + fut: None, + } + } + + fn poll_accept_impl( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + let listener: &'static TcpListener = unsafe { &*(&self.listener as *const _) }; + if let Some(mut fut) = self.fut.take() { + match fut.as_mut().poll(cx) { + Poll::Pending => { + self.fut = Some(fut); + Poll::Pending + } + Poll::Ready(res) => Poll::Ready(res.map(|(s, _)| s)), + } + } else { + self.fut = Some(Box::pin(listener.accept())); + Poll::Pending + } + } +} + +impl Accept for Acceptor { + type Conn = HttpStream; + type Error = io::Error; + + fn poll_accept( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll>> { + let res = ready!(self.poll_accept_impl(cx)); + Poll::Ready(Some(res.map(HttpStream::from_tcp))) + } +} + +/// A TLS acceptor. +pub struct TlsAcceptor { + tcp_acceptor: Acceptor, + tls_acceptor: compio_tls::TlsAcceptor, + fut: Option>>>, +} + +impl TlsAcceptor { + /// Create [`TlsAcceptor`] from an existing [`compio_net::TcpListener`] and + /// [`compio_tls::TlsAcceptor`]. + pub fn from_listener(tcp_listener: TcpListener, tls_acceptor: compio_tls::TlsAcceptor) -> Self { + Self { + tcp_acceptor: Acceptor::from_listener(tcp_listener), + tls_acceptor, + fut: None, + } + } +} + +impl Accept for TlsAcceptor { + type Conn = HttpStream; + type Error = io::Error; + + fn poll_accept( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll>> { + let tcp_acceptor = Pin::new(&mut self.tcp_acceptor); + let res = ready!(tcp_acceptor.poll_accept_impl(cx)); + match res { + Ok(stream) => { + let acceptor: &'static compio_tls::TlsAcceptor = + unsafe { &*(&self.tls_acceptor as *const _) }; + if let Some(mut fut) = self.fut.take() { + match fut.as_mut().poll(cx) { + Poll::Pending => { + self.fut = Some(fut); + Poll::Pending + } + Poll::Ready(res) => Poll::Ready(Some(res.map(HttpStream::from_tls))), + } + } else { + self.fut = Some(Box::pin(acceptor.accept(stream))); + Poll::Pending + } + } + Err(e) => Poll::Ready(Some(Err(e))), + } + } +} diff --git a/compio-http/src/service.rs b/compio-http/src/service.rs index 23f3bbc2..71fa6e74 100644 --- a/compio-http/src/service.rs +++ b/compio-http/src/service.rs @@ -47,6 +47,6 @@ impl Service for Connector { } fn call(&mut self, req: Uri) -> Self::Future { - Box::pin(SendWrapper::new(HttpStream::new(req, self.tls))) + Box::pin(SendWrapper::new(HttpStream::connect(req, self.tls))) } } diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index ca7a33bf..8392d560 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -27,7 +27,7 @@ enum HttpStreamInner { } impl HttpStreamInner { - pub async fn new(uri: Uri, tls: TlsBackend) -> io::Result { + pub async fn connect(uri: Uri, tls: TlsBackend) -> io::Result { let scheme = uri.scheme_str().unwrap_or("http"); let host = uri.host().expect("there should be host"); let port = uri.port_u16(); @@ -47,6 +47,14 @@ impl HttpStreamInner { )), } } + + pub fn from_tcp(s: TcpStream) -> Self { + Self::Tcp(s) + } + + pub fn from_tls(s: TlsStream) -> Self { + Self::Tls(s) + } } impl AsyncRead for HttpStreamInner { @@ -104,12 +112,24 @@ struct HttpStreamBufInner { } impl HttpStreamBufInner { - pub async fn new(uri: Uri, tls: TlsBackend) -> io::Result { - Ok(Self { - inner: HttpStreamInner::new(uri, tls).await?, + pub async fn connect(uri: Uri, tls: TlsBackend) -> io::Result { + Ok(Self::from_inner(HttpStreamInner::connect(uri, tls).await?)) + } + + pub fn from_tcp(s: TcpStream) -> Self { + Self::from_inner(HttpStreamInner::from_tcp(s)) + } + + pub fn from_tls(s: TlsStream) -> Self { + Self::from_inner(HttpStreamInner::from_tls(s)) + } + + fn from_inner(s: HttpStreamInner) -> Self { + Self { + inner: s, read_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), write_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), - }) + } } pub async fn fill_read_buf(&mut self) -> io::Result<()> { @@ -187,14 +207,30 @@ pub struct HttpStream { impl HttpStream { /// Create [`HttpStream`] with target uri and TLS backend. - pub async fn new(uri: Uri, tls: TlsBackend) -> io::Result { - Ok(Self { - inner: SendWrapper::new(HttpStreamBufInner::new(uri, tls).await?), + pub async fn connect(uri: Uri, tls: TlsBackend) -> io::Result { + Ok(Self::from_inner( + HttpStreamBufInner::connect(uri, tls).await?, + )) + } + + /// Create [`HttpStream`] with connected TCP stream. + pub fn from_tcp(s: TcpStream) -> Self { + Self::from_inner(HttpStreamBufInner::from_tcp(s)) + } + + /// Create [`HttpStream`] with connected TLS stream. + pub fn from_tls(s: TlsStream) -> Self { + Self::from_inner(HttpStreamBufInner::from_tls(s)) + } + + fn from_inner(s: HttpStreamBufInner) -> Self { + Self { + inner: SendWrapper::new(s), read_future: None, write_future: None, flush_future: None, shutdown_future: None, - }) + } } } From 9972f4de02ac6ceca94e8d7b356d915ddc68d488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Tue, 7 Nov 2023 21:54:11 +0800 Subject: [PATCH 22/44] test: use http server service --- Cargo.toml | 1 + compio-http-client/Cargo.toml | 6 +- compio-http-client/tests/client.rs | 6 +- compio-http-client/tests/server/mod.rs | 88 ++++++++------------------ compio-net/Cargo.toml | 2 +- 5 files changed, 37 insertions(+), 66 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7e18d012..038d13b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ compio-http = { path = "./compio-http", version = "0.1.0" } cfg-if = "1.0.0" crossbeam-channel = "0.5.8" crossbeam-queue = "0.3.8" +futures-channel = "0.3.29" futures-util = "0.3.29" libc = "0.2.149" nix = "0.27.1" diff --git a/compio-http-client/Cargo.toml b/compio-http-client/Cargo.toml index 7d6a354d..3e867c68 100644 --- a/compio-http-client/Cargo.toml +++ b/compio-http-client/Cargo.toml @@ -28,10 +28,12 @@ thiserror = "1" url = "2" [dev-dependencies] +compio-net = { workspace = true, features = ["runtime"] } compio-macros = { workspace = true } +compio-http = { workspace = true, features = ["server"] } -hyper = { version = "0.14", features = ["server", "tcp"] } -tokio = { workspace = true, features = ["rt", "sync"] } +futures-channel = { workspace = true } +hyper = { workspace = true, features = ["server"] } [features] default = ["native-tls"] diff --git a/compio-http-client/tests/client.rs b/compio-http-client/tests/client.rs index 731e3d3e..bdb0501a 100644 --- a/compio-http-client/tests/client.rs +++ b/compio-http-client/tests/client.rs @@ -9,7 +9,7 @@ use http::header::CONTENT_TYPE; #[compio_macros::test] async fn response_text() { - let server = server::http(move |_req| async { http::Response::new("Hello".into()) }); + let server = server::http(move |_req| async { http::Response::new("Hello".into()) }).await; let client = Client::new(); @@ -25,7 +25,7 @@ async fn response_text() { #[compio_macros::test] async fn response_bytes() { - let server = server::http(move |_req| async { http::Response::new("Hello".into()) }); + let server = server::http(move |_req| async { http::Response::new("Hello".into()) }).await; let client = Client::new(); @@ -42,7 +42,7 @@ async fn response_bytes() { #[compio_macros::test] #[cfg(feature = "json")] async fn response_json() { - let server = server::http(move |_req| async { http::Response::new("\"Hello\"".into()) }); + let server = server::http(move |_req| async { http::Response::new("\"Hello\"".into()) }).await; let client = Client::new(); diff --git a/compio-http-client/tests/server/mod.rs b/compio-http-client/tests/server/mod.rs index 3609fb24..7cf7bb2c 100644 --- a/compio-http-client/tests/server/mod.rs +++ b/compio-http-client/tests/server/mod.rs @@ -1,22 +1,20 @@ use std::{ convert::Infallible, future::Future, - net::{self, Ipv4Addr}, - sync::mpsc as std_mpsc, - thread, - time::Duration, + net::{Ipv4Addr, SocketAddr}, }; -use tokio::{runtime, sync::oneshot}; +use compio_http::Acceptor; +use compio_net::TcpListener; +use futures_channel::oneshot; pub struct Server { - addr: net::SocketAddr, - panic_rx: std_mpsc::Receiver<()>, + addr: SocketAddr, shutdown_tx: Option>, } impl Server { - pub fn addr(&self) -> net::SocketAddr { + pub fn addr(&self) -> SocketAddr { self.addr } } @@ -26,66 +24,36 @@ impl Drop for Server { if let Some(tx) = self.shutdown_tx.take() { let _ = tx.send(()); } - - if !::std::thread::panicking() { - self.panic_rx - .recv_timeout(Duration::from_secs(3)) - .expect("test server should not panic"); - } } } -pub fn http(func: F) -> Server +pub async fn http(func: F) -> Server where F: Fn(http::Request) -> Fut + Clone + Send + 'static, Fut: Future> + Send + 'static, { - // Spawn new runtime in thread to prevent reactor execution context conflict - thread::spawn(move || { - let rt = runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("new rt"); - #[allow(clippy::async_yields_async)] - let srv = rt.block_on(async move { - hyper::Server::bind(&(Ipv4Addr::LOCALHOST, 0).into()).serve( - hyper::service::make_service_fn(move |_| { - let func = func.clone(); - async move { - Ok::<_, Infallible>(hyper::service::service_fn(move |req| { - let fut = func(req); - async move { Ok::<_, Infallible>(fut.await) } - })) - } - }), - ) - }); + let listener = TcpListener::bind(&(Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let addr = listener.local_addr().unwrap(); + let acceptor = Acceptor::from_listener(listener); + let srv = hyper::Server::builder(acceptor).serve(hyper::service::make_service_fn(move |_| { + let func = func.clone(); + async move { + Ok::<_, Infallible>(hyper::service::service_fn(move |req| { + let fut = func(req); + async move { Ok::<_, Infallible>(fut.await) } + })) + } + })); - let addr = srv.local_addr(); - let (shutdown_tx, shutdown_rx) = oneshot::channel(); - let srv = srv.with_graceful_shutdown(async move { - let _ = shutdown_rx.await; - }); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let srv = srv.with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }); - let (panic_tx, panic_rx) = std_mpsc::channel(); - let tname = format!( - "test({})-support-server", - thread::current().name().unwrap_or("") - ); - thread::Builder::new() - .name(tname) - .spawn(move || { - rt.block_on(srv).unwrap(); - let _ = panic_tx.send(()); - }) - .expect("thread spawn"); + compio_runtime::spawn(srv).detach(); - Server { - addr, - panic_rx, - shutdown_tx: Some(shutdown_tx), - } - }) - .join() - .unwrap() + Server { + addr, + shutdown_tx: Some(shutdown_tx), + } } diff --git a/compio-net/Cargo.toml b/compio-net/Cargo.toml index b027dd48..dc2dd137 100644 --- a/compio-net/Cargo.toml +++ b/compio-net/Cargo.toml @@ -39,7 +39,7 @@ libc = { workspace = true } # Shared dev dependencies for all platforms [dev-dependencies] compio-macros = { workspace = true } -futures-channel = "0.3.29" +futures-channel = { workspace = true } futures-util = { workspace = true } tempfile = { workspace = true } From cc7f666ea2b6090abc2f3f6a66e7faa114c5743f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Tue, 7 Nov 2023 22:33:19 +0800 Subject: [PATCH 23/44] fix: impl executor --- compio-http-client/src/request.rs | 8 +++--- compio-http-client/src/response.rs | 10 +++---- compio-http-client/tests/server/mod.rs | 22 +++++++++------- compio-http/src/server.rs | 36 ++++++++++++++------------ compio-http/src/service.rs | 6 ++--- 5 files changed, 43 insertions(+), 39 deletions(-) diff --git a/compio-http-client/src/request.rs b/compio-http-client/src/request.rs index e737d2e7..8d8c4472 100644 --- a/compio-http-client/src/request.rs +++ b/compio-http-client/src/request.rs @@ -209,10 +209,10 @@ impl RequestBuilder { /// Enable HTTP basic authentication. /// /// ```rust - /// # use compio_http::Error; + /// # use compio_http_client::Error; /// /// # async fn run() -> Result<(), Error> { - /// let client = compio_http::Client::new(); + /// let client = compio_http_client::Client::new(); /// let resp = client /// .delete("http://httpbin.org/delete") /// .basic_auth("admin", Some("good password")) @@ -311,14 +311,14 @@ impl RequestBuilder { /// header. /// /// ```rust - /// # use compio_http::Error; + /// # use compio_http_client::Error; /// # use std::collections::HashMap; /// # /// # async fn run() -> Result<(), Error> { /// let mut params = HashMap::new(); /// params.insert("lang", "rust"); /// - /// let client = compio_http::Client::new(); + /// let client = compio_http_client::Client::new(); /// let res = client /// .post("http://httpbin.org") /// .form(¶ms) diff --git a/compio-http-client/src/response.rs b/compio-http-client/src/response.rs index f8aa1224..c1ab11a9 100644 --- a/compio-http-client/src/response.rs +++ b/compio-http-client/src/response.rs @@ -87,7 +87,7 @@ impl Response { /// /// ``` /// # async fn run() -> Result<(), Box> { - /// let client = compio_http::Client::new(); + /// let client = compio_http_client::Client::new(); /// let content = client /// .get("http://httpbin.org/range/26") /// .send() @@ -120,7 +120,7 @@ impl Response { /// /// ``` /// # async fn run() -> Result<(), Box> { - /// let client = compio_http::Client::new(); + /// let client = compio_http_client::Client::new(); /// let content = client /// .get("http://httpbin.org/range/26") /// .send() @@ -162,7 +162,7 @@ impl Response { /// # extern crate compio_http; /// # extern crate serde; /// # - /// # use compio_http::Error; + /// # use compio_http_client::Error; /// # use serde::Deserialize; /// # /// // This `derive` requires the `serde` dependency. @@ -172,7 +172,7 @@ impl Response { /// } /// /// # async fn run() -> Result<(), Error> { - /// let client = compio_http::Client::new(); + /// let client = compio_http_client::Client::new(); /// let ip = client /// .get("http://httpbin.org/ip") /// .send() @@ -207,7 +207,7 @@ impl Response { /// /// ``` /// # async fn run() -> Result<(), Box> { - /// let client = compio_http::Client::new(); + /// let client = compio_http_client::Client::new(); /// let bytes = client /// .get("http://httpbin.org/ip") /// .send() diff --git a/compio-http-client/tests/server/mod.rs b/compio-http-client/tests/server/mod.rs index 7cf7bb2c..f55368fd 100644 --- a/compio-http-client/tests/server/mod.rs +++ b/compio-http-client/tests/server/mod.rs @@ -4,7 +4,7 @@ use std::{ net::{Ipv4Addr, SocketAddr}, }; -use compio_http::Acceptor; +use compio_http::{Acceptor, CompioExecutor}; use compio_net::TcpListener; use futures_channel::oneshot; @@ -35,15 +35,17 @@ where let listener = TcpListener::bind(&(Ipv4Addr::LOCALHOST, 0)).await.unwrap(); let addr = listener.local_addr().unwrap(); let acceptor = Acceptor::from_listener(listener); - let srv = hyper::Server::builder(acceptor).serve(hyper::service::make_service_fn(move |_| { - let func = func.clone(); - async move { - Ok::<_, Infallible>(hyper::service::service_fn(move |req| { - let fut = func(req); - async move { Ok::<_, Infallible>(fut.await) } - })) - } - })); + let srv = hyper::Server::builder(acceptor) + .executor(CompioExecutor) + .serve(hyper::service::make_service_fn(move |_| { + let func = func.clone(); + async move { + Ok::<_, Infallible>(hyper::service::service_fn(move |req| { + let fut = func(req); + async move { Ok::<_, Infallible>(fut.await) } + })) + } + })); let (shutdown_tx, shutdown_rx) = oneshot::channel(); let srv = srv.with_graceful_shutdown(async move { diff --git a/compio-http/src/server.rs b/compio-http/src/server.rs index e6b7247f..f2b876bf 100644 --- a/compio-http/src/server.rs +++ b/compio-http/src/server.rs @@ -50,6 +50,7 @@ impl Acceptor { } } else { self.fut = Some(Box::pin(listener.accept())); + cx.waker().wake_by_ref(); Poll::Pending } } @@ -95,26 +96,27 @@ impl Accept for TlsAcceptor { mut self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll>> { - let tcp_acceptor = Pin::new(&mut self.tcp_acceptor); - let res = ready!(tcp_acceptor.poll_accept_impl(cx)); - match res { - Ok(stream) => { - let acceptor: &'static compio_tls::TlsAcceptor = - unsafe { &*(&self.tls_acceptor as *const _) }; - if let Some(mut fut) = self.fut.take() { - match fut.as_mut().poll(cx) { - Poll::Pending => { - self.fut = Some(fut); - Poll::Pending - } - Poll::Ready(res) => Poll::Ready(Some(res.map(HttpStream::from_tls))), - } - } else { - self.fut = Some(Box::pin(acceptor.accept(stream))); + let acceptor: &'static compio_tls::TlsAcceptor = + unsafe { &*(&self.tls_acceptor as *const _) }; + if let Some(mut fut) = self.fut.take() { + match fut.as_mut().poll(cx) { + Poll::Pending => { + self.fut = Some(fut); Poll::Pending } + Poll::Ready(res) => Poll::Ready(Some(res.map(HttpStream::from_tls))), } - Err(e) => Poll::Ready(Some(Err(e))), + } else { + let tcp_acceptor = Pin::new(&mut self.tcp_acceptor); + let res = ready!(tcp_acceptor.poll_accept_impl(cx)); + match res { + Ok(stream) => { + self.fut = Some(Box::pin(acceptor.accept(stream))); + } + Err(e) => return Poll::Ready(Some(Err(e))), + } + cx.waker().wake_by_ref(); + Poll::Pending } } } diff --git a/compio-http/src/service.rs b/compio-http/src/service.rs index 71fa6e74..17484ebc 100644 --- a/compio-http/src/service.rs +++ b/compio-http/src/service.rs @@ -15,9 +15,9 @@ use crate::{HttpStream, TlsBackend}; #[derive(Debug, Default, Clone)] pub struct CompioExecutor; -impl Executor + Send>>> for CompioExecutor { - fn execute(&self, fut: Pin + Send>>) { - compio_runtime::spawn(fut).detach() +impl + Send + 'static> Executor for CompioExecutor { + fn execute(&self, fut: F) { + compio_runtime::spawn(fut).detach(); } } From 03d8059abfbfcc0ab5f9123cc1fe75811afba422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Tue, 7 Nov 2023 22:49:59 +0800 Subject: [PATCH 24/44] feat: client feature for compio-http --- compio-http-client/Cargo.toml | 2 +- compio-http/Cargo.toml | 3 ++- compio-http/src/stream.rs | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/compio-http-client/Cargo.toml b/compio-http-client/Cargo.toml index 3e867c68..c1a28a59 100644 --- a/compio-http-client/Cargo.toml +++ b/compio-http-client/Cargo.toml @@ -13,7 +13,7 @@ repository = { workspace = true } compio-buf = { workspace = true, features = ["bytes"] } compio-runtime = { workspace = true, features = ["time"] } compio-tls = { workspace = true } -compio-http = { workspace = true } +compio-http = { workspace = true, features = ["client"] } http = "0.2" hyper = { workspace = true, features = ["client", "http1"] } diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 2ce0e429..39da5676 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -31,6 +31,7 @@ native-tls = ["compio-tls/native-tls", "dep:native-tls"] rustls = ["compio-tls/rustls", "dep:rustls", "dep:rustls-native-certs"] vendored = ["native-tls?/vendored"] +client = ["hyper/client"] server = ["hyper/server"] -all = ["native-tls", "rustls", "server"] +all = ["native-tls", "rustls", "client", "server"] diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index 8392d560..cb0aa9af 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -12,10 +12,9 @@ use compio_buf::{ use compio_io::{AsyncRead, AsyncWrite, AsyncWriteExt, Buffer}; use compio_net::TcpStream; use compio_tls::TlsStream; -use hyper::{ - client::connect::{Connected, Connection}, - Uri, -}; +#[cfg(feature = "client")] +use hyper::client::connect::{Connected, Connection}; +use hyper::Uri; use send_wrapper::SendWrapper; use crate::TlsBackend; @@ -296,6 +295,7 @@ impl tokio::io::AsyncWrite for HttpStream { } } +#[cfg(feature = "client")] impl Connection for HttpStream { fn connected(&self) -> Connected { Connected::new() From 189b1ddaafd602473a7b622a379eef6cb8bc436b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Wed, 8 Nov 2023 19:51:51 +0800 Subject: [PATCH 25/44] fix: use shorter syntax in Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 038d13b0..63a6317b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ nix = "0.27.1" once_cell = "1.18.0" os_pipe = "1.1.4" paste = "1.0.14" -send_wrapper = { version = "0.6" } +send_wrapper = "0.6" slab = "0.4.9" socket2 = "0.5.5" tempfile = "3.8.1" From d2062594431c9c8cebffc92f09eaf2325ad93fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Fri, 10 Nov 2023 17:58:07 +0800 Subject: [PATCH 26/44] feat(io): add SyncStream wrapper Replace the different impls with this wrapper. --- compio-http/Cargo.toml | 2 +- compio-http/src/server.rs | 6 +- compio-http/src/stream.rs | 268 ++++++++++-------- compio-io/Cargo.toml | 3 + .../src/wrapper.rs => compio-io/src/compat.rs | 34 ++- compio-io/src/lib.rs | 3 +- compio-tls/Cargo.toml | 2 +- compio-tls/src/adapter/mod.rs | 12 +- compio-tls/src/adapter/rtls.rs | 18 +- compio-tls/src/lib.rs | 2 - compio-tls/src/stream/mod.rs | 18 +- 11 files changed, 203 insertions(+), 165 deletions(-) rename compio-tls/src/wrapper.rs => compio-io/src/compat.rs (74%) diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 39da5676..84fc0e67 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -11,7 +11,7 @@ repository = { workspace = true } [dependencies] compio-buf = { workspace = true } -compio-io = { workspace = true } +compio-io = { workspace = true, features = ["compat"] } compio-runtime = { workspace = true } compio-net = { workspace = true, features = ["runtime"] } compio-tls = { workspace = true } diff --git a/compio-http/src/server.rs b/compio-http/src/server.rs index f2b876bf..d8ba355d 100644 --- a/compio-http/src/server.rs +++ b/compio-http/src/server.rs @@ -112,11 +112,11 @@ impl Accept for TlsAcceptor { match res { Ok(stream) => { self.fut = Some(Box::pin(acceptor.accept(stream))); + cx.waker().wake_by_ref(); + Poll::Pending } - Err(e) => return Poll::Ready(Some(Err(e))), + Err(e) => Poll::Ready(Some(Err(e))), } - cx.waker().wake_by_ref(); - Poll::Pending } } } diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index cb0aa9af..bba8be72 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -6,10 +6,8 @@ use std::{ task::{Context, Poll}, }; -use compio_buf::{ - BufResult, IntoInner, IoBuf, IoBufMut, IoVectoredBuf, IoVectoredBufMut, SetBufInit, -}; -use compio_io::{AsyncRead, AsyncWrite, AsyncWriteExt, Buffer}; +use compio_buf::{BufResult, IoBuf, IoBufMut, IoVectoredBuf, IoVectoredBufMut}; +use compio_io::{compat::SyncStream, AsyncRead, AsyncWrite}; use compio_net::TcpStream; use compio_tls::TlsStream; #[cfg(feature = "client")] @@ -102,132 +100,128 @@ impl AsyncWrite for HttpStreamInner { } } -const DEFAULT_BUF_SIZE: usize = 8 * 1024; - -struct HttpStreamBufInner { - inner: HttpStreamInner, - read_buffer: Buffer, - write_buffer: Buffer, -} - -impl HttpStreamBufInner { - pub async fn connect(uri: Uri, tls: TlsBackend) -> io::Result { - Ok(Self::from_inner(HttpStreamInner::connect(uri, tls).await?)) - } - - pub fn from_tcp(s: TcpStream) -> Self { - Self::from_inner(HttpStreamInner::from_tcp(s)) - } - - pub fn from_tls(s: TlsStream) -> Self { - Self::from_inner(HttpStreamInner::from_tls(s)) - } - - fn from_inner(s: HttpStreamInner) -> Self { - Self { - inner: s, - read_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), - write_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), - } - } - - pub async fn fill_read_buf(&mut self) -> io::Result<()> { - if self.read_buffer.all_done() { - self.read_buffer.reset(); - } - if self.read_buffer.slice().is_empty() { - self.read_buffer - .with(|b| async { - let len = b.buf_len(); - let slice = b.slice(len..); - self.inner.read(slice).await.into_inner() - }) - .await?; - } - - Ok(()) - } - - pub fn read_buf_slice(&self) -> &[u8] { - self.read_buffer.slice() - } - - pub fn consume_read_buf(&mut self, amt: usize) { - self.read_buffer.advance(amt); - } - - pub async fn flush_write_buf_if_needed(&mut self) -> io::Result<()> { - if self.write_buffer.need_flush() { - self.flush_write_buf().await?; - } - Ok(()) - } - - pub fn write_slice(&mut self, buf: &[u8]) -> io::Result { - self.write_buffer.with_sync(|mut inner| { - let len = buf.len().min(inner.buf_capacity() - inner.buf_len()); - unsafe { - std::ptr::copy_nonoverlapping( - buf.as_ptr(), - inner.as_buf_mut_ptr().add(inner.buf_len()), - len, - ); - inner.set_buf_init(inner.buf_len() + len); - } - BufResult(Ok(len), inner) - }) - } - - pub async fn flush_write_buf(&mut self) -> io::Result<()> { - if !self.write_buffer.is_empty() { - self.write_buffer.with(|b| self.inner.write_all(b)).await?; - self.write_buffer.reset(); - } - self.inner.flush().await?; - Ok(()) - } - - pub async fn shutdown(&mut self) -> io::Result<()> { - self.inner.shutdown().await - } -} +// const DEFAULT_BUF_SIZE: usize = 8 * 1024; + +// struct HttpStreamBufInner { +// inner: HttpStreamInner, +// read_buffer: Buffer, +// write_buffer: Buffer, +// } + +// impl HttpStreamBufInner { +// pub async fn connect(uri: Uri, tls: TlsBackend) -> io::Result { +// Ok(Self::from_inner(HttpStreamInner::connect(uri, tls).await?)) +// } + +// pub fn from_tcp(s: TcpStream) -> Self { +// Self::from_inner(HttpStreamInner::from_tcp(s)) +// } + +// pub fn from_tls(s: TlsStream) -> Self { +// Self::from_inner(HttpStreamInner::from_tls(s)) +// } + +// fn from_inner(s: HttpStreamInner) -> Self { +// Self { +// inner: s, +// read_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), +// write_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), +// } +// } + +// pub async fn fill_read_buf(&mut self) -> io::Result<()> { +// if self.read_buffer.all_done() { +// self.read_buffer.reset(); +// } +// if self.read_buffer.slice().is_empty() { +// self.read_buffer +// .with(|b| async { +// let len = b.buf_len(); +// let slice = b.slice(len..); +// self.inner.read(slice).await.into_inner() +// }) +// .await?; +// } + +// Ok(()) +// } + +// pub fn read_buf_slice(&self) -> &[u8] { +// self.read_buffer.slice() +// } + +// pub fn consume_read_buf(&mut self, amt: usize) { +// self.read_buffer.advance(amt); +// } + +// pub async fn flush_write_buf_if_needed(&mut self) -> io::Result<()> { +// if self.write_buffer.need_flush() { +// self.flush_write_buf().await?; +// } +// Ok(()) +// } + +// pub fn write_slice(&mut self, buf: &[u8]) -> io::Result { +// self.write_buffer.with_sync(|mut inner| { +// let len = buf.len().min(inner.buf_capacity() - inner.buf_len()); +// unsafe { +// std::ptr::copy_nonoverlapping( +// buf.as_ptr(), +// inner.as_buf_mut_ptr().add(inner.buf_len()), +// len, +// ); +// inner.set_buf_init(inner.buf_len() + len); +// } +// BufResult(Ok(len), inner) +// }) +// } + +// pub async fn flush_write_buf(&mut self) -> io::Result<()> { +// if !self.write_buffer.is_empty() { +// self.write_buffer.with(|b| self.inner.write_all(b)).await?; +// self.write_buffer.reset(); +// } +// self.inner.flush().await?; +// Ok(()) +// } + +// pub async fn shutdown(&mut self) -> io::Result<()> { +// self.inner.shutdown().await +// } +// } type PinBoxFuture = Pin + Send>>; /// A HTTP stream wrapper, based on compio, and exposes [`tokio::io`] /// interfaces. pub struct HttpStream { - inner: SendWrapper, - read_future: Option>>, - write_future: Option>>, - flush_future: Option>>, + inner: SendWrapper>, + read_future: Option>>, + write_future: Option>>, shutdown_future: Option>>, } impl HttpStream { /// Create [`HttpStream`] with target uri and TLS backend. pub async fn connect(uri: Uri, tls: TlsBackend) -> io::Result { - Ok(Self::from_inner( - HttpStreamBufInner::connect(uri, tls).await?, - )) + Ok(Self::from_inner(HttpStreamInner::connect(uri, tls).await?)) } /// Create [`HttpStream`] with connected TCP stream. pub fn from_tcp(s: TcpStream) -> Self { - Self::from_inner(HttpStreamBufInner::from_tcp(s)) + Self::from_inner(HttpStreamInner::from_tcp(s)) } /// Create [`HttpStream`] with connected TLS stream. pub fn from_tls(s: TlsStream) -> Self { - Self::from_inner(HttpStreamBufInner::from_tls(s)) + Self::from_inner(HttpStreamInner::from_tls(s)) } - fn from_inner(s: HttpStreamBufInner) -> Self { + fn from_inner(s: HttpStreamInner) -> Self { Self { - inner: SendWrapper::new(s), + inner: SendWrapper::new(SyncStream::new(s)), read_future: None, write_future: None, - flush_future: None, shutdown_future: None, } } @@ -250,20 +244,48 @@ macro_rules! poll_future { }}; } +macro_rules! poll_future_would_block { + ($f:expr, $cx:expr, $e:expr, $io:expr) => {{ + if let Some(mut f) = $f.take() { + if f.as_mut().poll($cx).is_pending() { + $f = Some(f); + return Poll::Pending; + } + } + + match $io { + Ok(len) => Poll::Ready(Ok(len)), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + $f = Some(Box::pin(SendWrapper::new($e))); + $cx.waker().wake_by_ref(); + Poll::Pending + } + Err(e) => Poll::Ready(Err(e)), + } + }}; +} + impl tokio::io::AsyncRead for HttpStream { fn poll_read( mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut tokio::io::ReadBuf<'_>, ) -> Poll> { - let inner: &'static mut HttpStreamBufInner = + let inner: &'static mut SyncStream = unsafe { &mut *(self.inner.deref_mut() as *mut _) }; - poll_future!(self.read_future, cx, inner.fill_read_buf())?; - let slice = self.inner.read_buf_slice(); - let len = slice.len().min(buf.remaining()); - buf.put_slice(&slice[..len]); - self.inner.consume_read_buf(len); - Poll::Ready(Ok(())) + + let res = poll_future_would_block!(self.read_future, cx, inner.fill_read_buf(), { + let slice = buf.initialize_unfilled(); + io::Read::read(inner, slice) + }); + match res { + Poll::Ready(Ok(len)) => { + buf.advance(len); + Poll::Ready(Ok(())) + } + Poll::Ready(Err(e)) => Poll::Ready(Err(e)), + Poll::Pending => Poll::Pending, + } } } @@ -273,24 +295,28 @@ impl tokio::io::AsyncWrite for HttpStream { cx: &mut Context<'_>, buf: &[u8], ) -> Poll> { - let inner: &'static mut HttpStreamBufInner = + let inner: &'static mut SyncStream = unsafe { &mut *(self.inner.deref_mut() as *mut _) }; - poll_future!(self.write_future, cx, inner.flush_write_buf_if_needed())?; - let res = self.inner.write_slice(buf); - Poll::Ready(res) + + poll_future_would_block!( + self.write_future, + cx, + inner.flush_write_buf(), + io::Write::write(inner, buf) + ) } fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let inner: &'static mut HttpStreamBufInner = + let inner: &'static mut SyncStream = unsafe { &mut *(self.inner.deref_mut() as *mut _) }; - let res = poll_future!(self.flush_future, cx, inner.flush_write_buf()); - Poll::Ready(res) + let res = poll_future!(self.write_future, cx, inner.flush_write_buf()); + Poll::Ready(res.map(|_| ())) } fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let inner: &'static mut HttpStreamBufInner = + let inner: &'static mut SyncStream = unsafe { &mut *(self.inner.deref_mut() as *mut _) }; - let res = poll_future!(self.shutdown_future, cx, inner.shutdown()); + let res = poll_future!(self.shutdown_future, cx, inner.get_mut().shutdown()); Poll::Ready(res) } } diff --git a/compio-io/Cargo.toml b/compio-io/Cargo.toml index fdc8b8c2..7d25b51c 100644 --- a/compio-io/Cargo.toml +++ b/compio-io/Cargo.toml @@ -19,6 +19,9 @@ compio-runtime = { workspace = true } tokio = { workspace = true, features = ["macros", "rt"] } [features] +default = [] +compat = [] + # Nightly features allocator_api = [] nightly = ["allocator_api", "compio-buf/allocator_api"] diff --git a/compio-tls/src/wrapper.rs b/compio-io/src/compat.rs similarity index 74% rename from compio-tls/src/wrapper.rs rename to compio-io/src/compat.rs index 63ec6ad1..0b3dd47c 100644 --- a/compio-tls/src/wrapper.rs +++ b/compio-io/src/compat.rs @@ -1,24 +1,31 @@ +//! Compat wrappers for interop with other crates. + use std::io::{self, BufRead, Read, Write}; use compio_buf::{BufResult, IntoInner, IoBuf, IoBufMut, SetBufInit}; -use compio_io::{AsyncWriteExt, Buffer}; -const DEFAULT_BUF_SIZE: usize = 8 * 1024; +use crate::{buffer::Buffer, util::DEFAULT_BUF_SIZE, AsyncWriteExt}; +/// A wrapper for [`AsyncRead`](crate::AsyncRead) + +/// [`AsyncWrite`](crate::AsyncWrite), providing sync traits impl. The sync +/// methods will return [`io::ErrorKind::WouldBlock`] error if the inner buffer +/// needs more data. #[derive(Debug)] -pub struct StreamWrapper { +pub struct SyncStream { stream: S, eof: bool, read_buffer: Buffer, write_buffer: Buffer, } -impl StreamWrapper { +impl SyncStream { + /// Create [`SyncStream`] with the stream and default buffer size. pub fn new(stream: S) -> Self { - Self::with_capacity(stream, DEFAULT_BUF_SIZE) + Self::with_capacity(DEFAULT_BUF_SIZE, stream) } - pub fn with_capacity(stream: S, cap: usize) -> Self { + /// Create [`SyncStream`] with the stream and buffer size. + pub fn with_capacity(cap: usize, stream: S) -> Self { Self { stream, eof: false, @@ -27,14 +34,17 @@ impl StreamWrapper { } } + /// Get if the stream is at EOF. pub fn is_eof(&self) -> bool { self.eof } + /// Get the reference of the inner stream. pub fn get_ref(&self) -> &S { &self.stream } + /// Get the mutable reference of the inner stream. pub fn get_mut(&mut self) -> &mut S { &mut self.stream } @@ -48,7 +58,7 @@ impl StreamWrapper { } } -impl Read for StreamWrapper { +impl Read for SyncStream { fn read(&mut self, buf: &mut [u8]) -> io::Result { let mut slice = self.fill_buf()?; slice.read(buf).map(|res| { @@ -58,7 +68,7 @@ impl Read for StreamWrapper { } } -impl BufRead for StreamWrapper { +impl BufRead for SyncStream { fn fill_buf(&mut self) -> io::Result<&[u8]> { if self.read_buffer.all_done() { self.read_buffer.reset(); @@ -76,7 +86,7 @@ impl BufRead for StreamWrapper { } } -impl Write for StreamWrapper { +impl Write for SyncStream { fn write(&mut self, buf: &[u8]) -> io::Result { if self.write_buffer.need_flush() { self.flush_impl()?; @@ -110,7 +120,8 @@ fn would_block(msg: &str) -> io::Error { io::Error::new(io::ErrorKind::WouldBlock, msg) } -impl StreamWrapper { +impl SyncStream { + /// Fill the read buffer. pub async fn fill_read_buf(&mut self) -> io::Result { let stream = &mut self.stream; let len = self @@ -128,7 +139,8 @@ impl StreamWrapper { } } -impl StreamWrapper { +impl SyncStream { + /// Flush all data in the write buffer. pub async fn flush_write_buf(&mut self) -> io::Result { let stream = &mut self.stream; let len = self.write_buffer.with(|b| stream.write_all(b)).await?; diff --git a/compio-io/src/lib.rs b/compio-io/src/lib.rs index a1ea0aa3..9b59071d 100644 --- a/compio-io/src/lib.rs +++ b/compio-io/src/lib.rs @@ -102,13 +102,14 @@ #![cfg_attr(feature = "allocator_api", feature(allocator_api))] mod buffer; +#[cfg(feature = "compat")] +pub mod compat; mod read; pub mod util; mod write; pub(crate) type IoResult = std::io::Result; -pub use buffer::Buffer; pub use read::*; pub use util::{copy, null, repeat}; pub use write::*; diff --git a/compio-tls/Cargo.toml b/compio-tls/Cargo.toml index c781f0d7..7e08d8a6 100644 --- a/compio-tls/Cargo.toml +++ b/compio-tls/Cargo.toml @@ -11,7 +11,7 @@ repository = { workspace = true } [dependencies] compio-buf = { workspace = true } -compio-io = { workspace = true } +compio-io = { workspace = true, features = ["compat"] } native-tls = { workspace = true, optional = true } rustls = { workspace = true, optional = true } diff --git a/compio-tls/src/adapter/mod.rs b/compio-tls/src/adapter/mod.rs index 6912144a..84dc2643 100644 --- a/compio-tls/src/adapter/mod.rs +++ b/compio-tls/src/adapter/mod.rs @@ -1,8 +1,8 @@ use std::io; -use compio_io::{AsyncRead, AsyncWrite}; +use compio_io::{compat::SyncStream, AsyncRead, AsyncWrite}; -use crate::{wrapper::StreamWrapper, TlsStream}; +use crate::TlsStream; #[cfg(feature = "rustls")] mod rtls; @@ -55,7 +55,7 @@ impl TlsConnector { match &self.0 { #[cfg(feature = "native-tls")] TlsConnectorInner::NativeTls(c) => { - handshake_native_tls(c.connect(domain, StreamWrapper::new(stream))).await + handshake_native_tls(c.connect(domain, SyncStream::new(stream))).await } #[cfg(feature = "rustls")] TlsConnectorInner::Rustls(c) => handshake_rustls(c.connect(domain, stream)).await, @@ -105,7 +105,7 @@ impl TlsAcceptor { match &self.0 { #[cfg(feature = "native-tls")] TlsAcceptorInner::NativeTls(c) => { - handshake_native_tls(c.accept(StreamWrapper::new(stream))).await + handshake_native_tls(c.accept(SyncStream::new(stream))).await } #[cfg(feature = "rustls")] TlsAcceptorInner::Rustls(c) => handshake_rustls(c.accept(stream)).await, @@ -116,8 +116,8 @@ impl TlsAcceptor { #[cfg(feature = "native-tls")] async fn handshake_native_tls( mut res: Result< - native_tls::TlsStream>, - native_tls::HandshakeError>, + native_tls::TlsStream>, + native_tls::HandshakeError>, >, ) -> io::Result> { use native_tls::HandshakeError; diff --git a/compio-tls/src/adapter/rtls.rs b/compio-tls/src/adapter/rtls.rs index 350a8c9c..d364b6da 100644 --- a/compio-tls/src/adapter/rtls.rs +++ b/compio-tls/src/adapter/rtls.rs @@ -1,12 +1,12 @@ use std::{io, ops::DerefMut, sync::Arc}; -use compio_io::{AsyncRead, AsyncWrite}; +use compio_io::{compat::SyncStream, AsyncRead, AsyncWrite}; use rustls::{ ClientConfig, ClientConnection, ConnectionCommon, Error, ServerConfig, ServerConnection, ServerName, }; -use crate::{wrapper::StreamWrapper, TlsStream}; +use crate::TlsStream; pub enum HandshakeError { Rustls(Error), @@ -15,16 +15,16 @@ pub enum HandshakeError { } pub struct MidStream { - stream: StreamWrapper, + stream: SyncStream, conn: C, - result_fn: fn(StreamWrapper, C) -> TlsStream, + result_fn: fn(SyncStream, C) -> TlsStream, } impl MidStream { pub fn new( - stream: StreamWrapper, + stream: SyncStream, conn: C, - result_fn: fn(StreamWrapper, C) -> TlsStream, + result_fn: fn(SyncStream, C) -> TlsStream, ) -> Self { Self { stream, @@ -33,7 +33,7 @@ impl MidStream { } } - pub fn get_mut(&mut self) -> &mut StreamWrapper { + pub fn get_mut(&mut self) -> &mut SyncStream { &mut self.stream } @@ -107,7 +107,7 @@ impl TlsConnector { .map_err(HandshakeError::Rustls)?; MidStream::new( - StreamWrapper::new(stream), + SyncStream::new(stream), conn, TlsStream::::new_rustls_client, ) @@ -127,7 +127,7 @@ impl TlsAcceptor { let conn = ServerConnection::new(self.0.clone()).map_err(HandshakeError::Rustls)?; MidStream::new( - StreamWrapper::new(stream), + SyncStream::new(stream), conn, TlsStream::::new_rustls_server, ) diff --git a/compio-tls/src/lib.rs b/compio-tls/src/lib.rs index 6df4a662..5ea507c9 100644 --- a/compio-tls/src/lib.rs +++ b/compio-tls/src/lib.rs @@ -5,8 +5,6 @@ mod adapter; mod stream; -mod wrapper; pub use adapter::*; pub use stream::*; -pub(crate) use wrapper::*; diff --git a/compio-tls/src/stream/mod.rs b/compio-tls/src/stream/mod.rs index 746d21ce..169b3589 100644 --- a/compio-tls/src/stream/mod.rs +++ b/compio-tls/src/stream/mod.rs @@ -1,9 +1,7 @@ use std::{io, mem::MaybeUninit}; use compio_buf::{BufResult, IoBuf, IoBufMut}; -use compio_io::{AsyncRead, AsyncWrite}; - -use crate::StreamWrapper; +use compio_io::{compat::SyncStream, AsyncRead, AsyncWrite}; #[cfg(feature = "rustls")] mod rtls; @@ -12,13 +10,13 @@ mod rtls; #[allow(clippy::large_enum_variant)] enum TlsStreamInner { #[cfg(feature = "native-tls")] - NativeTls(native_tls::TlsStream>), + NativeTls(native_tls::TlsStream>), #[cfg(feature = "rustls")] - Rustls(rtls::TlsStream>), + Rustls(rtls::TlsStream>), } impl TlsStreamInner { - fn get_mut(&mut self) -> &mut StreamWrapper { + fn get_mut(&mut self) -> &mut SyncStream { match self { #[cfg(feature = "native-tls")] Self::NativeTls(s) => s.get_mut(), @@ -71,20 +69,20 @@ pub struct TlsStream(TlsStreamInner); impl TlsStream { #[cfg(feature = "rustls")] - pub(crate) fn new_rustls_client(s: StreamWrapper, conn: rustls::ClientConnection) -> Self { + pub(crate) fn new_rustls_client(s: SyncStream, conn: rustls::ClientConnection) -> Self { Self(TlsStreamInner::Rustls(rtls::TlsStream::new_client(s, conn))) } #[cfg(feature = "rustls")] - pub(crate) fn new_rustls_server(s: StreamWrapper, conn: rustls::ServerConnection) -> Self { + pub(crate) fn new_rustls_server(s: SyncStream, conn: rustls::ServerConnection) -> Self { Self(TlsStreamInner::Rustls(rtls::TlsStream::new_server(s, conn))) } } #[cfg(feature = "native-tls")] #[doc(hidden)] -impl From>> for TlsStream { - fn from(value: native_tls::TlsStream>) -> Self { +impl From>> for TlsStream { + fn from(value: native_tls::TlsStream>) -> Self { Self(TlsStreamInner::NativeTls(value)) } } From fe27a47f25ff81885f14da3b18c5a4bfe6a8543e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Fri, 10 Nov 2023 18:30:32 +0800 Subject: [PATCH 27/44] feat: add `read_buf` feature for wrappers --- compio-http-client/Cargo.toml | 3 +++ compio-http/Cargo.toml | 3 +++ compio-http/src/lib.rs | 1 + compio-http/src/stream.rs | 42 +++++++++++++++++++++++++---------- compio-io/Cargo.toml | 5 +++-- compio-io/src/compat.rs | 10 +++++++++ compio-io/src/lib.rs | 2 ++ compio-tls/Cargo.toml | 3 +++ compio-tls/src/lib.rs | 1 + compio-tls/src/stream/mod.rs | 35 ++++++++++++++++++++++++----- compio-tls/src/stream/rtls.rs | 17 +++++++++++--- compio/Cargo.toml | 2 +- 12 files changed, 101 insertions(+), 23 deletions(-) diff --git a/compio-http-client/Cargo.toml b/compio-http-client/Cargo.toml index c1a28a59..b8ab5bc2 100644 --- a/compio-http-client/Cargo.toml +++ b/compio-http-client/Cargo.toml @@ -42,3 +42,6 @@ rustls = ["compio-http/rustls"] vendored = ["compio-http/vendored"] json = ["dep:serde_json"] all = ["json", "native-tls", "rustls", "compio-http/all"] + +read_buf = ["compio-http/read_buf"] +nightly = ["read_buf"] diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 84fc0e67..00ae777b 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -35,3 +35,6 @@ client = ["hyper/client"] server = ["hyper/server"] all = ["native-tls", "rustls", "client", "server"] + +read_buf = ["compio-tls/read_buf"] +nightly = ["read_buf"] diff --git a/compio-http/src/lib.rs b/compio-http/src/lib.rs index e254ae47..83d8eb8a 100644 --- a/compio-http/src/lib.rs +++ b/compio-http/src/lib.rs @@ -1,6 +1,7 @@ //! A mid level HTTP services for [`hyper`]. #![warn(missing_docs)] +#![cfg_attr(feature = "read_buf", feature(read_buf))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] mod service; diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index bba8be72..2ab5e4aa 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -265,6 +265,30 @@ macro_rules! poll_future_would_block { }}; } +#[cfg(not(feature = "read_buf"))] +#[inline] +fn read_buf(reader: &mut impl io::Read, buf: &mut tokio::io::ReadBuf<'_>) -> io::Result<()> { + let slice = buf.initialize_unfilled(); + let len = reader.read(slice)?; + buf.advance(len); + Ok(()) +} + +#[cfg(feature = "read_buf")] +#[inline] +fn read_buf(reader: &mut impl io::Read, buf: &mut tokio::io::ReadBuf<'_>) -> io::Result<()> { + let slice = unsafe { buf.unfilled_mut() }; + let len = { + let mut borrowed_buf = io::BorrowedBuf::from(slice); + let mut cursor = borrowed_buf.unfilled(); + reader.read_buf(cursor.reborrow())?; + cursor.written() + }; + unsafe { buf.assume_init(len) }; + buf.advance(len); + Ok(()) +} + impl tokio::io::AsyncRead for HttpStream { fn poll_read( mut self: Pin<&mut Self>, @@ -274,18 +298,12 @@ impl tokio::io::AsyncRead for HttpStream { let inner: &'static mut SyncStream = unsafe { &mut *(self.inner.deref_mut() as *mut _) }; - let res = poll_future_would_block!(self.read_future, cx, inner.fill_read_buf(), { - let slice = buf.initialize_unfilled(); - io::Read::read(inner, slice) - }); - match res { - Poll::Ready(Ok(len)) => { - buf.advance(len); - Poll::Ready(Ok(())) - } - Poll::Ready(Err(e)) => Poll::Ready(Err(e)), - Poll::Pending => Poll::Pending, - } + poll_future_would_block!( + self.read_future, + cx, + inner.fill_read_buf(), + read_buf(inner, buf) + ) } } diff --git a/compio-io/Cargo.toml b/compio-io/Cargo.toml index 7d25b51c..0086f1cd 100644 --- a/compio-io/Cargo.toml +++ b/compio-io/Cargo.toml @@ -23,5 +23,6 @@ default = [] compat = [] # Nightly features -allocator_api = [] -nightly = ["allocator_api", "compio-buf/allocator_api"] +allocator_api = ["compio-buf/allocator_api"] +read_buf = ["compio-buf/read_buf"] +nightly = ["allocator_api", "read_buf"] diff --git a/compio-io/src/compat.rs b/compio-io/src/compat.rs index 0b3dd47c..5793b4d0 100644 --- a/compio-io/src/compat.rs +++ b/compio-io/src/compat.rs @@ -66,6 +66,16 @@ impl Read for SyncStream { res }) } + + #[cfg(feature = "read_buf")] + fn read_buf(&mut self, mut buf: io::BorrowedCursor<'_>) -> io::Result<()> { + let mut slice = self.fill_buf()?; + let old_written = buf.written(); + slice.read_buf(buf.reborrow())?; + let len = buf.written() - old_written; + self.consume(len); + Ok(()) + } } impl BufRead for SyncStream { diff --git a/compio-io/src/lib.rs b/compio-io/src/lib.rs index 9b59071d..a3537dfe 100644 --- a/compio-io/src/lib.rs +++ b/compio-io/src/lib.rs @@ -100,6 +100,8 @@ // This is OK as we're thread-per-core and don't need `Send` or other auto trait on anonymous future #![allow(async_fn_in_trait)] #![cfg_attr(feature = "allocator_api", feature(allocator_api))] +#![cfg_attr(feature = "read_buf", feature(read_buf))] +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] mod buffer; #[cfg(feature = "compat")] diff --git a/compio-tls/Cargo.toml b/compio-tls/Cargo.toml index 7e08d8a6..add88212 100644 --- a/compio-tls/Cargo.toml +++ b/compio-tls/Cargo.toml @@ -26,3 +26,6 @@ rustls-native-certs = { workspace = true } [features] default = ["native-tls"] all = ["native-tls", "rustls"] + +read_buf = ["compio-buf/read_buf", "compio-io/read_buf"] +nightly = ["read_buf"] diff --git a/compio-tls/src/lib.rs b/compio-tls/src/lib.rs index 5ea507c9..2900459b 100644 --- a/compio-tls/src/lib.rs +++ b/compio-tls/src/lib.rs @@ -1,6 +1,7 @@ //! Async TLS streams. #![warn(missing_docs)] +#![cfg_attr(feature = "read_buf", feature(read_buf))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] mod adapter; diff --git a/compio-tls/src/stream/mod.rs b/compio-tls/src/stream/mod.rs index 169b3589..8557f20d 100644 --- a/compio-tls/src/stream/mod.rs +++ b/compio-tls/src/stream/mod.rs @@ -35,6 +35,16 @@ impl io::Read for TlsStreamInner { Self::Rustls(s) => io::Read::read(s, buf), } } + + #[cfg(feature = "read_buf")] + fn read_buf(&mut self, buf: io::BorrowedCursor<'_>) -> io::Result<()> { + match self { + #[cfg(feature = "native-tls")] + Self::NativeTls(s) => io::Read::read_buf(s, buf), + #[cfg(feature = "rustls")] + Self::Rustls(s) => io::Read::read_buf(s, buf), + } + } } impl io::Write for TlsStreamInner { @@ -87,14 +97,29 @@ impl From>> for TlsStream { } } +#[cfg(not(feature = "read_buf"))] +#[inline] +fn read_buf(reader: &mut impl io::Read, buf: &mut B) -> io::Result { + let slice: &mut [MaybeUninit] = buf.as_mut_slice(); + slice.fill(MaybeUninit::new(0)); + let slice = unsafe { std::slice::from_raw_parts_mut(slice.as_mut_ptr().cast(), slice.len()) }; + reader.read(slice) +} + +#[cfg(feature = "read_buf")] +#[inline] +fn read_buf(reader: &mut impl io::Read, buf: &mut B) -> io::Result { + let slice: &mut [MaybeUninit] = buf.as_mut_slice(); + let mut borrowed_buf = io::BorrowedBuf::from(slice); + let mut cursor = borrowed_buf.unfilled(); + reader.read_buf(cursor.reborrow())?; + Ok(cursor.written()) +} + impl AsyncRead for TlsStream { async fn read(&mut self, mut buf: B) -> BufResult { - let slice: &mut [MaybeUninit] = buf.as_mut_slice(); - slice.fill(MaybeUninit::new(0)); - let slice = - unsafe { std::slice::from_raw_parts_mut(slice.as_mut_ptr().cast(), slice.len()) }; loop { - let res = io::Read::read(&mut self.0, slice); + let res = read_buf(&mut self.0, &mut buf); match res { Ok(res) => { unsafe { buf.set_buf_init(res) }; diff --git a/compio-tls/src/stream/rtls.rs b/compio-tls/src/stream/rtls.rs index f669fa98..08e01a93 100644 --- a/compio-tls/src/stream/rtls.rs +++ b/compio-tls/src/stream/rtls.rs @@ -85,8 +85,8 @@ impl TlsStream { } } -impl io::Read for TlsStream { - fn read(&mut self, buf: &mut [u8]) -> io::Result { +impl TlsStream { + fn read_impl(&mut self, mut f: impl FnMut(Reader) -> io::Result) -> io::Result { loop { while self.conn.wants_read() { self.conn.read_tls(&mut self.inner)?; @@ -95,7 +95,7 @@ impl io::Read for TlsStream { .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; } - match self.conn.reader().read(buf) { + match f(self.conn.reader()) { Ok(len) => { return Ok(len); } @@ -106,6 +106,17 @@ impl io::Read for TlsStream { } } +impl io::Read for TlsStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.read_impl(|mut reader| reader.read(buf)) + } + + #[cfg(feature = "read_buf")] + fn read_buf(&mut self, mut buf: io::BorrowedCursor<'_>) -> io::Result<()> { + self.read_impl(|mut reader| reader.read_buf(buf.reborrow())) + } +} + impl io::Write for TlsStream { fn write(&mut self, buf: &[u8]) -> io::Result { self.flush()?; diff --git a/compio/Cargo.toml b/compio/Cargo.toml index 4a984033..c688c0ff 100644 --- a/compio/Cargo.toml +++ b/compio/Cargo.toml @@ -91,7 +91,7 @@ once_cell_try = [ "compio-runtime?/once_cell_try", "compio-signal?/once_cell_try", ] -read_buf = ["compio-buf/read_buf"] +read_buf = ["compio-buf/read_buf", "compio-io?/read_buf"] try_trait_v2 = ["compio-buf/try_trait_v2"] nightly = [ "allocator_api", From 71230278be2ae4e2ae0c60d720c49e66aa848c38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Fri, 10 Nov 2023 18:34:26 +0800 Subject: [PATCH 28/44] fix: remove useless commented code --- compio-http/src/stream.rs | 90 --------------------------------------- 1 file changed, 90 deletions(-) diff --git a/compio-http/src/stream.rs b/compio-http/src/stream.rs index 2ab5e4aa..61bbf6f9 100644 --- a/compio-http/src/stream.rs +++ b/compio-http/src/stream.rs @@ -100,96 +100,6 @@ impl AsyncWrite for HttpStreamInner { } } -// const DEFAULT_BUF_SIZE: usize = 8 * 1024; - -// struct HttpStreamBufInner { -// inner: HttpStreamInner, -// read_buffer: Buffer, -// write_buffer: Buffer, -// } - -// impl HttpStreamBufInner { -// pub async fn connect(uri: Uri, tls: TlsBackend) -> io::Result { -// Ok(Self::from_inner(HttpStreamInner::connect(uri, tls).await?)) -// } - -// pub fn from_tcp(s: TcpStream) -> Self { -// Self::from_inner(HttpStreamInner::from_tcp(s)) -// } - -// pub fn from_tls(s: TlsStream) -> Self { -// Self::from_inner(HttpStreamInner::from_tls(s)) -// } - -// fn from_inner(s: HttpStreamInner) -> Self { -// Self { -// inner: s, -// read_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), -// write_buffer: Buffer::with_capacity(DEFAULT_BUF_SIZE), -// } -// } - -// pub async fn fill_read_buf(&mut self) -> io::Result<()> { -// if self.read_buffer.all_done() { -// self.read_buffer.reset(); -// } -// if self.read_buffer.slice().is_empty() { -// self.read_buffer -// .with(|b| async { -// let len = b.buf_len(); -// let slice = b.slice(len..); -// self.inner.read(slice).await.into_inner() -// }) -// .await?; -// } - -// Ok(()) -// } - -// pub fn read_buf_slice(&self) -> &[u8] { -// self.read_buffer.slice() -// } - -// pub fn consume_read_buf(&mut self, amt: usize) { -// self.read_buffer.advance(amt); -// } - -// pub async fn flush_write_buf_if_needed(&mut self) -> io::Result<()> { -// if self.write_buffer.need_flush() { -// self.flush_write_buf().await?; -// } -// Ok(()) -// } - -// pub fn write_slice(&mut self, buf: &[u8]) -> io::Result { -// self.write_buffer.with_sync(|mut inner| { -// let len = buf.len().min(inner.buf_capacity() - inner.buf_len()); -// unsafe { -// std::ptr::copy_nonoverlapping( -// buf.as_ptr(), -// inner.as_buf_mut_ptr().add(inner.buf_len()), -// len, -// ); -// inner.set_buf_init(inner.buf_len() + len); -// } -// BufResult(Ok(len), inner) -// }) -// } - -// pub async fn flush_write_buf(&mut self) -> io::Result<()> { -// if !self.write_buffer.is_empty() { -// self.write_buffer.with(|b| self.inner.write_all(b)).await?; -// self.write_buffer.reset(); -// } -// self.inner.flush().await?; -// Ok(()) -// } - -// pub async fn shutdown(&mut self) -> io::Result<()> { -// self.inner.shutdown().await -// } -// } - type PinBoxFuture = Pin + Send>>; /// A HTTP stream wrapper, based on compio, and exposes [`tokio::io`] From 9d8a82cde8acb262d727cbb95caaabb6513ffa5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Fri, 10 Nov 2023 20:57:35 +0800 Subject: [PATCH 29/44] fix: new feature core_io_borrowed_buf --- compio-buf/src/lib.rs | 2 +- compio-http/src/lib.rs | 2 +- compio-io/src/lib.rs | 2 +- compio-tls/src/lib.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compio-buf/src/lib.rs b/compio-buf/src/lib.rs index 54fcb9d6..916b1eb5 100644 --- a/compio-buf/src/lib.rs +++ b/compio-buf/src/lib.rs @@ -6,7 +6,7 @@ #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![cfg_attr(feature = "allocator_api", feature(allocator_api))] -#![cfg_attr(feature = "read_buf", feature(read_buf))] +#![cfg_attr(feature = "read_buf", feature(read_buf, core_io_borrowed_buf))] #![cfg_attr(feature = "try_trait_v2", feature(try_trait_v2, try_trait_v2_residual))] #![warn(missing_docs)] diff --git a/compio-http/src/lib.rs b/compio-http/src/lib.rs index 83d8eb8a..5c8c0c6b 100644 --- a/compio-http/src/lib.rs +++ b/compio-http/src/lib.rs @@ -1,7 +1,7 @@ //! A mid level HTTP services for [`hyper`]. #![warn(missing_docs)] -#![cfg_attr(feature = "read_buf", feature(read_buf))] +#![cfg_attr(feature = "read_buf", feature(read_buf, core_io_borrowed_buf))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] mod service; diff --git a/compio-io/src/lib.rs b/compio-io/src/lib.rs index a3537dfe..f0139d09 100644 --- a/compio-io/src/lib.rs +++ b/compio-io/src/lib.rs @@ -100,7 +100,7 @@ // This is OK as we're thread-per-core and don't need `Send` or other auto trait on anonymous future #![allow(async_fn_in_trait)] #![cfg_attr(feature = "allocator_api", feature(allocator_api))] -#![cfg_attr(feature = "read_buf", feature(read_buf))] +#![cfg_attr(feature = "read_buf", feature(read_buf, core_io_borrowed_buf))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] mod buffer; diff --git a/compio-tls/src/lib.rs b/compio-tls/src/lib.rs index 2900459b..1c9dd16d 100644 --- a/compio-tls/src/lib.rs +++ b/compio-tls/src/lib.rs @@ -1,7 +1,7 @@ //! Async TLS streams. #![warn(missing_docs)] -#![cfg_attr(feature = "read_buf", feature(read_buf))] +#![cfg_attr(feature = "read_buf", feature(read_buf, core_io_borrowed_buf))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] mod adapter; From ce9f02c2da22a282d71f57d2165c4f6af15c73a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Sat, 11 Nov 2023 16:04:06 +0800 Subject: [PATCH 30/44] refactor: rename ClientRef to ClientInner --- compio-http-client/src/client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compio-http-client/src/client.rs b/compio-http-client/src/client.rs index 3ce1f0c9..b7e4251c 100644 --- a/compio-http-client/src/client.rs +++ b/compio-http-client/src/client.rs @@ -8,7 +8,7 @@ use crate::{IntoUrl, Request, RequestBuilder, Response, Result}; /// An asynchronous `Client` to make Requests with. #[derive(Debug, Clone)] pub struct Client { - client: Rc, + client: Rc, } impl Client { @@ -91,7 +91,7 @@ impl Client { } #[derive(Debug)] -struct ClientRef { +struct ClientInner { client: hyper::Client, headers: HeaderMap, } @@ -126,7 +126,7 @@ impl ClientBuilder { .executor(CompioExecutor) .set_host(true) .build(Connector::new(self.tls)); - let client_ref = ClientRef { + let client_ref = ClientInner { client, headers: self.headers, }; From 14445b75716ae034086c92a297fc8b7a8da81b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Sat, 11 Nov 2023 16:05:15 +0800 Subject: [PATCH 31/44] fix: add docsrs cfg --- compio-http-client/Cargo.toml | 4 ++++ compio-http/Cargo.toml | 4 ++++ compio-tls/Cargo.toml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/compio-http-client/Cargo.toml b/compio-http-client/Cargo.toml index b8ab5bc2..34436a94 100644 --- a/compio-http-client/Cargo.toml +++ b/compio-http-client/Cargo.toml @@ -9,6 +9,10 @@ readme = { workspace = true } license = { workspace = true } repository = { workspace = true } +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg docsrs"] + [dependencies] compio-buf = { workspace = true, features = ["bytes"] } compio-runtime = { workspace = true, features = ["time"] } diff --git a/compio-http/Cargo.toml b/compio-http/Cargo.toml index 00ae777b..594b8bb9 100644 --- a/compio-http/Cargo.toml +++ b/compio-http/Cargo.toml @@ -9,6 +9,10 @@ readme = { workspace = true } license = { workspace = true } repository = { workspace = true } +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg docsrs"] + [dependencies] compio-buf = { workspace = true } compio-io = { workspace = true, features = ["compat"] } diff --git a/compio-tls/Cargo.toml b/compio-tls/Cargo.toml index add88212..61e4b430 100644 --- a/compio-tls/Cargo.toml +++ b/compio-tls/Cargo.toml @@ -9,6 +9,10 @@ readme = { workspace = true } license = { workspace = true } repository = { workspace = true } +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg docsrs"] + [dependencies] compio-buf = { workspace = true } compio-io = { workspace = true, features = ["compat"] } From e4fccde6e23297657ecf30e548812939320e7758 Mon Sep 17 00:00:00 2001 From: George Miao Date: Tue, 7 Nov 2023 07:18:53 -0500 Subject: [PATCH 32/44] refactor(runtime): expose `async_task::Task` --- compio-runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compio-runtime/src/lib.rs b/compio-runtime/src/lib.rs index 761d459b..c1cd9337 100644 --- a/compio-runtime/src/lib.rs +++ b/compio-runtime/src/lib.rs @@ -26,7 +26,7 @@ pub mod time; use std::{cell::RefCell, future::Future, io, mem::ManuallyDrop}; -use async_task::Task; +pub use async_task::Task; pub use attacher::*; use compio_buf::BufResult; use compio_driver::{OpCode, ProactorBuilder, RawFd}; From 639d29259a3a5b4ce9b782e94a0d0e55328d1dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Thu, 9 Nov 2023 20:28:50 +0800 Subject: [PATCH 33/44] refactor: ensure Event being oneshot --- compio-net/src/resolve/windows.rs | 7 ++++--- compio-runtime/src/event/eventfd.rs | 4 ++-- compio-runtime/src/event/iocp.rs | 4 ++-- compio-runtime/src/event/pipe.rs | 4 ++-- compio-signal/src/unix.rs | 23 ++++++++++++++++------- compio-signal/src/windows.rs | 12 ++++++++---- 6 files changed, 34 insertions(+), 20 deletions(-) diff --git a/compio-net/src/resolve/windows.rs b/compio-net/src/resolve/windows.rs index d333b8c1..655ddacf 100644 --- a/compio-net/src/resolve/windows.rs +++ b/compio-net/src/resolve/windows.rs @@ -42,9 +42,10 @@ impl AsyncResolver { _dwbytes: u32, lpoverlapped: *const OVERLAPPED, ) { - let overlapped_ptr = lpoverlapped.cast::(); - if let Some(overlapped) = overlapped_ptr.as_ref() { - if let Some(handle) = overlapped.handle.as_ref() { + // We won't access the overlapped struct outside callback. + let overlapped_ptr = lpoverlapped.cast::().cast_mut(); + if let Some(overlapped) = overlapped_ptr.as_mut() { + if let Some(handle) = overlapped.handle.take() { handle.notify().ok(); } } diff --git a/compio-runtime/src/event/eventfd.rs b/compio-runtime/src/event/eventfd.rs index 5106ca5f..1bf76aec 100644 --- a/compio-runtime/src/event/eventfd.rs +++ b/compio-runtime/src/event/eventfd.rs @@ -33,7 +33,7 @@ impl Event { } /// Wait for [`EventHandle::notify`] called. - pub async fn wait(&self) -> io::Result<()> { + pub async fn wait(self) -> io::Result<()> { self.attacher.attach(&self.fd)?; let buffer = ArrayVec::::new(); // Trick: Recv uses readv which doesn't seek. @@ -61,7 +61,7 @@ impl EventHandle { } /// Notify the event. - pub fn notify(&self) -> io::Result<()> { + pub fn notify(self) -> io::Result<()> { let data = 1u64; syscall!(libc::write( self.fd.as_raw_fd(), diff --git a/compio-runtime/src/event/iocp.rs b/compio-runtime/src/event/iocp.rs index 7ed09370..5f58d4f7 100644 --- a/compio-runtime/src/event/iocp.rs +++ b/compio-runtime/src/event/iocp.rs @@ -29,7 +29,7 @@ impl Event { } /// Wait for [`EventHandle::notify`] called. - pub async fn wait(&self) -> io::Result<()> { + pub async fn wait(self) -> io::Result<()> { let future = OpFuture::new(self.user_data); future.await.0?; Ok(()) @@ -56,7 +56,7 @@ impl EventHandle { } /// Notify the event. - pub fn notify(&self) -> io::Result<()> { + pub fn notify(self) -> io::Result<()> { post_driver_nop(self.handle, self.user_data) } } diff --git a/compio-runtime/src/event/pipe.rs b/compio-runtime/src/event/pipe.rs index bef9b7b0..d5aa5c8e 100644 --- a/compio-runtime/src/event/pipe.rs +++ b/compio-runtime/src/event/pipe.rs @@ -42,7 +42,7 @@ impl Event { } /// Wait for [`EventHandle::notify`] called. - pub async fn wait(&self) -> io::Result<()> { + pub async fn wait(self) -> io::Result<()> { self.attacher.attach(&self.receiver)?; let buffer = ArrayVec::::new(); // Trick: Recv uses readv which doesn't seek. @@ -70,7 +70,7 @@ impl EventHandle { } /// Notify the event. - pub fn notify(&self) -> io::Result<()> { + pub fn notify(self) -> io::Result<()> { let data = &[1]; syscall!(libc::write(self.fd.as_raw_fd(), data.as_ptr() as _, 1))?; Ok(()) diff --git a/compio-signal/src/unix.rs b/compio-signal/src/unix.rs index 4e2228a1..fd7c4a01 100644 --- a/compio-signal/src/unix.rs +++ b/compio-signal/src/unix.rs @@ -70,24 +70,33 @@ fn unregister(sig: i32, fd: RawFd) { #[derive(Debug)] struct SignalFd { sig: i32, - fd: Event, + fd: RawFd, + event: Option, } impl SignalFd { fn new(sig: i32) -> io::Result { - let fd = Event::new()?; - register(sig, &fd)?; - Ok(Self { sig, fd }) + let event = Event::new()?; + register(sig, &event)?; + Ok(Self { + sig, + fd: event.as_raw_fd(), + event: Some(event), + }) } - async fn wait(&self) -> io::Result<()> { - self.fd.wait().await + async fn wait(mut self) -> io::Result<()> { + self.event + .take() + .expect("event could not be None") + .wait() + .await } } impl Drop for SignalFd { fn drop(&mut self) { - unregister(self.sig, self.fd.as_raw_fd()); + unregister(self.sig, self.fd); } } diff --git a/compio-signal/src/windows.rs b/compio-signal/src/windows.rs index e278e8d2..7de0153e 100644 --- a/compio-signal/src/windows.rs +++ b/compio-signal/src/windows.rs @@ -64,7 +64,7 @@ fn unregister(ctrltype: u32, key: usize) { #[derive(Debug)] struct CtrlEvent { ctrltype: u32, - event: Event, + event: Option, handler_key: usize, } @@ -76,13 +76,17 @@ impl CtrlEvent { let handler_key = register(ctrltype, &event)?; Ok(Self { ctrltype, - event, + event: Some(event), handler_key, }) } - pub async fn wait(&self) -> io::Result<()> { - self.event.wait().await + pub async fn wait(mut self) -> io::Result<()> { + self.event + .take() + .expect("event could not be None") + .wait() + .await } } From 11bfe59372d91ba64ed78b204c372db1a094b044 Mon Sep 17 00:00:00 2001 From: Ho 229 Date: Sat, 4 Nov 2023 19:09:00 +0800 Subject: [PATCH 34/44] feat: init compio-log --- Cargo.toml | 2 + compio-log/Cargo.toml | 17 +++++++ compio-log/src/lib.rs | 99 ++++++++++++++++++++++++++++++++++++++++ compio-log/tests/test.rs | 15 ++++++ 4 files changed, 133 insertions(+) create mode 100644 compio-log/Cargo.toml create mode 100644 compio-log/src/lib.rs create mode 100644 compio-log/tests/test.rs diff --git a/Cargo.toml b/Cargo.toml index 63a6317b..7a8092c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "compio-tls", "compio-http", "compio-http-client", + "compio-log", ] resolver = "2" @@ -35,6 +36,7 @@ compio-signal = { path = "./compio-signal", version = "0.1.1" } compio-dispatcher = { path = "./compio-dispatcher", version = "0.1.0" } compio-tls = { path = "./compio-tls", version = "0.1.0" } compio-http = { path = "./compio-http", version = "0.1.0" } +compio-log = { path = "./compio-log", version = "0.1.0" } cfg-if = "1.0.0" crossbeam-channel = "0.5.8" diff --git a/compio-log/Cargo.toml b/compio-log/Cargo.toml new file mode 100644 index 00000000..a2ab0ab9 --- /dev/null +++ b/compio-log/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "compio-log" +version = "0.1.0" +description = "log of compio" +categories = ["asynchronous"] +edition = { workspace = true } +authors = { workspace = true } +readme = { workspace = true } +license = { workspace = true } +repository = { workspace = true } + +[dependencies] +tracing = "0.1" +tracing-subscriber = "0.3" + +[features] +enable-log = [] diff --git a/compio-log/src/lib.rs b/compio-log/src/lib.rs new file mode 100644 index 00000000..c5e3a312 --- /dev/null +++ b/compio-log/src/lib.rs @@ -0,0 +1,99 @@ +#[doc(hidden)] +pub use tracing::*; +pub use tracing_subscriber as subscriber; + +#[macro_export] +macro_rules! debug { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::debug!($($args)*) + }; +} + +#[macro_export] +macro_rules! debug_span { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::debug_span!($($args)*) + }; +} + +#[macro_export] +macro_rules! error { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::error!($($args)*) + }; +} + +#[macro_export] +macro_rules! error_span { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::error_span!($($args)*) + }; +} + +#[macro_export] +macro_rules! event { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::event!($($args)*) + }; +} + +#[macro_export] +macro_rules! info { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::info!($($args)*) + }; +} + +#[macro_export] +macro_rules! info_span { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::info_span!($($args)*) + }; +} + +#[macro_export] +macro_rules! span { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::span!($($args)*) + }; +} + +#[macro_export] +macro_rules! trace { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::trace!($($args)*) + }; +} + +#[macro_export] +macro_rules! trace_span { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::trace_span!($($args)*) + }; +} + +#[macro_export] +macro_rules! warn { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::warn!($($args)*) + }; +} + +#[macro_export] +macro_rules! warn_span { + ($($args:tt)*) => { + #[cfg(feature = "enable-log")] + ::tracing::warn_span!($($args)*) + }; +} diff --git a/compio-log/tests/test.rs b/compio-log/tests/test.rs new file mode 100644 index 00000000..a54c13a2 --- /dev/null +++ b/compio-log/tests/test.rs @@ -0,0 +1,15 @@ +use compio_log::Level; + +#[test] +fn test_log() { + compio_log::subscriber::fmt() + .with_max_level(Level::TRACE) + .init(); + + compio_log::debug!("debug"); + compio_log::error!("error"); + compio_log::event!(Level::DEBUG, "event"); + compio_log::info!("info"); + compio_log::warn!("warn"); + compio_log::trace!("trace"); +} From e7dfa3380e3b4a7ddf7cb15a40711189c49dd4f8 Mon Sep 17 00:00:00 2001 From: Ho 229 Date: Sun, 5 Nov 2023 16:05:15 +0800 Subject: [PATCH 35/44] refactor: compio-log --- compio-log/Cargo.toml | 2 +- compio-log/src/dummy.rs | 71 ++++++++++++++++++++++ compio-log/src/lib.rs | 97 +------------------------------ compio-runtime/Cargo.toml | 1 + compio-runtime/src/key.rs | 8 ++- compio-runtime/src/runtime/mod.rs | 4 ++ compio/Cargo.toml | 3 +- 7 files changed, 88 insertions(+), 98 deletions(-) create mode 100644 compio-log/src/dummy.rs diff --git a/compio-log/Cargo.toml b/compio-log/Cargo.toml index a2ab0ab9..b93dbd86 100644 --- a/compio-log/Cargo.toml +++ b/compio-log/Cargo.toml @@ -14,4 +14,4 @@ tracing = "0.1" tracing-subscriber = "0.3" [features] -enable-log = [] +enable_log = [] diff --git a/compio-log/src/dummy.rs b/compio-log/src/dummy.rs new file mode 100644 index 00000000..514dcc6b --- /dev/null +++ b/compio-log/src/dummy.rs @@ -0,0 +1,71 @@ +#[macro_export] +macro_rules! debug { + ($($args:tt)*) => {}; +} + +#[macro_export] +macro_rules! debug_span { + ($($args:tt)*) => { + $crate::Span::none() + }; +} + +#[macro_export] +macro_rules! error { + ($($args:tt)*) => {}; +} + +#[macro_export] +macro_rules! error_span { + ($($args:tt)*) => { + $crate::Span::none() + }; +} + +#[macro_export] +macro_rules! event { + ($($args:tt)*) => {}; +} + +#[macro_export] +macro_rules! info { + ($($args:tt)*) => {}; +} + +#[macro_export] +macro_rules! info_span { + ($($args:tt)*) => { + $crate::Span::none() + }; +} + +#[macro_export] +macro_rules! span { + ($($args:tt)*) => { + $crate::Span::none() + }; +} + +#[macro_export] +macro_rules! trace { + ($($args:tt)*) => {}; +} + +#[macro_export] +macro_rules! trace_span { + ($($args:tt)*) => { + $crate::Span::none() + }; +} + +#[macro_export] +macro_rules! warn { + ($($args:tt)*) => {}; +} + +#[macro_export] +macro_rules! warn_span { + ($($args:tt)*) => { + $crate::Span::none() + }; +} diff --git a/compio-log/src/lib.rs b/compio-log/src/lib.rs index c5e3a312..5b5fc3f4 100644 --- a/compio-log/src/lib.rs +++ b/compio-log/src/lib.rs @@ -2,98 +2,5 @@ pub use tracing::*; pub use tracing_subscriber as subscriber; -#[macro_export] -macro_rules! debug { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::debug!($($args)*) - }; -} - -#[macro_export] -macro_rules! debug_span { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::debug_span!($($args)*) - }; -} - -#[macro_export] -macro_rules! error { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::error!($($args)*) - }; -} - -#[macro_export] -macro_rules! error_span { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::error_span!($($args)*) - }; -} - -#[macro_export] -macro_rules! event { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::event!($($args)*) - }; -} - -#[macro_export] -macro_rules! info { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::info!($($args)*) - }; -} - -#[macro_export] -macro_rules! info_span { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::info_span!($($args)*) - }; -} - -#[macro_export] -macro_rules! span { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::span!($($args)*) - }; -} - -#[macro_export] -macro_rules! trace { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::trace!($($args)*) - }; -} - -#[macro_export] -macro_rules! trace_span { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::trace_span!($($args)*) - }; -} - -#[macro_export] -macro_rules! warn { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::warn!($($args)*) - }; -} - -#[macro_export] -macro_rules! warn_span { - ($($args:tt)*) => { - #[cfg(feature = "enable-log")] - ::tracing::warn_span!($($args)*) - }; -} +#[cfg(not(feature = "enable_log"))] +pub mod dummy; diff --git a/compio-runtime/Cargo.toml b/compio-runtime/Cargo.toml index b3a0545c..e3e6a696 100644 --- a/compio-runtime/Cargo.toml +++ b/compio-runtime/Cargo.toml @@ -31,6 +31,7 @@ targets = [ # Workspace dependencies compio-driver = { workspace = true } compio-buf = { workspace = true } +compio-log = { workspace = true } async-task = "4.5.0" cfg-if = { workspace = true, optional = true } diff --git a/compio-runtime/src/key.rs b/compio-runtime/src/key.rs index 75276384..2545631d 100644 --- a/compio-runtime/src/key.rs +++ b/compio-runtime/src/key.rs @@ -1,5 +1,5 @@ /// A typed wrapper for key of Ops submitted into driver -#[derive(Debug, PartialEq, Eq, Hash)] +#[derive(PartialEq, Eq, Hash)] pub struct Key { user_data: usize, _p: std::marker::PhantomData, @@ -38,3 +38,9 @@ impl std::ops::Deref for Key { &self.user_data } } + +impl std::fmt::Debug for Key { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.user_data) + } +} diff --git a/compio-runtime/src/runtime/mod.rs b/compio-runtime/src/runtime/mod.rs index d46732a1..c1976cef 100644 --- a/compio-runtime/src/runtime/mod.rs +++ b/compio-runtime/src/runtime/mod.rs @@ -9,6 +9,7 @@ use std::{ use async_task::{Runnable, Task}; use compio_driver::{AsRawFd, Entry, OpCode, Proactor, ProactorBuilder, PushEntry, RawFd}; +use compio_log::{self as tracing, instrument, trace}; use futures_util::future::Either; use send_wrapper::SendWrapper; use smallvec::SmallVec; @@ -117,6 +118,7 @@ impl Runtime { self.timer_runtime.borrow_mut().cancel(key); } + #[instrument(level = "trace", skip(self))] pub fn poll_task( &self, cx: &mut Context, @@ -124,6 +126,7 @@ impl Runtime { ) -> Poll> { let mut op_runtime = self.op_runtime.borrow_mut(); if op_runtime.has_result(*user_data) { + trace!("has result"); let op = op_runtime.remove(*user_data); let res = self .driver @@ -133,6 +136,7 @@ impl Runtime { .expect("the result should have come"); Poll::Ready(res.map_buffer(|op| unsafe { op.into_op::() })) } else { + trace!("update waker"); op_runtime.update_waker(*user_data, cx.waker().clone()); Poll::Pending } diff --git a/compio/Cargo.toml b/compio/Cargo.toml index c688c0ff..f6974fe2 100644 --- a/compio/Cargo.toml +++ b/compio/Cargo.toml @@ -39,6 +39,7 @@ compio-io = { workspace = true, optional = true } compio-net = { workspace = true } compio-signal = { workspace = true, optional = true } compio-dispatcher = { workspace = true, optional = true } +compio-log = { workspace = true, optional = true } # Shared dev dependencies for all platforms [dev-dependencies] @@ -93,6 +94,7 @@ once_cell_try = [ ] read_buf = ["compio-buf/read_buf", "compio-io?/read_buf"] try_trait_v2 = ["compio-buf/try_trait_v2"] +enable_log = ["compio-log/enable_log"] nightly = [ "allocator_api", "lazy_cell", @@ -101,7 +103,6 @@ nightly = [ "try_trait_v2", ] - [[example]] name = "basic" required-features = ["macros"] From 6bae688df8048873c575eee647b091c232c1812f Mon Sep 17 00:00:00 2001 From: Ho 229 Date: Sun, 5 Nov 2023 18:45:42 +0800 Subject: [PATCH 36/44] feat: add logging for runtime --- compio-runtime/src/key.rs | 2 +- compio-runtime/src/runtime/mod.rs | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/compio-runtime/src/key.rs b/compio-runtime/src/key.rs index 2545631d..6b3f9fe4 100644 --- a/compio-runtime/src/key.rs +++ b/compio-runtime/src/key.rs @@ -41,6 +41,6 @@ impl std::ops::Deref for Key { impl std::fmt::Debug for Key { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.user_data) + write!(f, "Key({})", self.user_data) } } diff --git a/compio-runtime/src/runtime/mod.rs b/compio-runtime/src/runtime/mod.rs index c1976cef..06f440b7 100644 --- a/compio-runtime/src/runtime/mod.rs +++ b/compio-runtime/src/runtime/mod.rs @@ -143,26 +143,32 @@ impl Runtime { } #[cfg(feature = "time")] + #[instrument(level = "trace", skip(self))] pub fn poll_timer(&self, cx: &mut Context, key: usize) -> Poll<()> { let mut timer_runtime = self.timer_runtime.borrow_mut(); if timer_runtime.contains(key) { + trace!("pending"); timer_runtime.update_waker(key, cx.waker().clone()); Poll::Pending } else { + trace!("ready"); Poll::Ready(()) } } + #[instrument(level = "trace", skip(self))] fn poll(&self) { #[cfg(not(feature = "time"))] let timeout = None; #[cfg(feature = "time")] let timeout = self.timer_runtime.borrow().min_timeout(); + trace!("timeout: {:?}", timeout); let mut entries = SmallVec::<[Entry; 1024]>::new(); let mut driver = self.driver.borrow_mut(); match driver.poll(timeout, &mut entries) { Ok(_) => { + trace!("poll driver ok, entries: {}", entries.len()); for entry in entries { self.op_runtime .borrow_mut() @@ -170,7 +176,9 @@ impl Runtime { } } Err(e) => match e.kind() { - io::ErrorKind::TimedOut | io::ErrorKind::Interrupted => {} + io::ErrorKind::TimedOut | io::ErrorKind::Interrupted => { + trace!("expected error: {e}"); + } _ => panic!("{:?}", e), }, } From fe81ca5ffb8959794f4b064f2ae521d3a7023926 Mon Sep 17 00:00:00 2001 From: Ho 229 Date: Mon, 6 Nov 2023 17:19:43 +0800 Subject: [PATCH 37/44] refactor(log): rewrite instrument --- compio-log/Cargo.toml | 2 +- compio-log/src/lib.rs | 20 +++++++++++++++++++- compio-runtime/src/runtime/mod.rs | 8 ++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/compio-log/Cargo.toml b/compio-log/Cargo.toml index b93dbd86..c0660591 100644 --- a/compio-log/Cargo.toml +++ b/compio-log/Cargo.toml @@ -10,7 +10,7 @@ license = { workspace = true } repository = { workspace = true } [dependencies] -tracing = "0.1" +tracing = { version = "0.1", default-features = false } tracing-subscriber = "0.3" [features] diff --git a/compio-log/src/lib.rs b/compio-log/src/lib.rs index 5b5fc3f4..0c7973da 100644 --- a/compio-log/src/lib.rs +++ b/compio-log/src/lib.rs @@ -1,6 +1,24 @@ -#[doc(hidden)] +#[cfg_attr(not(feature = "enable_log"), doc(hidden))] pub use tracing::*; pub use tracing_subscriber as subscriber; #[cfg(not(feature = "enable_log"))] pub mod dummy; + +#[cfg(feature = "enable_log")] +#[macro_export] +macro_rules! instrument { + ($lvl:expr, $name:expr, $($fields:tt)*) => { + let _guard = $crate::span!(target:module_path!(), $lvl, $name, $($fields)*).entered(); + }; + ($lvl:expr, $name:expr) => { + let _guard = $crate::span!(target:module_path!(), $lvl, $name).entered(); + }; +} + +#[cfg(not(feature = "enable_log"))] +#[macro_export] +macro_rules! instrument { + ($lvl:expr, $name:expr, $($fields:tt)*) => {}; + ($lvl:expr, $name:expr) => {}; +} diff --git a/compio-runtime/src/runtime/mod.rs b/compio-runtime/src/runtime/mod.rs index 06f440b7..bd84af95 100644 --- a/compio-runtime/src/runtime/mod.rs +++ b/compio-runtime/src/runtime/mod.rs @@ -9,7 +9,7 @@ use std::{ use async_task::{Runnable, Task}; use compio_driver::{AsRawFd, Entry, OpCode, Proactor, ProactorBuilder, PushEntry, RawFd}; -use compio_log::{self as tracing, instrument, trace}; +use compio_log::{instrument, trace}; use futures_util::future::Either; use send_wrapper::SendWrapper; use smallvec::SmallVec; @@ -118,12 +118,12 @@ impl Runtime { self.timer_runtime.borrow_mut().cancel(key); } - #[instrument(level = "trace", skip(self))] pub fn poll_task( &self, cx: &mut Context, user_data: Key, ) -> Poll> { + instrument!(compio_log::Level::TRACE, "poll_task", ?cx, ?user_data,); let mut op_runtime = self.op_runtime.borrow_mut(); if op_runtime.has_result(*user_data) { trace!("has result"); @@ -143,8 +143,8 @@ impl Runtime { } #[cfg(feature = "time")] - #[instrument(level = "trace", skip(self))] pub fn poll_timer(&self, cx: &mut Context, key: usize) -> Poll<()> { + instrument!(compio_log::Level::TRACE, "poll_timer", ?cx, ?key); let mut timer_runtime = self.timer_runtime.borrow_mut(); if timer_runtime.contains(key) { trace!("pending"); @@ -156,8 +156,8 @@ impl Runtime { } } - #[instrument(level = "trace", skip(self))] fn poll(&self) { + instrument!(compio_log::Level::TRACE, "poll"); #[cfg(not(feature = "time"))] let timeout = None; #[cfg(feature = "time")] From cd7ccf191af901141ea6aea1c4679c30673682d8 Mon Sep 17 00:00:00 2001 From: Ho 229 Date: Mon, 6 Nov 2023 17:23:59 +0800 Subject: [PATCH 38/44] chore(log): remove unused import --- compio-log/Cargo.toml | 1 - compio-log/src/lib.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/compio-log/Cargo.toml b/compio-log/Cargo.toml index c0660591..65572092 100644 --- a/compio-log/Cargo.toml +++ b/compio-log/Cargo.toml @@ -11,7 +11,6 @@ repository = { workspace = true } [dependencies] tracing = { version = "0.1", default-features = false } -tracing-subscriber = "0.3" [features] enable_log = [] diff --git a/compio-log/src/lib.rs b/compio-log/src/lib.rs index 0c7973da..f0bdd072 100644 --- a/compio-log/src/lib.rs +++ b/compio-log/src/lib.rs @@ -1,6 +1,5 @@ #[cfg_attr(not(feature = "enable_log"), doc(hidden))] pub use tracing::*; -pub use tracing_subscriber as subscriber; #[cfg(not(feature = "enable_log"))] pub mod dummy; From 793bcd07cfa0a66f73add577d827ff6c8e371bc5 Mon Sep 17 00:00:00 2001 From: Ho 229 Date: Mon, 6 Nov 2023 17:28:13 +0800 Subject: [PATCH 39/44] fix(log): test --- compio-log/Cargo.toml | 3 +++ compio-log/tests/test.rs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/compio-log/Cargo.toml b/compio-log/Cargo.toml index 65572092..9a0de18f 100644 --- a/compio-log/Cargo.toml +++ b/compio-log/Cargo.toml @@ -12,5 +12,8 @@ repository = { workspace = true } [dependencies] tracing = { version = "0.1", default-features = false } +[dev-dependencies] +tracing-subscriber = "0.3" + [features] enable_log = [] diff --git a/compio-log/tests/test.rs b/compio-log/tests/test.rs index a54c13a2..ca211a4b 100644 --- a/compio-log/tests/test.rs +++ b/compio-log/tests/test.rs @@ -2,7 +2,7 @@ use compio_log::Level; #[test] fn test_log() { - compio_log::subscriber::fmt() + tracing_subscriber::fmt() .with_max_level(Level::TRACE) .init(); From 6a993f59853072623b0bf37493ce284922992ca5 Mon Sep 17 00:00:00 2001 From: Ho 229 Date: Tue, 7 Nov 2023 20:50:12 +0800 Subject: [PATCH 40/44] feat(driver): add logging for iour --- compio-driver/Cargo.toml | 1 + compio-driver/src/iour/mod.rs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/compio-driver/Cargo.toml b/compio-driver/Cargo.toml index 9bfb1e65..8aa24cae 100644 --- a/compio-driver/Cargo.toml +++ b/compio-driver/Cargo.toml @@ -30,6 +30,7 @@ targets = [ [dependencies] # Workspace dependencies compio-buf = { workspace = true } +compio-log = { workspace = true } # Utils cfg-if = { workspace = true } diff --git a/compio-driver/src/iour/mod.rs b/compio-driver/src/iour/mod.rs index d4719d75..9559ac66 100644 --- a/compio-driver/src/iour/mod.rs +++ b/compio-driver/src/iour/mod.rs @@ -3,6 +3,7 @@ pub use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; use std::{collections::VecDeque, io, pin::Pin, task::Poll, time::Duration}; +use compio_log::{instrument, trace}; use io_uring::{ cqueue, opcode::AsyncCancel, @@ -34,6 +35,8 @@ impl Driver { const CANCEL: u64 = u64::MAX; pub fn new(builder: &ProactorBuilder) -> io::Result { + instrument!(compio_log::Level::TRACE, "new", ?builder); + trace!("new iour driver"); Ok(Self { inner: IoUring::new(builder.capacity)?, squeue: VecDeque::with_capacity(builder.capacity as usize), @@ -42,6 +45,7 @@ impl Driver { // Auto means that it choose to wait or not automatically. fn submit_auto(&mut self, timeout: Option, wait: bool) -> io::Result<()> { + instrument!(compio_log::Level::TRACE, "submit_auto", ?timeout, wait); let res = if wait { // Last part of submission queue, wait till timeout. if let Some(duration) = timeout { @@ -54,6 +58,7 @@ impl Driver { } else { self.inner.submit() }; + trace!("submit result: {res:?}"); match res { Ok(_) => Ok(()), Err(e) => match e.raw_os_error() { @@ -65,12 +70,15 @@ impl Driver { } fn flush_submissions(&mut self) -> bool { + instrument!(compio_log::Level::TRACE, "flush_submissions"); + let mut ended_ops = false; let mut inner_squeue = self.inner.submission(); while !inner_squeue.is_full() { if self.squeue.len() <= inner_squeue.capacity() - inner_squeue.len() { + trace!("inner_squeue have enough space, flush all entries"); let (s1, s2) = self.squeue.as_slices(); unsafe { inner_squeue @@ -84,8 +92,10 @@ impl Driver { ended_ops = true; break; } else if let Some(entry) = self.squeue.pop_front() { + trace!("inner_squeue have not enough space, flush an entry"); unsafe { inner_squeue.push(&entry) }.expect("queue has enough space"); } else { + trace!("self.squeue is empty, skip"); ended_ops = true; break; } @@ -112,6 +122,8 @@ impl Driver { } pub fn cancel(&mut self, user_data: usize, _registry: &mut Slab) { + instrument!(compio_log::Level::TRACE, "cancel", user_data); + trace!("cancel RawOp"); self.squeue.push_back( AsyncCancel::new(user_data as _) .build() @@ -120,7 +132,9 @@ impl Driver { } pub fn push(&mut self, user_data: usize, op: &mut RawOp) -> Poll> { + instrument!(compio_log::Level::TRACE, "push", user_data); let op = op.as_pin(); + trace!("push RawOp"); self.squeue .push_back(op.create_entry().user_data(user_data as _)); Poll::Pending @@ -132,7 +146,9 @@ impl Driver { entries: &mut impl Extend, _registry: &mut Slab, ) -> io::Result<()> { + instrument!(compio_log::Level::TRACE, "poll", ?timeout); // Anyway we need to submit once, no matter there are entries in squeue. + trace!("start polling"); loop { let ended = self.flush_submissions(); @@ -141,6 +157,7 @@ impl Driver { self.poll_entries(entries); if ended { + trace!("polling ended"); break; } } From dc19f59909b88daae427e3d89a9f93c4b5af392e Mon Sep 17 00:00:00 2001 From: Ho 229 Date: Tue, 7 Nov 2023 20:50:43 +0800 Subject: [PATCH 41/44] chore(runtime): change log level to debug --- compio-runtime/src/runtime/mod.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/compio-runtime/src/runtime/mod.rs b/compio-runtime/src/runtime/mod.rs index bd84af95..4df31a24 100644 --- a/compio-runtime/src/runtime/mod.rs +++ b/compio-runtime/src/runtime/mod.rs @@ -9,7 +9,7 @@ use std::{ use async_task::{Runnable, Task}; use compio_driver::{AsRawFd, Entry, OpCode, Proactor, ProactorBuilder, PushEntry, RawFd}; -use compio_log::{instrument, trace}; +use compio_log::{debug, instrument}; use futures_util::future::Either; use send_wrapper::SendWrapper; use smallvec::SmallVec; @@ -123,10 +123,10 @@ impl Runtime { cx: &mut Context, user_data: Key, ) -> Poll> { - instrument!(compio_log::Level::TRACE, "poll_task", ?cx, ?user_data,); + instrument!(compio_log::Level::DEBUG, "poll_task", ?user_data,); let mut op_runtime = self.op_runtime.borrow_mut(); if op_runtime.has_result(*user_data) { - trace!("has result"); + debug!("has result"); let op = op_runtime.remove(*user_data); let res = self .driver @@ -136,7 +136,7 @@ impl Runtime { .expect("the result should have come"); Poll::Ready(res.map_buffer(|op| unsafe { op.into_op::() })) } else { - trace!("update waker"); + debug!("update waker"); op_runtime.update_waker(*user_data, cx.waker().clone()); Poll::Pending } @@ -144,31 +144,31 @@ impl Runtime { #[cfg(feature = "time")] pub fn poll_timer(&self, cx: &mut Context, key: usize) -> Poll<()> { - instrument!(compio_log::Level::TRACE, "poll_timer", ?cx, ?key); + instrument!(compio_log::Level::DEBUG, "poll_timer", ?cx, ?key); let mut timer_runtime = self.timer_runtime.borrow_mut(); if timer_runtime.contains(key) { - trace!("pending"); + debug!("pending"); timer_runtime.update_waker(key, cx.waker().clone()); Poll::Pending } else { - trace!("ready"); + debug!("ready"); Poll::Ready(()) } } fn poll(&self) { - instrument!(compio_log::Level::TRACE, "poll"); + instrument!(compio_log::Level::DEBUG, "poll"); #[cfg(not(feature = "time"))] let timeout = None; #[cfg(feature = "time")] let timeout = self.timer_runtime.borrow().min_timeout(); - trace!("timeout: {:?}", timeout); + debug!("timeout: {:?}", timeout); let mut entries = SmallVec::<[Entry; 1024]>::new(); let mut driver = self.driver.borrow_mut(); match driver.poll(timeout, &mut entries) { Ok(_) => { - trace!("poll driver ok, entries: {}", entries.len()); + debug!("poll driver ok, entries: {}", entries.len()); for entry in entries { self.op_runtime .borrow_mut() @@ -177,7 +177,7 @@ impl Runtime { } Err(e) => match e.kind() { io::ErrorKind::TimedOut | io::ErrorKind::Interrupted => { - trace!("expected error: {e}"); + debug!("expected error: {e}"); } _ => panic!("{:?}", e), }, From e54803277bee58144f436f9819aa59fc88837439 Mon Sep 17 00:00:00 2001 From: Ho 229 Date: Wed, 8 Nov 2023 19:03:06 +0800 Subject: [PATCH 42/44] feat(driver): add logging for iocp --- compio-driver/src/iocp/mod.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/compio-driver/src/iocp/mod.rs b/compio-driver/src/iocp/mod.rs index b1b3ae35..a90707f1 100644 --- a/compio-driver/src/iocp/mod.rs +++ b/compio-driver/src/iocp/mod.rs @@ -13,6 +13,7 @@ use std::{ }; use compio_buf::arrayvec::ArrayVec; +use compio_log::{instrument, trace}; use slab::Slab; use windows_sys::Win32::{ Foundation::{ @@ -155,10 +156,12 @@ impl Driver { const DEFAULT_CAPACITY: usize = 1024; pub fn new(builder: &ProactorBuilder) -> io::Result { + instrument!(compio_log::Level::TRACE, "new", ?builder); let mut data: WSADATA = unsafe { std::mem::zeroed() }; syscall!(SOCKET, WSAStartup(0x202, &mut data))?; let port = syscall!(BOOL, CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0))?; + trace!("new iocp driver at port: {port}"); let port = unsafe { OwnedHandle::from_raw_handle(port as _) }; Ok(Self { port, @@ -173,6 +176,7 @@ impl Driver { timeout: Option, iocp_entries: &mut ArrayVec, ) -> io::Result<()> { + instrument!(compio_log::Level::TRACE, "poll_impl", ?timeout); let mut recv_count = 0; let timeout = match timeout { Some(timeout) => timeout.as_millis() as u32, @@ -189,6 +193,7 @@ impl Driver { 0, ) )?; + trace!("recv_count: {recv_count}"); unsafe { iocp_entries.set_len(recv_count as _); } @@ -199,6 +204,7 @@ impl Driver { if iocp_entry.lpOverlapped.is_null() { // This entry is posted by `post_driver_nop`. let user_data = iocp_entry.lpCompletionKey; + trace!("entry {user_data} is posted by post_driver_nop"); let result = if self.cancelled.remove(&user_data) { Err(io::Error::from_raw_os_error(ERROR_OPERATION_ABORTED as _)) } else { @@ -208,6 +214,7 @@ impl Driver { } else { let transferred = iocp_entry.dwNumberOfBytesTransferred; // Any thin pointer is OK because we don't use the type of opcode. + trace!("entry transferred: {transferred}"); let overlapped_ptr: *mut Overlapped<()> = iocp_entry.lpOverlapped.cast(); let overlapped = unsafe { &*overlapped_ptr }; let res = if matches!( @@ -242,21 +249,27 @@ impl Driver { } pub fn cancel(&mut self, user_data: usize, registry: &mut Slab) { + instrument!(compio_log::Level::TRACE, "cancel", user_data); + trace!("cancel RawOp"); self.cancelled.insert(user_data); if let Some(op) = registry.get_mut(user_data) { let overlapped_ptr = op.as_mut_ptr(); let op = op.as_op_pin(); // It's OK to fail to cancel. + trace!("call OpCode::cancel"); unsafe { op.cancel(overlapped_ptr.cast()) }.ok(); } } pub fn push(&mut self, user_data: usize, op: &mut RawOp) -> Poll> { + instrument!(compio_log::Level::TRACE, "push", user_data); if self.cancelled.remove(&user_data) { + trace!("pushed RawOp already cancelled"); Poll::Ready(Err(io::Error::from_raw_os_error( ERROR_OPERATION_ABORTED as _, ))) } else { + trace!("push RawOp"); let optr = op.as_mut_ptr(); let op_pin = op.as_op_pin(); if op_pin.is_overlapped() { @@ -313,6 +326,7 @@ impl Driver { entries: &mut impl Extend, _registry: &mut Slab, ) -> io::Result<()> { + instrument!(compio_log::Level::TRACE, "poll", ?timeout); // Prevent stack growth. let mut iocp_entries = ArrayVec::::new(); self.poll_impl(timeout, &mut iocp_entries)?; @@ -325,7 +339,10 @@ impl Driver { entries.extend(iocp_entries.drain(..).filter_map(|e| self.create_entry(e))); } Err(e) => match e.kind() { - io::ErrorKind::TimedOut => break, + io::ErrorKind::TimedOut => { + trace!("poll timeout"); + break; + } _ => return Err(e), }, } From 89309ec5fe7dd78650097b9ac0f8c2f531242d04 Mon Sep 17 00:00:00 2001 From: Ho 229 Date: Thu, 9 Nov 2023 14:47:42 +0800 Subject: [PATCH 43/44] feat(driver): add logging for fusion --- compio-driver/src/poll/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compio-driver/src/poll/mod.rs b/compio-driver/src/poll/mod.rs index 3c6e3325..aad464ae 100644 --- a/compio-driver/src/poll/mod.rs +++ b/compio-driver/src/poll/mod.rs @@ -13,6 +13,7 @@ use std::{ time::Duration, }; +use compio_log::{instrument, trace}; use crossbeam_queue::SegQueue; pub(crate) use libc::{sockaddr_storage, socklen_t}; use polling::{Event, Events, Poller}; @@ -144,6 +145,8 @@ pub(crate) struct Driver { impl Driver { pub fn new(builder: &ProactorBuilder) -> io::Result { + instrument!(compio_log::Level::TRACE, "new", ?builder); + trace!("new poll driver"); let entries = builder.capacity as usize; // for the sake of consistency, use u32 like iour let events = if entries == 0 { Events::new() From e60a8549f061f0b2ae386e7a33296be587723025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AE=87=E9=80=B8?= Date: Sat, 11 Nov 2023 16:14:14 +0800 Subject: [PATCH 44/44] fix: fix merge missing compio-tls