From 059a8f833c4e150a14950f16a9a3da079aa4111b Mon Sep 17 00:00:00 2001 From: grey Date: Tue, 14 May 2019 23:15:27 -0700 Subject: [PATCH 1/3] add tide-compression crate --- .travis.yml | 6 +- Cargo.toml | 1 + tide-compression/Cargo.toml | 30 ++ tide-compression/README.md | 16 ++ tide-compression/examples/simple.rs | 22 ++ tide-compression/src/lib.rs | 423 ++++++++++++++++++++++++++++ tide/src/context.rs | 5 + 7 files changed, 500 insertions(+), 3 deletions(-) create mode 100644 tide-compression/Cargo.toml create mode 100644 tide-compression/README.md create mode 100644 tide-compression/examples/simple.rs create mode 100644 tide-compression/src/lib.rs diff --git a/.travis.yml b/.travis.yml index 3e4d10906..7a662d4f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ before_script: | rustup component add rustfmt clippy script: | cargo fmt --all -- --check && - cargo clippy --all -- -D clippy::all && + cargo clippy --all --all-features -- -D clippy::all && cargo build --no-default-features --verbose && - cargo build --all --verbose && - cargo test --all --verbose + cargo build --all --all-features --verbose && + cargo test --all --all-features --verbose cache: cargo diff --git a/Cargo.toml b/Cargo.toml index 78f2f7172..6105ea9a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "tide", + "tide-compression", "examples", ] diff --git a/tide-compression/Cargo.toml b/tide-compression/Cargo.toml new file mode 100644 index 000000000..19bb2c92c --- /dev/null +++ b/tide-compression/Cargo.toml @@ -0,0 +1,30 @@ +[package] +authors = [ + "Tide Developers", +] +description = "Compression-related middleware for Tide" +documentation = "https://docs.rs/tide-compression" +keywords = ["tide", "web", "async", "middleware", "compression"] +categories = ["network-programming", "compression", "asynchronous"] +edition = "2018" +license = "MIT OR Apache-2.0" +name = "tide-compression" +readme = "README.md" +repository = "https://github.com/rustasync/tide" +version = "0.1.0" + +[dependencies] +tide = { path = "../tide" } +accept-encoding = "0.2.0-alpha.2" +bytes = "0.4.12" +futures-preview = "0.3.0-alpha.16" +http = "0.1" +http-service = "0.2.0" + +[dependencies.async-compression] +default-features = false +features = ["stream", "gzip", "zlib", "brotli", "zstd"] +version = "0.1.0-alpha.1" + +[dev-dependencies] +http-service-mock = "0.2.0" diff --git a/tide-compression/README.md b/tide-compression/README.md new file mode 100644 index 000000000..7f13f1f9f --- /dev/null +++ b/tide-compression/README.md @@ -0,0 +1,16 @@ +# tide-compression + +This crate provides compression-related middleware for Tide. + +## Examples + +Examples are in the `/examples` folder of this crate. + +__Simple Example__ + +You can test the simple example by running `cargo run --example simple` while in this crate's directory, and then running either of the following commands: + +```console +$ curl http://127.0.0.1:8000/ -v +$ curl http://127.0.0.1:8000/echo -v -d "why hello there" +``` diff --git a/tide-compression/examples/simple.rs b/tide-compression/examples/simple.rs new file mode 100644 index 000000000..981236353 --- /dev/null +++ b/tide-compression/examples/simple.rs @@ -0,0 +1,22 @@ +#![feature(async_await)] +use tide::{App, Context}; +use tide_compression::{Compression, Decompression, Encoding}; + +// Returns a portion of the lorem ipsum text. +async fn lorem_ipsum(_cx: Context<()>) -> String { + String::from("Lorem ipsum dolor sit amet, consectetur adipiscing elit.") +} + +// Echoes the request body in bytes. +async fn echo_bytes(mut cx: Context<()>) -> Vec { + cx.body_bytes().await.unwrap() +} + +pub fn main() { + let mut app = App::new(); + app.at("/").get(lorem_ipsum); + app.at("/echo").post(echo_bytes); + app.middleware(Compression::with_default(Encoding::Brotli)); + app.middleware(Decompression::new()); + app.serve("127.0.0.1:8000").unwrap(); +} diff --git a/tide-compression/src/lib.rs b/tide-compression/src/lib.rs new file mode 100644 index 000000000..e0f88be3a --- /dev/null +++ b/tide-compression/src/lib.rs @@ -0,0 +1,423 @@ +#![cfg_attr(feature = "nightly", deny(missing_docs))] +#![cfg_attr(feature = "nightly", feature(external_doc))] +#![cfg_attr(feature = "nightly", doc(include = "../README.md"))] +#![cfg_attr(test, deny(warnings))] +#![feature(async_await)] +#![deny( + nonstandard_style, + rust_2018_idioms, + future_incompatible, + missing_debug_implementations +)] + +pub use accept_encoding::Encoding; +use async_compression::stream; +use futures::future::BoxFuture; +use http::{header::CONTENT_ENCODING, status::StatusCode, HeaderMap}; +use http_service::{Body, Request}; +use tide::{ + middleware::{Middleware, Next}, + response::IntoResponse, + Context, Error, Response, +}; + +macro_rules! box_async { + {$($t:tt)*} => { + ::futures::future::FutureExt::boxed(async move { $($t)* }) + }; +} + +/// Encode settings for the compression middleware. +/// +/// This can be modified in the case that you want more control over the speed or quality of compression. +/// +/// For more information on how to configure each of these settings, see the async-compression crate. +#[derive(Debug)] +pub struct EncodeSettings { + /// Settings for gzip compression. + pub gzip: async_compression::flate2::Compression, + /// Settings for deflate compression. + pub deflate: async_compression::flate2::Compression, + /// Settings for brotli compression. Ranges from 0-11. (default: `11`) + pub brotli: u32, + /// Settings for zstd compression. Ranges from 1-21. (default: `3`) + pub zstd: i32, +} + +impl Default for EncodeSettings { + fn default() -> Self { + Self { + gzip: Default::default(), + deflate: Default::default(), + brotli: 11, + zstd: 3, + } + } +} + +/// Middleware for automatically handling outgoing response compression. +/// +/// This middleware currently supports HTTP compression using `gzip`, `deflate`, `br`, and `zstd`. +#[derive(Debug)] +pub struct Compression { + default_encoding: Encoding, + settings: EncodeSettings, +} + +impl Default for Compression { + fn default() -> Self { + Self::new() + } +} + +impl Compression { + /// Creates a new Compression middleware. The default encoding is [`Encoding::Identity`] (no encoding). + pub fn new() -> Self { + Self { + default_encoding: Encoding::Identity, + settings: Default::default(), + } + } + + /// Creates a new Compression middleware with a provided default encoding. + /// + /// This encoding will be selected if the client has not set the `Accept-Encoding` header or `*` is set as the most preferred encoding. + pub fn with_default(default_encoding: Encoding) -> Self { + Self { + default_encoding, + settings: Default::default(), + } + } + + /// Accesses a mutable handle to this middleware's [`EncodeSettings`]. + /// + /// This will allow you to configure this middleware's settings. + pub fn settings_mut(&mut self) -> &mut EncodeSettings { + &mut self.settings + } + + fn preferred_encoding(&self, headers: &HeaderMap) -> Result { + let encoding = match accept_encoding::parse(headers) { + Ok(encoding) => encoding, + Err(_) => return Err(Error::from(StatusCode::BAD_REQUEST)), + }; + Ok(encoding.unwrap_or(self.default_encoding)) + } + + /// Consumes the response and returns an encoded version of it. + fn encode(&self, mut res: Response, encoding: Encoding) -> Response { + if res.headers().get(CONTENT_ENCODING).is_some() || encoding == Encoding::Identity { + return res; // avoid double-encoding a given response + } + let body = std::mem::replace(res.body_mut(), Body::empty()); + match encoding { + Encoding::Gzip => { + let stream = stream::GzipEncoder::new(body, self.settings.gzip); + *res.body_mut() = Body::from_stream(stream); + } + Encoding::Deflate => { + let stream = stream::ZlibEncoder::new(body, self.settings.deflate); + *res.body_mut() = Body::from_stream(stream); + } + Encoding::Brotli => { + let stream = stream::BrotliEncoder::new(body, self.settings.brotli); + *res.body_mut() = Body::from_stream(stream); + } + Encoding::Zstd => { + let stream = stream::ZstdEncoder::new(body, self.settings.zstd); + *res.body_mut() = Body::from_stream(stream); + } + Encoding::Identity => unreachable!(), + }; + res.headers_mut() + .append(CONTENT_ENCODING, encoding.to_header_value()); + res + } +} + +impl Middleware for Compression { + fn handle<'a>(&'a self, cx: Context, next: Next<'a, Data>) -> BoxFuture<'a, Response> { + box_async! { + let encoding = match self.preferred_encoding(cx.headers()) { + Ok(encoding) => encoding, + Err(e) => return e.into_response(), + }; + let res = next.run(cx).await; + self.encode(res, encoding) + } + } +} + +/// Middleware for handling incoming request decompression. +/// +/// This middleware currently supports HTTP decompression under the `gzip`, `deflate`, `br`, and `zstd` algorithms. +#[derive(Debug, Default)] +pub struct Decompression {} + +impl Decompression { + /// Creates a new Decompression middleware. + pub fn new() -> Self { + Self {} + } + + fn parse_encoding(s: &str) -> Result { + match s { + "gzip" => Ok(Encoding::Gzip), + "deflate" => Ok(Encoding::Deflate), + "br" => Ok(Encoding::Brotli), + "zstd" => Ok(Encoding::Zstd), + "identity" => Ok(Encoding::Identity), + _ => Err(Error::from(StatusCode::UNSUPPORTED_MEDIA_TYPE)), + } + } + + fn decode(&self, req: &mut Request) -> Result<(), Error> { + let encodings = if let Some(hval) = req.headers().get(CONTENT_ENCODING) { + let hval = match hval.to_str() { + Ok(hval) => hval, + Err(_) => return Err(Error::from(StatusCode::BAD_REQUEST)), + }; + hval.split(',') + .map(str::trim) + .rev() // apply decodings in reverse order + .map(Decompression::parse_encoding) + .collect::, Error>>()? + } else { + return Ok(()); + }; + + for encoding in encodings { + match encoding { + Encoding::Gzip => { + let body = std::mem::replace(req.body_mut(), Body::empty()); + let stream = stream::GzipDecoder::new(body); + *req.body_mut() = Body::from_stream(stream); + } + Encoding::Deflate => { + let body = std::mem::replace(req.body_mut(), Body::empty()); + let stream = stream::ZlibDecoder::new(body); + *req.body_mut() = Body::from_stream(stream); + } + Encoding::Brotli => { + let body = std::mem::replace(req.body_mut(), Body::empty()); + let stream = stream::BrotliDecoder::new(body); + *req.body_mut() = Body::from_stream(stream); + } + Encoding::Zstd => { + let body = std::mem::replace(req.body_mut(), Body::empty()); + let stream = stream::ZstdDecoder::new(body); + *req.body_mut() = Body::from_stream(stream); + } + Encoding::Identity => (), + } + } + + // strip the content-encoding header + req.headers_mut().remove(CONTENT_ENCODING).unwrap(); + + Ok(()) + } +} + +impl Middleware for Decompression { + fn handle<'a>( + &'a self, + mut cx: Context, + next: Next<'a, Data>, + ) -> BoxFuture<'a, Response> { + box_async! { + match self.decode(cx.request_mut()) { + Ok(_) => (), + Err(e) => return e.into_response(), + }; + next.run(cx).await + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use async_compression::flate2; + use bytes::Bytes; + use futures::{ + executor::{block_on, block_on_stream}, + stream::StreamExt, + }; + use http::header::ACCEPT_ENCODING; + use http_service::Body; + use http_service_mock::make_server; + + async fn lorem_ipsum(_cx: Context<()>) -> String { + String::from(r#" + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam rutrum et risus sed egestas. Maecenas dapibus enim a posuere + semper. Cras venenatis et turpis quis aliquam. Suspendisse eget risus in libero tristique consectetur. Ut ut risus cursus, scelerisque + enim ac, tempus tellus. Vestibulum ac porta felis. Aenean fringilla posuere felis, in blandit enim tristique ut. Sed elementum iaculis + enim eu commodo. + "#) + } + + fn lorem_ipsum_bytes() -> Vec { + String::from(r#" + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam rutrum et risus sed egestas. Maecenas dapibus enim a posuere + semper. Cras venenatis et turpis quis aliquam. Suspendisse eget risus in libero tristique consectetur. Ut ut risus cursus, scelerisque + enim ac, tempus tellus. Vestibulum ac porta felis. Aenean fringilla posuere felis, in blandit enim tristique ut. Sed elementum iaculis + enim eu commodo. + "#).into_bytes() + } + + // Echoes the request body in bytes. + async fn echo_bytes(mut cx: Context<()>) -> Vec { + cx.body_bytes().await.unwrap() + } + + // Generates the app. + fn app() -> tide::App<()> { + let mut app = tide::App::new(); + app.at("/").get(lorem_ipsum); + app.at("/echo").post(echo_bytes); + app.middleware(Compression::new()); + app.middleware(Decompression::new()); + app + } + + // Generates a response given a string that represents the Accept-Encoding header value. + fn get_encoded_response(hval: &str) -> Response { + let app = app(); + let mut server = make_server(app.into_http_service()).unwrap(); + let req = http::Request::get("/") + .header(ACCEPT_ENCODING, hval) + .body(Body::empty()) + .unwrap(); + let res = server.simulate(req).unwrap(); + res + } + + // Generates a decoded response given a request body and the header value representing its encoding. + fn get_decoded_response(body: Body, hval: &str) -> Response { + let app = app(); + let mut server = make_server(app.into_http_service()).unwrap(); + let req = http::Request::post("/echo") + .header(CONTENT_ENCODING, hval) + .body(body) + .unwrap(); + let res = server.simulate(req).unwrap(); + res + } + + #[test] + fn compressed_gzip_response() { + let res = get_encoded_response("gzip"); + assert_eq!(res.status(), 200); + let body = res.into_body(); + let stream = stream::GzipDecoder::new(body); + let decompressed_body: Vec = block_on_stream(stream) + .map(Result::unwrap) + .flatten() + .collect(); + let lorem_ipsum = lorem_ipsum_bytes(); + assert_eq!(decompressed_body, lorem_ipsum); + } + + #[test] + fn compressed_deflate_response() { + let res = get_encoded_response("deflate"); + assert_eq!(res.status(), 200); + let body = res.into_body(); + let stream = stream::ZlibDecoder::new(body); + let decompressed_body: Vec = block_on_stream(stream) + .map(Result::unwrap) + .flatten() + .collect(); + let lorem_ipsum = lorem_ipsum_bytes(); + assert_eq!(decompressed_body, lorem_ipsum); + } + + #[test] + fn compressed_brotli_response() { + let res = get_encoded_response("br"); + assert_eq!(res.status(), 200); + let body = res.into_body(); + let stream = stream::BrotliDecoder::new(body); + let decompressed_body: Vec = block_on_stream(stream) + .map(Result::unwrap) + .flatten() + .collect(); + let lorem_ipsum = lorem_ipsum_bytes(); + assert_eq!(decompressed_body, lorem_ipsum); + } + + #[test] + fn compressed_zstd_response() { + let res = get_encoded_response("zstd"); + assert_eq!(res.status(), 200); + let body = res.into_body(); + let stream = stream::ZstdDecoder::new(body); + let decompressed_body: Vec = block_on_stream(stream) + .map(Result::unwrap) + .flatten() + .collect(); + let lorem_ipsum = lorem_ipsum_bytes(); + assert_eq!(decompressed_body, lorem_ipsum); + } + + #[test] + fn decompressed_gzip_response() { + let lorem_ipsum = lorem_ipsum_bytes(); + let req_body = Body::from_stream(stream::GzipEncoder::new( + futures::stream::iter(vec![lorem_ipsum]) + .map(Bytes::from) + .map(Ok), + flate2::Compression::default(), + )); + let res = get_decoded_response(req_body, "gzip"); + let body = block_on(res.into_body().into_vec()).unwrap(); + let lorem_ipsum = lorem_ipsum_bytes(); + assert_eq!(body, lorem_ipsum); + } + + #[test] + fn decompressed_deflate_response() { + let lorem_ipsum = lorem_ipsum_bytes(); + let req_body = Body::from_stream(stream::ZlibEncoder::new( + futures::stream::iter(vec![lorem_ipsum]) + .map(Bytes::from) + .map(Ok), + flate2::Compression::default(), + )); + let res = get_decoded_response(req_body, "deflate"); + let body = block_on(res.into_body().into_vec()).unwrap(); + let lorem_ipsum = lorem_ipsum_bytes(); + assert_eq!(body, lorem_ipsum); + } + + #[test] + fn decompressed_brotli_response() { + let lorem_ipsum = lorem_ipsum_bytes(); + let req_body = Body::from_stream(stream::BrotliEncoder::new( + futures::stream::iter(vec![lorem_ipsum]) + .map(Bytes::from) + .map(Ok), + 11, + )); + let res = get_decoded_response(req_body, "br"); + let body = block_on(res.into_body().into_vec()).unwrap(); + let lorem_ipsum = lorem_ipsum_bytes(); + assert_eq!(body, lorem_ipsum); + } + + #[test] + fn decompressed_zstd_response() { + let lorem_ipsum = lorem_ipsum_bytes(); + let req_body = Body::from_stream(stream::ZstdEncoder::new( + futures::stream::iter(vec![lorem_ipsum]) + .map(Bytes::from) + .map(Ok), + 3, + )); + let res = get_decoded_response(req_body, "zstd"); + let body = block_on(res.into_body().into_vec()).unwrap(); + let lorem_ipsum = lorem_ipsum_bytes(); + assert_eq!(body, lorem_ipsum); + } +} diff --git a/tide/src/context.rs b/tide/src/context.rs index 4c96cc896..543ac8a5d 100644 --- a/tide/src/context.rs +++ b/tide/src/context.rs @@ -55,6 +55,11 @@ impl Context { &self.request } + /// Access a mutable handle to the entire request. + pub fn request_mut(&mut self) -> &mut http_service::Request { + &mut self.request + } + /// Access app-global data. pub fn state(&self) -> &State { &self.state From 80f0af6ba0b6825a404ee33846c858fd7fda6a28 Mon Sep 17 00:00:00 2001 From: grey Date: Tue, 14 May 2019 23:44:12 -0700 Subject: [PATCH 2/3] use run instead of serve --- tide-compression/examples/simple.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tide-compression/examples/simple.rs b/tide-compression/examples/simple.rs index 981236353..2f5d0e174 100644 --- a/tide-compression/examples/simple.rs +++ b/tide-compression/examples/simple.rs @@ -18,5 +18,5 @@ pub fn main() { app.at("/echo").post(echo_bytes); app.middleware(Compression::with_default(Encoding::Brotli)); app.middleware(Decompression::new()); - app.serve("127.0.0.1:8000").unwrap(); + app.run("127.0.0.1:8000").unwrap(); } From 962fddc55ad2c98aa37715327a74d97726404bdb Mon Sep 17 00:00:00 2001 From: Allen Date: Tue, 14 May 2019 23:45:30 -0700 Subject: [PATCH 3/3] Update tide-compression/README.md Co-Authored-By: Wonwoo Choi --- tide-compression/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tide-compression/README.md b/tide-compression/README.md index 7f13f1f9f..c84f65b12 100644 --- a/tide-compression/README.md +++ b/tide-compression/README.md @@ -12,5 +12,5 @@ You can test the simple example by running `cargo run --example simple` while in ```console $ curl http://127.0.0.1:8000/ -v -$ curl http://127.0.0.1:8000/echo -v -d "why hello there" +$ echo 'why hello there' | gzip | curl -v --compressed -H 'Content-Encoding: gzip' 'http://127.0.0.1:8000/echo' --data-binary @- ```