diff --git a/Cargo.lock b/Cargo.lock index 618b3f0..4a7eac6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,27 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "async-attributes" version = "1.1.2" @@ -29,6 +50,8 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b0122885821398cc923ece939e24d1056a2384ee719432397fa9db87230ff11" dependencies = [ + "brotli", + "flate2", "futures-core", "futures-io", "memchr", @@ -154,7 +177,7 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bincache" -version = "0.3.3" +version = "0.3.4" dependencies = [ "async-compression", "async-std", @@ -187,6 +210,27 @@ dependencies = [ "log", ] +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.13.0" @@ -223,6 +267,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -268,6 +321,16 @@ dependencies = [ "instant", ] +[[package]] +name = "flate2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "futures-channel" version = "0.3.28" @@ -449,6 +512,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "num_cpus" version = "1.15.0" diff --git a/bincache/Cargo.toml b/bincache/Cargo.toml index 61c067b..923804f 100644 --- a/bincache/Cargo.toml +++ b/bincache/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bincache" -version = "0.3.3" +version = "0.3.4" edition = "2021" description = "ZitaneLabs binary cache." license = "MIT" @@ -12,6 +12,8 @@ blocking = ["dep:futures-util", "async-compression/futures-io"] rt_tokio_1 = ["dep:tokio", "async-compression/tokio"] rt_async-std_1 = ["dep:async-std", "async-compression/futures-io"] comp_zstd = ["async-compression/zstd"] +comp_brotli = ["async-compression/brotli"] +comp_gzip = ["async-compression/gzip"] [dependencies] paste = "1" diff --git a/bincache/src/compression.rs b/bincache/src/compression.rs index 3231e80..f4e4343 100644 --- a/bincache/src/compression.rs +++ b/bincache/src/compression.rs @@ -5,10 +5,17 @@ pub use compression_level::CompressionLevel; /// A no-op compression strategy. pub const NO_COMPRESSION: Option = None; -// zstd - #[cfg(feature = "comp_zstd")] mod zstd_compressor; - #[cfg(feature = "comp_zstd")] pub use zstd_compressor::Zstd; + +#[cfg(feature = "comp_brotli")] +mod brotli_compressor; +#[cfg(feature = "comp_brotli")] +pub use brotli_compressor::Brotli; + +#[cfg(feature = "comp_gzip")] +mod gzip_compressor; +#[cfg(feature = "comp_gzip")] +pub use gzip_compressor::Gzip; diff --git a/bincache/src/compression/brotli_compressor.rs b/bincache/src/compression/brotli_compressor.rs new file mode 100644 index 0000000..e8195cd --- /dev/null +++ b/bincache/src/compression/brotli_compressor.rs @@ -0,0 +1,114 @@ +use super::compression_level::CompressionLevel; +use crate::traits::CompressionStrategy; +use crate::Result; +use async_trait::async_trait; +use std::borrow::Cow; + +#[derive(Debug)] +pub struct Brotli { + level: CompressionLevel, +} + +impl Brotli { + /// Creates a new Brotli Compressor with the given compression level + pub fn new(level: CompressionLevel) -> Self { + Self { level } + } +} + +impl Default for Brotli { + /// Creates a new Brotli Compressor with the default compression level + fn default() -> Self { + Self { + level: CompressionLevel::Default, + } + } +} + +#[async_trait] +impl CompressionStrategy for Brotli { + async fn compress<'a>(&self, data: Cow<'a, [u8]>) -> Result> { + #[cfg(feature = "rt_tokio_1")] + { + use async_compression::tokio::write; + use tokio::io::AsyncWriteExt; + let mut encoder = write::BrotliEncoder::with_quality( + Vec::with_capacity(data.len()), + self.level.into(), + ); + encoder.write_all(data.as_ref()).await?; + encoder.shutdown().await?; + return Ok(encoder.into_inner().into()); + } + #[cfg(any(feature = "blocking", feature = "implicit-blocking"))] + { + use async_compression::futures::write; + use futures_util::AsyncWriteExt; + let mut encoder = write::BrotliEncoder::with_quality( + Vec::with_capacity(data.len()), + self.level.into(), + ); + encoder.write_all(data.as_ref()).await?; + encoder.close().await?; + return Ok(encoder.into_inner().into()); + } + #[cfg(feature = "rt_async-std_1")] + { + use async_compression::futures::write; + use async_std::io::WriteExt; + let mut encoder = write::BrotliEncoder::with_quality( + Vec::with_capacity(data.len()), + self.level.into(), + ); + encoder.write_all(data.as_ref()).await?; + encoder.flush().await?; + return Ok(encoder.into_inner().into()); + } + } + + async fn decompress<'a>(&self, data: Cow<'a, [u8]>) -> Result> { + #[cfg(feature = "rt_tokio_1")] + { + use async_compression::tokio::write; + use tokio::io::AsyncWriteExt; + let mut encoder = write::BrotliDecoder::new(Vec::with_capacity(data.len())); + encoder.write_all(data.as_ref()).await?; + encoder.shutdown().await?; + return Ok(encoder.into_inner().into()); + } + #[cfg(any(feature = "blocking", feature = "implicit-blocking"))] + { + use async_compression::futures::write; + use futures_util::AsyncWriteExt; + let mut encoder = write::BrotliDecoder::new(Vec::with_capacity(data.len())); + encoder.write_all(data.as_ref()).await?; + encoder.close().await?; + return Ok(encoder.into_inner().into()); + } + #[cfg(feature = "rt_async-std_1")] + { + use async_compression::futures::write; + use async_std::io::WriteExt; + let mut encoder = write::BrotliDecoder::new(Vec::with_capacity(data.len())); + encoder.write_all(data.as_ref()).await?; + encoder.flush().await?; + return Ok(encoder.into_inner().into()); + } + } +} + +#[cfg(test)] +mod tests { + use super::Brotli; + use crate::{async_test, traits::CompressionStrategy, utils::test::create_arb_data}; + + async_test! { + async fn test_compression() { + let data = create_arb_data(1024); + let brotli = Brotli::default(); + let compressed = brotli.compress(data.clone().into()).await.unwrap(); + let decompressed = brotli.decompress(compressed).await.unwrap(); + assert_eq!(data.as_slice(), decompressed.as_ref()); + } + } +} diff --git a/bincache/src/compression/gzip_compressor.rs b/bincache/src/compression/gzip_compressor.rs new file mode 100644 index 0000000..3ca09f6 --- /dev/null +++ b/bincache/src/compression/gzip_compressor.rs @@ -0,0 +1,108 @@ +use super::compression_level::CompressionLevel; +use crate::traits::CompressionStrategy; +use crate::Result; +use async_trait::async_trait; +use std::borrow::Cow; + +#[derive(Debug)] +pub struct Gzip { + level: CompressionLevel, +} + +impl Gzip { + /// Creates a new Gzip Compressor with the given compression level + pub fn new(level: CompressionLevel) -> Self { + Self { level } + } +} + +impl Default for Gzip { + /// Creates a new Gzip Compressor with the default compression level + fn default() -> Self { + Self { + level: CompressionLevel::Default, + } + } +} + +#[async_trait] +impl CompressionStrategy for Gzip { + async fn compress<'a>(&self, data: Cow<'a, [u8]>) -> Result> { + #[cfg(feature = "rt_tokio_1")] + { + use async_compression::tokio::write; + use tokio::io::AsyncWriteExt; + let mut encoder = + write::GzipEncoder::with_quality(Vec::with_capacity(data.len()), self.level.into()); + encoder.write_all(data.as_ref()).await?; + encoder.shutdown().await?; + return Ok(encoder.into_inner().into()); + } + #[cfg(any(feature = "blocking", feature = "implicit-blocking"))] + { + use async_compression::futures::write; + use futures_util::AsyncWriteExt; + let mut encoder = + write::GzipEncoder::with_quality(Vec::with_capacity(data.len()), self.level.into()); + encoder.write_all(data.as_ref()).await?; + encoder.close().await?; + return Ok(encoder.into_inner().into()); + } + #[cfg(feature = "rt_async-std_1")] + { + use async_compression::futures::write; + use async_std::io::WriteExt; + let mut encoder = + write::GzipEncoder::with_quality(Vec::with_capacity(data.len()), self.level.into()); + encoder.write_all(data.as_ref()).await?; + encoder.flush().await?; + return Ok(encoder.into_inner().into()); + } + } + + async fn decompress<'a>(&self, data: Cow<'a, [u8]>) -> Result> { + #[cfg(feature = "rt_tokio_1")] + { + use async_compression::tokio::write; + use tokio::io::AsyncWriteExt; + let mut encoder = write::GzipDecoder::new(Vec::with_capacity(data.len())); + encoder.write_all(data.as_ref()).await?; + encoder.shutdown().await?; + return Ok(encoder.into_inner().into()); + } + #[cfg(any(feature = "blocking", feature = "implicit-blocking"))] + { + use async_compression::futures::write; + use futures_util::AsyncWriteExt; + let mut encoder = write::GzipDecoder::new(Vec::with_capacity(data.len())); + encoder.write_all(data.as_ref()).await?; + encoder.close().await?; + return Ok(encoder.into_inner().into()); + } + #[cfg(feature = "rt_async-std_1")] + { + use async_compression::futures::write; + use async_std::io::WriteExt; + let mut encoder = write::GzipDecoder::new(Vec::with_capacity(data.len())); + encoder.write_all(data.as_ref()).await?; + encoder.flush().await?; + return Ok(encoder.into_inner().into()); + } + } +} + +#[cfg(test)] +mod tests { + use super::Gzip; + use crate::{async_test, traits::CompressionStrategy, utils::test::create_arb_data}; + + async_test! { + async fn test_compression() { + let data = create_arb_data(1024); + let gzip = Gzip::default(); + let compressed = gzip.compress(data.clone().into()).await.unwrap(); + let decompressed = gzip.decompress(compressed).await.unwrap(); + assert_eq!(data.as_slice(), decompressed.as_ref()); + } + } +} diff --git a/bincache/src/compression/zstd_compressor.rs b/bincache/src/compression/zstd_compressor.rs index 257cea1..43bd9fb 100644 --- a/bincache/src/compression/zstd_compressor.rs +++ b/bincache/src/compression/zstd_compressor.rs @@ -13,14 +13,14 @@ pub struct Zstd { impl Zstd { /// Creates a new Zstd Compressor with the given compression level pub fn new(level: CompressionLevel) -> Self { - Zstd { level } + Self { level } } } impl Default for Zstd { /// Creates a new Zstd Compressor with the default compression level fn default() -> Self { - Zstd { + Self { level: CompressionLevel::Default, } } @@ -95,15 +95,7 @@ impl CompressionStrategy for Zstd { #[cfg(test)] mod tests { use super::Zstd; - use crate::{async_test, traits::CompressionStrategy}; - - fn create_arb_data(range: usize) -> Vec { - let mut vec = Vec::with_capacity(range); - for i in 0..range { - vec.push((i % 255) as u8); - } - vec - } + use crate::{async_test, traits::CompressionStrategy, utils::test::create_arb_data}; async_test! { async fn test_compression() { diff --git a/bincache/src/utils/test.rs b/bincache/src/utils/test.rs index 9436826..5da8ba2 100644 --- a/bincache/src/utils/test.rs +++ b/bincache/src/utils/test.rs @@ -1,2 +1,4 @@ mod temp_dir; pub use temp_dir::TempDir; +mod temp_arb_data; +pub use temp_arb_data::create_arb_data; diff --git a/bincache/src/utils/test/temp_arb_data.rs b/bincache/src/utils/test/temp_arb_data.rs new file mode 100644 index 0000000..6767c1d --- /dev/null +++ b/bincache/src/utils/test/temp_arb_data.rs @@ -0,0 +1,7 @@ +pub fn create_arb_data(range: usize) -> Vec { + let mut vec = Vec::with_capacity(range); + for i in 0..range { + vec.push((i % 255) as u8); + } + vec +}