diff --git a/Cargo.toml b/Cargo.toml index 4cd76e4..1394fd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ocidir" description = "A Rust library for reading and writing OCI (opencontainers) layout directories" -version = "0.4.0" +version = "0.5.0" edition = "2021" license = "MIT OR Apache-2.0" repository = "https://github.com/containers/ocidir-rs" diff --git a/examples/custom_compressor.rs b/examples/custom_compressor.rs new file mode 100644 index 0000000..2360132 --- /dev/null +++ b/examples/custom_compressor.rs @@ -0,0 +1,61 @@ +/// Example that shows how to use a custom compression and media type for image layers. +/// The example below does no compression. +use std::{env, io, path::PathBuf}; + +use oci_spec::image::Platform; +use ocidir::{cap_std::fs::Dir, BlobWriter, OciDir, WriteComplete}; + +struct NoCompression<'a>(BlobWriter<'a>); + +impl io::Write for NoCompression<'_> { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.0.flush() + } +} + +impl<'a> WriteComplete> for NoCompression<'a> { + fn complete(self) -> io::Result> { + Ok(self.0) + } +} + +fn main() { + let dir = Dir::open_ambient_dir(env::temp_dir(), ocidir::cap_std::ambient_authority()).unwrap(); + let oci_dir = OciDir::ensure(dir).unwrap(); + + let mut manifest = oci_dir.new_empty_manifest().unwrap().build().unwrap(); + let mut config = ocidir::oci_spec::image::ImageConfigurationBuilder::default() + .build() + .unwrap(); + + // Add the src as a layer + let writer = oci_dir + .create_custom_layer( + |bw| Ok(NoCompression(bw)), + oci_spec::image::MediaType::ImageLayer, + ) + .unwrap(); + let mut builder = tar::Builder::new(writer); + builder.follow_symlinks(false); + + builder + .append_dir_all(".", PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src")) + .unwrap(); + + let layer = builder.into_inner().unwrap().complete().unwrap(); + oci_dir.push_layer(&mut manifest, &mut config, layer, "src", None); + + println!( + "Created image with manifest: {}", + manifest.to_string_pretty().unwrap() + ); + + // Add the image manifest + let _descriptor = oci_dir + .insert_manifest_and_config(manifest.clone(), config, None, Platform::default()) + .unwrap(); +} diff --git a/src/lib.rs b/src/lib.rs index 4e0859e..6286f07 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,8 @@ use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use std::fs::File; -use std::io::{prelude::*, BufReader}; +use std::io::{prelude::*, BufReader, BufWriter}; +use std::marker::PhantomData; use std::path::{Path, PathBuf}; use std::str::FromStr; use thiserror::Error; @@ -141,7 +142,7 @@ pub struct BlobWriter<'a> { /// Compute checksum hash: Hasher, /// Target file - target: Option>, + target: Option>>, size: u64, } @@ -154,13 +155,6 @@ impl Debug for BlobWriter<'_> { } } -/// Create an OCI tar+gzip layer. -pub struct GzipLayerWriter<'a>(Sha256Writer>>); - -#[cfg(feature = "zstd")] -/// Writer for a OCI tar+zstd layer. -pub struct ZstdLayerWriter<'a>(Sha256Writer>>); - #[derive(Debug)] /// An opened OCI directory. pub struct OciDir { @@ -280,17 +274,32 @@ impl OciDir { BlobWriter::new(&self.dir) } + /// Create a layer writer with a custom encoder and + /// media type + pub fn create_custom_layer<'a, W: WriteComplete>>( + &'a self, + create: impl FnOnce(BlobWriter<'a>) -> std::io::Result, + media_type: MediaType, + ) -> Result> { + let bw = BlobWriter::new(&self.dir)?; + Ok(LayerWriter::new(create(bw)?, media_type)) + } + /// Create a writer for a new gzip+tar blob; the contents /// are not parsed, but are expected to be a tarball. - pub fn create_gzip_layer(&self, c: Option) -> Result { - GzipLayerWriter::new(&self.dir, c) + pub fn create_gzip_layer<'a>( + &'a self, + c: Option, + ) -> Result>>> { + let creator = |bw: BlobWriter<'a>| Ok(GzEncoder::new(bw, c.unwrap_or_default())); + self.create_custom_layer(creator, MediaType::ImageLayerGzip) } /// Create a tar output stream, backed by a blob pub fn create_layer( &self, c: Option, - ) -> Result> { + ) -> Result>>> { Ok(tar::Builder::new(self.create_gzip_layer(c)?)) } @@ -299,8 +308,12 @@ impl OciDir { /// are not parsed, but are expected to be a tarball. /// /// This method is only available when the `zstd` feature is enabled. - pub fn create_layer_zstd(&self, compression_level: Option) -> Result { - ZstdLayerWriter::new(&self.dir, compression_level) + pub fn create_layer_zstd<'a>( + &'a self, + compression_level: Option, + ) -> Result>>> { + let creator = |bw: BlobWriter<'a>| zstd::Encoder::new(bw, compression_level.unwrap_or(0)); + self.create_custom_layer(creator, MediaType::ImageLayerZstd) } #[cfg(feature = "zstdmt")] @@ -312,12 +325,17 @@ impl OciDir { /// [zstd::Encoder::multithread]] /// /// This method is only available when the `zstdmt` feature is enabled. - pub fn create_layer_zstd_multithread( - &self, + pub fn create_layer_zstd_multithread<'a>( + &'a self, compression_level: Option, n_workers: u32, - ) -> Result { - ZstdLayerWriter::multithread(&self.dir, compression_level, n_workers) + ) -> Result>>> { + let creator = |bw: BlobWriter<'a>| { + let mut encoder = zstd::Encoder::new(bw, compression_level.unwrap_or(0))?; + encoder.multithread(n_workers)?; + Ok(encoder) + }; + self.create_custom_layer(creator, MediaType::ImageLayerZstd) } /// Add a layer to the top of the image stack. The firsh pushed layer becomes the root. @@ -655,7 +673,7 @@ impl<'a> BlobWriter<'a> { Ok(Self { hash: Hasher::new(MessageDigest::sha256())?, // FIXME add ability to choose filename after completion - target: Some(cap_tempfile::TempFile::new(ocidir)?), + target: Some(BufWriter::new(cap_tempfile::TempFile::new(ocidir)?)), size: 0, }) } @@ -684,7 +702,7 @@ impl<'a> BlobWriter<'a> { fn complete_as(mut self, sha256_digest: &str) -> Result { let destname = &format!("{}/{}", BLOBDIR, sha256_digest); let target = self.target.take().unwrap(); - target.replace(destname)?; + target.into_inner().unwrap().replace(destname)?; Ok(Blob { sha256: Sha256Digest::from_str(sha256_digest).unwrap(), size: self.size, @@ -700,14 +718,10 @@ impl<'a> BlobWriter<'a> { impl std::io::Write for BlobWriter<'_> { fn write(&mut self, srcbuf: &[u8]) -> std::io::Result { - self.hash.update(srcbuf)?; - self.target - .as_mut() - .unwrap() - .as_file_mut() - .write_all(srcbuf)?; - self.size += srcbuf.len() as u64; - Ok(srcbuf.len()) + let written = self.target.as_mut().unwrap().write(srcbuf)?; + self.hash.update(&srcbuf[..written])?; + self.size += written as u64; + Ok(written) } fn flush(&mut self) -> std::io::Result<()> { @@ -715,79 +729,73 @@ impl std::io::Write for BlobWriter<'_> { } } -impl<'a> GzipLayerWriter<'a> { - /// Create a writer for a gzip compressed layer blob. - fn new(ocidir: &'a Dir, c: Option) -> Result { - let bw = BlobWriter::new(ocidir)?; - let enc = flate2::write::GzEncoder::new(bw, c.unwrap_or_default()); - Ok(Self(Sha256Writer::new(enc))) - } +/// A writer that can be finalized to return an inner writer. +pub trait WriteComplete: Write { + fn complete(self) -> std::io::Result; +} - /// Consume this writer, flushing buffered data and put the blob in place. - pub fn complete(self) -> Result { - let (uncompressed_sha256, enc) = self.0.finish(); - let blob = enc.finish()?.complete()?; - Ok(Layer { - blob, - uncompressed_sha256, - media_type: MediaType::ImageLayerGzip, - }) +impl WriteComplete for GzEncoder +where + W: Write, +{ + fn complete(self) -> std::io::Result { + self.finish() } } -impl std::io::Write for GzipLayerWriter<'_> { - fn write(&mut self, data: &[u8]) -> std::io::Result { - self.0.write(data) +#[cfg(feature = "zstd")] +impl WriteComplete for zstd::Encoder<'_, W> +where + W: Write, +{ + fn complete(self) -> std::io::Result { + self.finish() } +} - fn flush(&mut self) -> std::io::Result<()> { - self.0.flush() - } +/// A writer for a layer. +pub struct LayerWriter<'a, W> +where + W: WriteComplete>, +{ + inner: Sha256Writer, + media_type: MediaType, + marker: PhantomData<&'a ()>, } -#[cfg(feature = "zstd")] -impl<'a> ZstdLayerWriter<'a> { - /// Create a writer for a gzip compressed layer blob. - fn new(ocidir: &'a Dir, c: Option) -> Result { - let bw = BlobWriter::new(ocidir)?; - let encoder = zstd::Encoder::new(bw, c.unwrap_or(0))?; - Ok(Self(Sha256Writer::new(encoder))) +impl<'a, W> LayerWriter<'a, W> +where + W: WriteComplete>, +{ + pub fn new(inner: W, media_type: oci_image::MediaType) -> Self { + Self { + inner: Sha256Writer::new(inner), + media_type, + marker: PhantomData, + } } - /// Consume this writer, flushing buffered data and put the blob in place. pub fn complete(self) -> Result { - let (uncompressed_sha256, enc) = self.0.finish(); - let blob = enc.finish()?.complete()?; + let (uncompressed_sha256, enc) = self.inner.finish(); + let blob = enc.complete()?.complete()?; Ok(Layer { blob, uncompressed_sha256, - media_type: MediaType::ImageLayerZstd, + media_type: self.media_type, }) } } -#[cfg(feature = "zstdmt")] -impl<'a> ZstdLayerWriter<'a> { - /// Create a writer for a zstd compressed layer blob, with multithreaded compression enabled. - /// - /// The `n_workers` parameter specifies the number of threads to use for compression, per - /// [Encoder::multithread]] - fn multithread(ocidir: &'a Dir, c: Option, n_workers: u32) -> Result { - let bw = BlobWriter::new(ocidir)?; - let mut encoder = zstd::Encoder::new(bw, c.unwrap_or(0))?; - encoder.multithread(n_workers)?; - Ok(Self(Sha256Writer::new(encoder))) - } -} - -#[cfg(feature = "zstd")] -impl std::io::Write for ZstdLayerWriter<'_> { +impl<'a, W> std::io::Write for LayerWriter<'a, W> +where + W: WriteComplete>, +{ fn write(&mut self, data: &[u8]) -> std::io::Result { - self.0.write(data) + self.inner.write(data) } fn flush(&mut self) -> std::io::Result<()> { - self.0.flush() + self.inner.flush() } }