diff --git a/lib/src/chunking.rs b/lib/src/chunking.rs new file mode 100644 index 00000000..9cec43ea --- /dev/null +++ b/lib/src/chunking.rs @@ -0,0 +1,296 @@ +//! Split an OSTree commit into separate chunks + +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use std::borrow::Borrow; +use std::collections::{BTreeMap, BTreeSet}; +use std::rc::Rc; + +use crate::objgv::*; +use anyhow::Result; +use camino::Utf8PathBuf; +use gvariant::aligned_bytes::TryAsAligned; +use gvariant::{Marker, Structure}; +use ostree; +use ostree::prelude::*; +use ostree::{gio, glib}; + +//const MODULES: &str = "/usr/lib/modules"; +const FIRMWARE: &str = "/usr/lib/firmware"; + +const QUERYATTRS: &str = "standard::name,standard::type"; + +/// Size in bytes of the smallest chunk we will emit. +// pub(crate) const MIN_CHUNK_SIZE: u32 = 10 * 1024; +/// Maximum number of layers (chunks) we will use. +// We take half the limit of 128. +// https://github.com/ostreedev/ostree-rs-ext/issues/69 +pub(crate) const MAX_CHUNKS: u32 = 64; + +#[derive(Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Clone)] +pub(crate) struct RcStr(Rc); + +impl Borrow for RcStr { + fn borrow(&self) -> &str { + &*self.0 + } +} + +impl From<&str> for RcStr { + fn from(s: &str) -> Self { + Self(Rc::from(s)) + } +} + +#[derive(Debug, Default)] +pub(crate) struct Chunk { + pub(crate) content: BTreeMap)>, + pub(crate) size: u64, +} + +#[derive(Debug)] +pub(crate) enum Meta { + DirTree(RcStr), + DirMeta(RcStr), +} + +impl Meta { + pub(crate) fn objtype(&self) -> ostree::ObjectType { + match self { + Meta::DirTree(_) => ostree::ObjectType::DirTree, + Meta::DirMeta(_) => ostree::ObjectType::DirMeta, + } + } + + pub(crate) fn checksum(&self) -> &str { + match self { + Meta::DirTree(v) => &*v.0, + Meta::DirMeta(v) => &*v.0, + } + } +} + +#[derive(Debug, Default)] +pub(crate) struct Chunking { + pub(crate) metadata_size: u64, + pub(crate) commit: Box, + pub(crate) meta: Vec, + pub(crate) remainder: Chunk, + pub(crate) chunks: Vec, +} + +// pub(crate) struct ChunkConfig { +// pub(crate) min_size: u32, +// pub(crate) max_chunks: u32, +// } +// +// impl Default for ChunkConfig { +// fn default() -> Self { +// Self { +// min_size: MIN_CHUNK_SIZE, +// max_chunks: MAX_CHUNKS, +// } +// } +// } + +#[derive(Default)] +struct Generation { + path: Utf8PathBuf, + metadata_size: u64, + meta: Vec, + dirtree_found: BTreeSet, + dirmeta_found: BTreeSet, +} + +fn generate_chunking_recurse( + repo: &ostree::Repo, + gen: &mut Generation, + chunk: &mut Chunk, + dt: &glib::Variant, +) -> Result<()> { + let dt = dt.data_as_bytes(); + let dt = dt.try_as_aligned()?; + let dt = gv_dirtree!().cast(dt); + let (files, dirs) = dt.to_tuple(); + // A reusable buffer to avoid heap allocating these + let mut hexbuf = [0u8; 64]; + for file in files { + let (name, csum) = file.to_tuple(); + let fpath = gen.path.join(name.to_str()); + hex::encode_to_slice(csum, &mut hexbuf)?; + let checksum = std::str::from_utf8(&hexbuf)?; + let (_, meta, _) = repo.load_file(checksum, gio::NONE_CANCELLABLE)?; + // SAFETY: We know this API returns this value; it only has a return nullable because the + // caller can pass NULL to skip it. + let meta = meta.unwrap(); + let size = meta.size() as u64; + let entry = chunk.content.entry(RcStr::from(checksum)).or_default(); + entry.0 = size; + let first = entry.1.is_empty(); + if first { + chunk.size += size; + } + entry.1.push(fpath); + } + for item in dirs { + let (name, contents_csum, meta_csum) = item.to_tuple(); + let name = name.to_str(); + // Extend our current path + gen.path.push(name); + hex::encode_to_slice(contents_csum, &mut hexbuf)?; + let checksum_s = std::str::from_utf8(&hexbuf)?; + if !gen.dirtree_found.contains(checksum_s) { + let checksum = RcStr::from(checksum_s); + gen.dirtree_found.insert(RcStr::clone(&checksum)); + gen.meta.push(Meta::DirTree(checksum)); + let child_v = repo.load_variant(ostree::ObjectType::DirTree, checksum_s)?; + gen.metadata_size += child_v.data_as_bytes().as_ref().len() as u64; + generate_chunking_recurse(repo, gen, chunk, &child_v)?; + } + hex::encode_to_slice(meta_csum, &mut hexbuf)?; + let checksum_s = std::str::from_utf8(&hexbuf)?; + if !gen.dirtree_found.contains(checksum_s) { + let checksum = RcStr::from(checksum_s); + gen.dirmeta_found.insert(RcStr::clone(&checksum)); + let child_v = repo.load_variant(ostree::ObjectType::DirMeta, checksum_s)?; + gen.metadata_size += child_v.data_as_bytes().as_ref().len() as u64; + gen.meta.push(Meta::DirMeta(checksum)); + } + // We did a push above, so pop must succeed. + assert!(gen.path.pop()); + } + Ok(()) +} + +impl Chunk { + fn new() -> Self { + Default::default() + } + + fn move_obj(&mut self, dest: &mut Self, checksum: &str) -> bool { + // In most cases, we expect the object to exist in the source. However, it's + // conveneient here to simply ignore objects which were already moved into + // a chunk. + if let Some((name, (size, paths))) = self.content.remove_entry(checksum) { + let v = dest.content.insert(name, (size, paths)); + debug_assert!(v.is_none()); + self.size -= size; + dest.size += size; + true + } else { + false + } + } + + // fn split(self) -> (Self, Self) { + // todo!() + // } +} + +impl Chunking { + /// Generate an initial single chunk. + pub(crate) fn new(repo: &ostree::Repo, rev: &str) -> Result { + // Find the target commit + let rev = repo.resolve_rev(rev, false)?.unwrap(); + + // Load and parse the commit object + let (commit_v, _) = repo.load_commit(&rev)?; + let commit_v = commit_v.data_as_bytes(); + let commit_v = commit_v.try_as_aligned()?; + let commit = gv_commit!().cast(commit_v); + let commit = commit.to_tuple(); + + // Find the root directory tree + let contents_checksum = &hex::encode(commit.6); + let contents_v = repo.load_variant(ostree::ObjectType::DirTree, contents_checksum)?; + + // Load it all into a single chunk + let mut gen: Generation = Default::default(); + gen.path = Utf8PathBuf::from("/"); + let mut chunk: Chunk = Default::default(); + generate_chunking_recurse(repo, &mut gen, &mut chunk, &contents_v)?; + + let chunking = Chunking { + commit: Box::from(rev.as_str()), + metadata_size: gen.metadata_size, + meta: gen.meta, + remainder: chunk, + ..Default::default() + }; + Ok(chunking) + } + + /// Find the object named by `path` in `src`, and move it to `dest`. + fn extend_chunk( + repo: &ostree::Repo, + src: &mut Chunk, + dest: &mut Chunk, + path: &ostree::RepoFile, + ) -> Result<()> { + let cancellable = gio::NONE_CANCELLABLE; + let ft = path.query_file_type(gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS, cancellable); + if ft == gio::FileType::Directory { + let e = path.enumerate_children( + QUERYATTRS, + gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS, + cancellable, + )?; + for child in e { + let childi = child?; + let child = path.child(childi.name()); + let child = child.downcast::().unwrap(); + Self::extend_chunk(repo, src, dest, &child)?; + } + } else { + let checksum = path.checksum().unwrap(); + src.move_obj(dest, checksum.as_str()); + } + Ok(()) + } + + /// Create a new chunk from the provided filesystem paths. + pub(crate) fn chunk_paths<'a>( + &mut self, + repo: &ostree::Repo, + paths: impl IntoIterator, + ) -> Result<()> { + // Do nothing if we've hit our max. + if self.chunks.len() as u32 == MAX_CHUNKS { + return Ok(()); + } + let cancellable = gio::NONE_CANCELLABLE; + let (root, _) = repo.read_commit(&self.commit, cancellable)?; + let root = root.downcast::().unwrap(); + let mut chunk = Chunk::new(); + for path in paths { + let child = root.resolve_relative_path(path); + if !child.query_exists(cancellable) { + continue; + } + let child = child.downcast::().unwrap(); + Self::extend_chunk(repo, &mut self.remainder, &mut chunk, &child)?; + } + self.chunks.push(chunk); + Ok(()) + } + + /// Apply built-in heuristics to automatically create chunks. + pub(crate) fn auto_chunk(&mut self, repo: &ostree::Repo) -> Result<()> { + self.chunk_paths(repo, [FIRMWARE])?; + Ok(()) + } +} + +pub(crate) fn print(src: &Chunking) { + println!("Metadata: {}", glib::format_size(src.metadata_size)); + for (n, chunk) in src.chunks.iter().enumerate() { + let sz = glib::format_size(chunk.size); + println!("Chunk {}: objects:{} size:{}", n, chunk.content.len(), sz); + } + let sz = glib::format_size(src.remainder.size); + println!( + "Remainder: objects:{} size:{}", + src.remainder.content.len(), + sz + ); +} diff --git a/lib/src/cli.rs b/lib/src/cli.rs index a0b2fbad..9f84ff25 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -13,7 +13,7 @@ use std::ffi::OsString; use structopt::StructOpt; use crate::container::store::{LayeredImageImporter, PrepareResult}; -use crate::container::{Config, ImageReference, UnencapsulateOptions, OstreeImageReference}; +use crate::container::{Config, ImageReference, OstreeImageReference, UnencapsulateOptions}; fn parse_imgref(s: &str) -> Result { OstreeImageReference::try_from(s) @@ -118,6 +118,10 @@ enum ContainerOpts { /// Corresponds to the Dockerfile `CMD` instruction. #[structopt(long)] cmd: Option>, + + #[structopt(long)] + /// Output in multiple blobs + ex_chunked: bool, }, /// Commands for working with (possibly layered, non-encapsulated) container images. @@ -206,6 +210,19 @@ struct ImaSignOpts { key: String, } +/// Experimental options +#[derive(Debug, StructOpt)] +enum ExperimentalOpts { + /// Print chunking + PrintChunks { + /// Path to the repository + #[structopt(long)] + repo: String, + /// The ostree ref or commt + rev: String, + }, +} + /// Toplevel options for extended ostree functionality. #[derive(Debug, StructOpt)] #[structopt(name = "ostree-ext")] @@ -217,6 +234,8 @@ enum Opt { Container(ContainerOpts), /// IMA signatures ImaSign(ImaSignOpts), + /// Experimental/debug CLI + Experimental(ExperimentalOpts), } /// Import a tar archive containing an ostree commit. @@ -310,13 +329,15 @@ async fn container_export( imgref: &ImageReference, labels: BTreeMap, cmd: Option>, + chunked: bool, ) -> Result<()> { let repo = &ostree::Repo::open_at(libc::AT_FDCWD, repo, gio::NONE_CANCELLABLE)?; let config = Config { labels: Some(labels), cmd, }; - let pushed = crate::container::encapsulate(repo, rev, &config, &imgref).await?; + let opts = Some(crate::container::ExportOpts { chunked }); + let pushed = crate::container::encapsulate(repo, rev, &config, opts, &imgref).await?; println!("{}", pushed); Ok(()) } @@ -417,6 +438,7 @@ where imgref, labels, cmd, + ex_chunked, } => { let labels: Result> = labels .into_iter() @@ -429,7 +451,8 @@ where Ok((k.to_string(), v.to_string())) }) .collect(); - container_export(&repo, &rev, &imgref, labels?, cmd).await + + container_export(&repo, &rev, &imgref, labels?, cmd, ex_chunked).await } ContainerOpts::Image(opts) => match opts { ContainerImageOpts::List { repo } => { @@ -476,5 +499,14 @@ where }, }, Opt::ImaSign(ref opts) => ima_sign(opts), + Opt::Experimental(ref opts) => match opts { + ExperimentalOpts::PrintChunks { repo, rev } => { + let repo = &ostree::Repo::open_at(libc::AT_FDCWD, &repo, gio::NONE_CANCELLABLE)?; + let mut chunks = crate::chunking::Chunking::new(repo, rev)?; + chunks.auto_chunk(repo)?; + crate::chunking::print(&chunks); + Ok(()) + } + }, } } diff --git a/lib/src/container/encapsulate.rs b/lib/src/container/encapsulate.rs index 6c20fba5..925a9e52 100644 --- a/lib/src/container/encapsulate.rs +++ b/lib/src/container/encapsulate.rs @@ -2,6 +2,7 @@ use super::ociwriter::OciWriter; use super::*; +use crate::chunking::Chunking; use crate::tar as ostree_tar; use anyhow::Context; use fn_error_context::context; @@ -34,6 +35,29 @@ fn export_ostree_ref( w.complete() } +/// Write an ostree commit to an OCI blob +#[context("Writing ostree root to blob")] +fn export_chunked( + repo: &ostree::Repo, + ociw: &mut OciWriter, + chunking: &Chunking, + compression: Option, +) -> Result<()> { + for layer in chunking.chunks.iter() { + let mut w = ociw.create_layer(compression)?; + ostree_tar::export_chunk(repo, layer, &mut w)?; + let w = w.into_inner()?; + let final_layer = w.complete()?; + ociw.push_layer(final_layer); + } + let mut w = ociw.create_layer(compression)?; + ostree_tar::export_final_chunk(repo, chunking, &mut w)?; + let w = w.into_inner()?; + let final_layer = w.complete()?; + ociw.push_layer(final_layer); + Ok(()) +} + /// Generate an OCI image from a given ostree root #[context("Building oci")] fn build_oci( @@ -41,6 +65,7 @@ fn build_oci( rev: &str, ocidir_path: &Path, config: &Config, + opts: ExportOpts, compression: Option, ) -> Result { // Explicitly error if the target exists @@ -54,6 +79,14 @@ fn build_oci( let commit_meta = &commit_v.child_value(0); let commit_meta = glib::VariantDict::new(Some(commit_meta)); + let chunking = if opts.chunked { + let mut c = crate::chunking::Chunking::new(repo, commit)?; + c.auto_chunk(repo)?; + Some(c) + } else { + None + }; + if let Some(version) = commit_meta.lookup_value("version", Some(glib::VariantTy::new("s").unwrap())) { @@ -73,8 +106,12 @@ fn build_oci( writer.set_cmd(&cmd); } - let rootfs_blob = export_ostree_ref(repo, commit, &mut writer, compression)?; - writer.push_layer(rootfs_blob); + if let Some(chunking) = chunking { + export_chunked(repo, &mut writer, &chunking, compression)?; + } else { + let rootfs_blob = export_ostree_ref(repo, commit, &mut writer, compression)?; + writer.push_layer(rootfs_blob); + } writer.complete()?; Ok(ImageReference { @@ -89,8 +126,10 @@ async fn build_impl( repo: &ostree::Repo, ostree_ref: &str, config: &Config, + opts: Option, dest: &ImageReference, ) -> Result { + let opts = opts.unwrap_or_default(); let compression = if dest.transport == Transport::ContainerStorage { Some(flate2::Compression::none()) } else { @@ -102,6 +141,7 @@ async fn build_impl( ostree_ref, Path::new(dest.name.as_str()), config, + opts, compression, )?; None @@ -115,7 +155,14 @@ async fn build_impl( None }; - let src = build_oci(repo, ostree_ref, Path::new(tempdest), config, compression)?; + let src = build_oci( + repo, + ostree_ref, + Path::new(tempdest), + config, + opts, + compression, + )?; let mut cmd = skopeo::new_cmd(); tracing::event!(Level::DEBUG, "Copying {} to {}", src, dest); @@ -149,6 +196,13 @@ async fn build_impl( } } +/// Options controlling commit export into OCI +#[derive(Debug, Default)] +pub struct ExportOpts { + /// Whether or not to generate multiple layers + pub chunked: bool, +} + /// Given an OSTree repository and ref, generate a container image. /// /// The returned `ImageReference` will contain a digested (e.g. `@sha256:`) version of the destination. @@ -156,7 +210,8 @@ pub async fn encapsulate>( repo: &ostree::Repo, ostree_ref: S, config: &Config, + opts: Option, dest: &ImageReference, ) -> Result { - build_impl(repo, ostree_ref.as_ref(), config, dest).await + build_impl(repo, ostree_ref.as_ref(), config, opts, dest).await } diff --git a/lib/src/container/store.rs b/lib/src/container/store.rs index 72a70740..82910bcf 100644 --- a/lib/src/container/store.rs +++ b/lib/src/container/store.rs @@ -109,7 +109,10 @@ pub struct CompletedImport { } // Given a manifest, compute its ostree ref name and cached ostree commit -fn query_layer(repo: &ostree::Repo, layer: oci_image::Descriptor) -> Result { +pub(crate) fn query_layer( + repo: &ostree::Repo, + layer: oci_image::Descriptor, +) -> Result { let ostree_ref = ref_for_layer(&layer)?; let commit = repo.resolve_rev(&ostree_ref, true)?.map(|s| s.to_string()); Ok(ManifestLayerState { diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 21b818b4..64382963 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -32,8 +32,10 @@ pub mod refescape; pub mod tar; pub mod tokio_util; +mod chunking; mod cmdext; pub(crate) mod objgv; + /// Prelude, intended for glob import. pub mod prelude { #[doc(hidden)] diff --git a/lib/src/tar/export.rs b/lib/src/tar/export.rs index 06b33e93..1dfff487 100644 --- a/lib/src/tar/export.rs +++ b/lib/src/tar/export.rs @@ -1,5 +1,7 @@ //! APIs for creating container images from OSTree commits +use crate::chunking; +use crate::chunking::Chunking; use crate::objgv::*; use anyhow::Result; use camino::{Utf8Path, Utf8PathBuf}; @@ -9,6 +11,7 @@ use gio::prelude::*; use gvariant::aligned_bytes::TryAsAligned; use gvariant::{Marker, Structure}; use ostree::gio; +use std::borrow::Borrow; use std::borrow::Cow; use std::collections::HashSet; use std::io::BufReader; @@ -193,7 +196,7 @@ impl<'a, W: std::io::Write> OstreeTarWriter<'a, W> { /// Write a content object, returning the path/header that should be used /// as a hard link to it in the target path. This matches how ostree checkouts work. - fn append_content(&mut self, checksum: &str) -> Result<(Utf8PathBuf, tar::Header)> { + fn append_content_obj(&mut self, checksum: &str) -> Result<(Utf8PathBuf, tar::Header)> { let path = object_path(ostree::ObjectType::File, checksum); let (instream, meta, xattrs) = self.repo.load_file(checksum, gio::NONE_CANCELLABLE)?; @@ -236,6 +239,18 @@ impl<'a, W: std::io::Write> OstreeTarWriter<'a, W> { Ok((path, target_header)) } + fn append_content_hardlink( + &mut self, + srcpath: &Utf8Path, + mut h: tar::Header, + dest: &Utf8Path, + ) -> Result<()> { + h.set_entry_type(tar::EntryType::Link); + h.set_link_name(srcpath)?; + self.out.append_data(&mut h, dest, &mut std::io::empty())?; + Ok(()) + } + /// Write a dirtree object. fn append_dirtree>( &mut self, @@ -264,13 +279,10 @@ impl<'a, W: std::io::Write> OstreeTarWriter<'a, W> { let name = name.to_str(); hex::encode_to_slice(csum, &mut hexbuf)?; let checksum = std::str::from_utf8(&hexbuf)?; - let (objpath, mut h) = self.append_content(checksum)?; - h.set_entry_type(tar::EntryType::Link); - h.set_link_name(&objpath)?; + let (objpath, h) = self.append_content_obj(checksum)?; let subpath = &dirpath.join(name); let subpath = map_path(subpath); - self.out - .append_data(&mut h, &*subpath, &mut std::io::empty())?; + self.append_content_hardlink(&objpath, h, &*subpath)?; } for item in dirs { @@ -319,6 +331,66 @@ pub fn export_commit(repo: &ostree::Repo, rev: &str, out: impl std::io::Write) - Ok(()) } +/// Output a chunk. +pub(crate) fn export_chunk( + repo: &ostree::Repo, + chunk: &chunking::Chunk, + out: &mut tar::Builder, +) -> Result<()> { + let writer = &mut OstreeTarWriter::new(repo, out); + writer.write_initial_directories()?; + for (checksum, (_size, paths)) in chunk.content.iter() { + let (objpath, h) = writer.append_content_obj(checksum.borrow())?; + for path in paths.iter() { + let path = path.strip_prefix("/").unwrap_or(path); + let h = h.clone(); + writer.append_content_hardlink(&objpath, h, path)?; + } + } + Ok(()) +} + +/// Output the last chunk in a chunking. +pub(crate) fn export_final_chunk( + repo: &ostree::Repo, + chunking: &Chunking, + out: &mut tar::Builder, +) -> Result<()> { + let cancellable = gio::NONE_CANCELLABLE; + let writer = &mut OstreeTarWriter::new(repo, out); + writer.write_initial_directories()?; + + let (commit_v, _) = repo.load_commit(&chunking.commit)?; + let commit_v = &commit_v; + writer.append(ostree::ObjectType::Commit, &chunking.commit, commit_v)?; + if let Some(commitmeta) = repo.read_commit_detached_metadata(&chunking.commit, cancellable)? { + writer.append( + ostree::ObjectType::CommitMeta, + &chunking.commit, + &commitmeta, + )?; + } + + // In the chunked case, the final layer has all ostree metadata objects. + for meta in &chunking.meta { + let objtype = meta.objtype(); + let checksum = meta.checksum(); + let v = repo.load_variant(objtype, checksum)?; + writer.append(objtype, checksum, &v)?; + } + + for (checksum, (_size, paths)) in chunking.remainder.content.iter() { + let (objpath, h) = writer.append_content_obj(checksum.borrow())?; + for path in paths.iter() { + let path = path.strip_prefix("/").unwrap_or(path); + let h = h.clone(); + writer.append_content_hardlink(&objpath, h, path)?; + } + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/lib/tests/it/main.rs b/lib/tests/it/main.rs index 75594fb8..ec12880e 100644 --- a/lib/tests/it/main.rs +++ b/lib/tests/it/main.rs @@ -326,9 +326,15 @@ async fn test_container_import_export() -> Result<()> { ), cmd: Some(vec!["/bin/bash".to_string()]), }; - let digest = ostree_ext::container::encapsulate(&fixture.srcrepo, TESTREF, &config, &srcoci_imgref) - .await - .context("exporting")?; + let digest = ostree_ext::container::encapsulate( + &fixture.srcrepo, + TESTREF, + &config, + None, + &srcoci_imgref, + ) + .await + .context("exporting")?; assert!(srcoci_path.exists()); let inspect = skopeo_inspect(&srcoci_imgref.to_string())?; @@ -555,9 +561,10 @@ async fn test_container_import_export_registry() -> Result<()> { cmd: Some(vec!["/bin/bash".to_string()]), ..Default::default() }; - let digest = ostree_ext::container::encapsulate(&fixture.srcrepo, TESTREF, &config, &src_imgref) - .await - .context("exporting to registry")?; + let digest = + ostree_ext::container::encapsulate(&fixture.srcrepo, TESTREF, &config, None, &src_imgref) + .await + .context("exporting to registry")?; let mut digested_imgref = src_imgref.clone(); digested_imgref.name = format!("{}@{}", src_imgref.name, digest);