container: Add support for re-exporting a fetched container

lib/Cargo.toml

@@ -53,6 +53,7 @@ zstd = { version = "0.13.1", features = ["pkg-config"] }
 
 indoc = { version = "2", optional = true }
 xshell = { version = "0.2", optional = true }
+similar-asserts = { version = "1.5.0", optional = true }
 
 [dev-dependencies]
 quickcheck = "1"
@@ -66,4 +67,4 @@ features = ["dox"]
 [features]
 docgen = ["clap_mangen"]
 dox = ["ostree/dox"]
-internal-testing-api = ["xshell", "indoc"]
+internal-testing-api = ["xshell", "indoc", "similar-asserts"]

lib/src/chunking.rs

@@ -30,7 +30,10 @@ pub(crate) const MAX_CHUNKS: u32 = 64;
 /// we will just drop down to one.
 const MIN_CHUNKED_LAYERS: u32 = 4;
 
-type RcStr = Rc<str>;
+/// A convenient alias for a reference-counted, immutable string.
+pub(crate) type RcStr = Rc<str>;
+/// Maps from a checksum to its size and file names (multiple in the case of
+/// hard links).
 pub(crate) type ChunkMapping = BTreeMap<RcStr, (u64, Vec<Utf8PathBuf>)>;
 // TODO type PackageSet = HashSet<RcStr>;
 
@@ -212,7 +215,7 @@ impl Chunk {
  }
  }
 
- fn move_obj(&mut self, dest: &mut Self, checksum: &str) -> bool {
+ pub(crate) fn move_obj(&mut self, dest: &mut Self, checksum: &str) -> bool {
  // In most cases, we expect the object to exist in the source. However, it's
  // conveneient here to simply ignore objects which were already moved into
  // a chunk.

lib/src/cli.rs

@@ -24,7 +24,7 @@ use std::process::Command;
 use tokio::sync::mpsc::Receiver;
 
 use crate::commit::container_commit;
-use crate::container::store::{ImportProgress, LayerProgress, PreparedImport};
+use crate::container::store::{ExportToOCIOpts, ImportProgress, LayerProgress, PreparedImport};
 use crate::container::{self as ostree_container, ManifestDiff};
 use crate::container::{Config, ImageReference, OstreeImageReference};
 use crate::sysroot::SysrootLock;
@@ -117,7 +117,13 @@ pub(crate) enum ContainerOpts {
  imgref: OstreeImageReference,
  },
 
- /// Wrap an ostree commit into a container
+ /// Wrap an ostree commit into a container image.
+ ///
+ /// The resulting container image will have a single layer, which is
+ /// very often not what's desired. To handle things more intelligently,
+ /// you will need to use (or create) a higher level tool that splits
+ /// content into distinct "chunks"; functionality for this is
+ /// exposed by the API but not CLI currently.
  #[clap(alias = "export")]
  Encapsulate {
  /// Path to the repository
@@ -277,6 +283,32 @@ pub(crate) enum ContainerImageOpts {
  imgref: OstreeImageReference,
  },
 
+ /// Re-export a fetched image.
+ ///
+ /// Unlike `encapsulate`, this verb handles layered images, and will
+ /// also automatically preserve chunked structure from the fetched image.
+ Reexport {
+ /// Path to the repository
+ #[clap(long, value_parser)]
+ repo: Utf8PathBuf,
+
+ /// Source image reference, e.g. registry:quay.io/exampleos/exampleos:latest
+ #[clap(value_parser = parse_base_imgref)]
+ src_imgref: ImageReference,
+
+ /// Destination image reference, e.g. registry:quay.io/exampleos/exampleos:latest
+ #[clap(value_parser = parse_base_imgref)]
+ dest_imgref: ImageReference,
+
+ #[clap(long)]
+ /// Path to Docker-formatted authentication file.
+ authfile: Option<PathBuf>,
+
+ /// Compress at the fastest level (e.g. gzip level 1)
+ #[clap(long)]
+ compression_fast: bool,
+ },
+
  /// Replace the detached metadata (e.g. to add a signature)
  ReplaceDetachedMetadata {
  /// Path to the source repository
@@ -969,6 +1001,29 @@ async fn run_from_opt(opt: Opt) -> Result<()> {
  let repo = parse_repo(&repo)?;
  container_store(&repo, &imgref, proxyopts, quiet, check).await
  }
+ ContainerImageOpts::Reexport {
+ repo,
+ src_imgref,
+ dest_imgref,
+ authfile,
+ compression_fast,
+ } => {
+ let repo = &parse_repo(&repo)?;
+ let opts = ExportToOCIOpts {
+ authfile,
+ skip_compression: compression_fast,
+ ..Default::default()
+ };
+ let digest = ostree_container::store::export(
+ repo,
+ &src_imgref,
+ &dest_imgref,
+ Some(opts),
+ )
+ .await?;
+ println!("Exported: {digest}");
+ Ok(())
+ }
  ContainerImageOpts::History { repo, imgref } => {
  let repo = parse_repo(&repo)?;
  container_history(&repo, &imgref).await

lib/src/container/encapsulate.rs

@@ -102,7 +102,7 @@ fn export_chunks(
 /// Write an ostree commit to an OCI blob
 #[context("Writing ostree root to blob")]
 #[allow(clippy::too_many_arguments)]
-fn export_chunked(
+pub(crate) fn export_chunked(
  repo: &ostree::Repo,
  commit: &str,
  ociw: &mut OciDir,

lib/src/container/store.rs

@@ -6,14 +6,18 @@
 //! base. See [`encapsulate`][`super::encapsulate()`] for more information on encaspulation of images.
 
 use super::*;
+use crate::chunking::{self, Chunk};
 use crate::logging::system_repo_journal_print;
 use crate::refescape;
 use crate::sysroot::SysrootLock;
 use crate::utils::ResultExt;
 use anyhow::{anyhow, Context};
 use camino::{Utf8Path, Utf8PathBuf};
+use cap_std_ext::cap_std;
 use cap_std_ext::cap_std::fs::{Dir, MetadataExt};
+use cap_std_ext::cmdext::CapStdExtCommandExt;
 use containers_image_proxy::{ImageProxy, OpenedImage};
+use flate2::Compression;
 use fn_error_context::context;
 use futures_util::TryFutureExt;
 use oci_spec::image::{self as oci_image, Descriptor, History, ImageConfiguration, ImageManifest};
@@ -1209,6 +1213,187 @@ pub async fn copy(
  Ok(())
 }
 
+/// Options controlling commit export into OCI
+#[derive(Clone, Debug, Default)]
+#[non_exhaustive]
+pub struct ExportToOCIOpts {
+ /// If true, do not perform gzip compression of the tar layers.
+ pub skip_compression: bool,
+ /// Path to Docker-formatted authentication file.
+ pub authfile: Option<std::path::PathBuf>,
+}
+
+/// The way we store "chunk" layers in ostree is by writing a commit
+/// whose filenames are their own object identifier. This function parses
+/// what is written by the `ImporterMode::ObjectSet` logic, turning
+/// it back into a "chunked" structure that is used by the export code.
+fn chunking_from_layer_committed(
+ repo: &ostree::Repo,
+ l: &Descriptor,
+ chunking: &mut chunking::Chunking,
+) -> Result<()> {
+ let mut chunk = Chunk::default();
+ let layer_ref = &ref_for_layer(l)?;
+ let root = repo.read_commit(&layer_ref, gio::Cancellable::NONE)?.0;
+ let e = root.enumerate_children(
+ "standard::name,standard::size",
+ gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS,
+ gio::Cancellable::NONE,
+ )?;
+ for child in e.clone() {
+ let child = &child?;
+ // The name here should be a valid checksum
+ let name = child.name();
+ // SAFETY: ostree doesn't give us non-UTF8 filenames
+ let name = Utf8Path::from_path(&name).unwrap();
+ ostree::validate_checksum_string(name.as_str())?;
+ chunking.remainder.move_obj(&mut chunk, name.as_str());
+ }
+ chunking.chunks.push(chunk);
+ Ok(())
+}
+
+/// Export an imported container image to a target OCI directory.
+#[context("Copying image")]
+pub(crate) fn export_to_oci(
+ repo: &ostree::Repo,
+ imgref: &ImageReference,
+ dest_oci: &Dir,
+ tag: Option<&str>,
+ opts: ExportToOCIOpts,
+) -> Result<Descriptor> {
+ let srcinfo = query_image(repo, imgref)?.ok_or_else(|| anyhow!("No such image"))?;
+ let (commit_layer, component_layers, remaining_layers) =
+ parse_manifest_layout(&srcinfo.manifest, &srcinfo.configuration)?;
+ let commit_chunk_ref = ref_for_layer(commit_layer)?;
+ let commit_chunk_rev = repo.require_rev(&commit_chunk_ref)?;
+ let mut chunking = chunking::Chunking::new(repo, &commit_chunk_rev)?;
+ for layer in component_layers {
+ chunking_from_layer_committed(repo, layer, &mut chunking)?;
+ }
+ // Unfortunately today we can't guarantee we reserialize the same tar stream
+ // or compression, so we'll need to generate a new copy of the manifest and config
+ // with the layers reset.
+ let mut new_manifest = srcinfo.manifest.clone();
+ new_manifest.layers_mut().clear();
+ let mut new_config = srcinfo.configuration.clone();
+ new_config.history_mut().clear();
+
+ let mut dest_oci = ocidir::OciDir::ensure(&dest_oci)?;
+
+ let opts = ExportOpts {
+ skip_compression: opts.skip_compression,
+ authfile: opts.authfile,
+ ..Default::default()
+ };
+
+ let mut labels = HashMap::new();
+
+ // Given the object chunking information we recomputed from what
+ // we found on disk, re-serialize to layers (tarballs).
+ export_chunked(
+ repo,
+ &srcinfo.base_commit,
+ &mut dest_oci,
+ &mut new_manifest,
+ &mut new_config,
+ &mut labels,
+ chunking,
+ &opts,
+ "",
+ )?;
+
+ // Now, handle the non-ostree layers; this is a simple conversion of
+ //
+ let compression = opts.skip_compression.then_some(Compression::none());
+ for (i, layer) in remaining_layers.iter().enumerate() {
+ let layer_ref = &ref_for_layer(layer)?;
+ let mut target_blob = dest_oci.create_raw_layer(compression)?;
+ // Sadly the libarchive stuff isn't exposed via Rust due to type unsafety,
+ // so we'll just fork off the CLI.
+ let repo_dfd = repo.dfd_borrow();
+ let repo_dir = cap_std_ext::cap_std::fs::Dir::reopen_dir(&repo_dfd)?;
+ let mut subproc = std::process::Command::new("ostree")
+ .args(["--repo=.", "export", layer_ref.as_str()])
+ .stdout(std::process::Stdio::piped())
+ .cwd_dir(repo_dir)
+ .spawn()?;
+ // SAFETY: we piped just above
+ let mut stdout = subproc.stdout.take().unwrap();
+ std::io::copy(&mut stdout, &mut target_blob).context("Creating blob")?;
+ let layer = target_blob.complete()?;
+ let previous_annotations = srcinfo
+ .manifest
+ .layers()
+ .get(i)
+ .and_then(|l| l.annotations().as_ref())
+ .cloned();
+ let previous_description = srcinfo
+ .configuration
+ .history()
+ .get(i)
+ .and_then(|h| h.comment().as_deref())
+ .unwrap_or_default();
+ dest_oci.push_layer(
+ &mut new_manifest,
+ &mut new_config,
+ layer,
+ previous_description,
+ previous_annotations,
+ )
+ }
+
+ let new_config = dest_oci.write_config(new_config)?;
+ new_manifest.set_config(new_config);
+
+ dest_oci.insert_manifest(new_manifest, tag, oci_image::Platform::default())
+}
+
+/// Given a container image reference which is stored in `repo`, export it to the
+/// target image location.
+#[context("Export")]
+pub async fn export(
+ repo: &ostree::Repo,
+ src_imgref: &ImageReference,
+ dest_imgref: &ImageReference,
+ opts: Option<ExportToOCIOpts>,
+) -> Result<String> {
+ let target_oci = dest_imgref.transport == Transport::OciDir;
+ let tempdir = if !target_oci {
+ let vartmp = cap_std::fs::Dir::open_ambient_dir("/var/tmp", cap_std::ambient_authority())?;
+ let td = cap_std_ext::cap_tempfile::TempDir::new_in(&vartmp)?;
+ // Always skip compression when making a temporary copy
+ let opts = ExportToOCIOpts {
+ skip_compression: true,
+ ..Default::default()
+ };
+ export_to_oci(repo, src_imgref, &td, None, opts)?;
+ td
+ } else {
+ let opts = opts.unwrap_or_default();
+ let (path, tag) = parse_oci_path_and_tag(dest_imgref.name.as_str());
+ tracing::debug!("using OCI path={path} tag={tag:?}");
+ let path = Dir::open_ambient_dir(path, cap_std::ambient_authority())
+ .with_context(|| format!("Opening {path}"))?;
+ let descriptor = export_to_oci(repo, src_imgref, &path, tag, opts)?;
+ return Ok(descriptor.digest().clone());
+ };
+ // Pass the temporary oci directory as the current working directory for the skopeo process
+ let target_fd = 3i32;
+ let tempoci = ImageReference {
+ transport: Transport::OciDir,
+ name: format!("/proc/self/fd/{target_fd}"),
+ };
+ let authfile = opts.as_ref().and_then(|o| o.authfile.as_deref());
+ skopeo::copy(
+ &tempoci,
+ dest_imgref,
+ authfile,
+ Some((std::sync::Arc::new(tempdir.try_clone()?.into()), target_fd)),
+ )
+ .await
+}
+
 /// Iterate over deployment commits, returning the manifests from
 /// commits which point to a container image.
 #[context("Listing deployment manifests")]