From a861ec792659e3ed1130190ca5b534c938a22bb4 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 15 Mar 2023 07:53:30 +0800 Subject: [PATCH] rafe: enhance builder/merger to support RAFS in TARFS mode Enhance builder/merger to support RAFS in TARFS mode, so we can merge multiple RAFS filesystems in TARFS mode into one. Signed-off-by: Jiang Liu --- rafs/src/builder/core/bootstrap.rs | 3 +- rafs/src/builder/merge.rs | 81 +++++++++++++++++------------- rafs/src/metadata/chunk.rs | 10 ++-- rafs/src/metadata/direct_v6.rs | 18 +++++++ rafs/src/metadata/mod.rs | 8 +++ src/bin/nydus-image/main.rs | 1 + 6 files changed, 81 insertions(+), 40 deletions(-) diff --git a/rafs/src/builder/core/bootstrap.rs b/rafs/src/builder/core/bootstrap.rs index f889a812fc7..74b9f67ee9a 100644 --- a/rafs/src/builder/core/bootstrap.rs +++ b/rafs/src/builder/core/bootstrap.rs @@ -15,7 +15,7 @@ use crate::builder::{ ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, Tree, }; use crate::metadata::layout::{RafsBlobTable, RAFS_V5_ROOT_INODE}; -use crate::metadata::{RafsSuper, RafsSuperConfig}; +use crate::metadata::{RafsSuper, RafsSuperConfig, RafsSuperFlags}; pub(crate) const STARGZ_DEFAULT_BLOCK_SIZE: u32 = 4 << 20; @@ -295,6 +295,7 @@ impl Bootstrap { chunk_size: ctx.chunk_size, explicit_uidgid: ctx.explicit_uidgid, version: ctx.fs_version, + is_tarfs_mode: rs.meta.flags.contains(RafsSuperFlags::TARTFS_MODE), }; config.check_compatibility(&rs.meta)?; diff --git a/rafs/src/builder/merge.rs b/rafs/src/builder/merge.rs index 35ec5876dba..cbed3de08bc 100644 --- a/rafs/src/builder/merge.rs +++ b/rafs/src/builder/merge.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::collections::btree_map::Entry; use std::collections::HashMap; use std::collections::HashSet; use std::convert::TryFrom; @@ -15,7 +16,7 @@ use nydus_storage::device::{BlobFeatures, BlobInfo}; use super::{ ArtifactStorage, BlobContext, BlobManager, Bootstrap, BootstrapContext, BuildContext, - BuildOutput, ChunkSource, HashChunkDict, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec, + BuildOutput, ChunkSource, ConversionType, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec, }; use crate::metadata::{RafsInodeExt, RafsSuper, RafsVersion}; @@ -50,7 +51,7 @@ impl Merger { }) } - /// Generate the merged RAFS bootstrap for an image from per layer RAFS bootstraps. + /// Overlay multiple RAFS filesystems into a merged RAFS filesystem. /// /// # Arguments /// - sources: contains one or more per layer bootstraps in order of lower to higher. @@ -79,14 +80,6 @@ impl Merger { sources.len(), ); } - if let Some(toc_digests) = blob_toc_digests.as_ref() { - ensure!( - toc_digests.len() == sources.len(), - "number of toc digest entries {} doesn't match number of sources {}", - toc_digests.len(), - sources.len(), - ); - } if let Some(sizes) = blob_sizes.as_ref() { ensure!( sizes.len() == sources.len(), @@ -95,6 +88,14 @@ impl Merger { sources.len(), ); } + if let Some(toc_digests) = blob_toc_digests.as_ref() { + ensure!( + toc_digests.len() == sources.len(), + "number of toc digest entries {} doesn't match number of sources {}", + toc_digests.len(), + sources.len(), + ); + } if let Some(sizes) = blob_toc_sizes.as_ref() { ensure!( sizes.len() == sources.len(), @@ -106,14 +107,18 @@ impl Merger { let mut tree: Option = None; let mut blob_mgr = BlobManager::new(ctx.digester); - - // Load parent bootstrap let mut blob_idx_map = HashMap::new(); + let mut chunk_size = None; + let mut config = None; + let mut fs_version = RafsVersion::V6; let mut parent_layers = 0; + + // Load parent bootstrap if let Some(parent_bootstrap_path) = &parent_bootstrap_path { let (rs, _) = RafsSuper::load_from_file(parent_bootstrap_path, config_v2.clone(), false, false) .context(format!("load parent bootstrap {:?}", parent_bootstrap_path))?; + config = Some(rs.meta.get_config()); tree = Some(Tree::from_bootstrap(&rs, &mut ())?); let blobs = rs.superblock.get_blob_infos(); for blob in &blobs { @@ -124,22 +129,20 @@ impl Merger { parent_layers = blobs.len(); } - // Get the blobs come from chunk dict bootstrap. + // Get the blobs come from chunk dictionary. let mut chunk_dict_blobs = HashSet::new(); - let mut config = None; if let Some(chunk_dict_path) = &chunk_dict { let (rs, _) = RafsSuper::load_from_file(chunk_dict_path, config_v2.clone(), true, false) .context(format!("load chunk dict bootstrap {:?}", chunk_dict_path))?; - config = Some(rs.meta.get_config()); + config + .get_or_insert_with(|| rs.meta.get_config()) + .check_compatibility(&rs.meta)?; for blob in rs.superblock.get_blob_infos() { chunk_dict_blobs.insert(blob.blob_id().to_string()); } } - let mut fs_version = RafsVersion::V6; - let mut chunk_size = None; - for (layer_idx, bootstrap_path) in sources.iter().enumerate() { let (rs, _) = RafsSuper::load_from_file(bootstrap_path, config_v2.clone(), true, false) .context(format!("load bootstrap {:?}", bootstrap_path))?; @@ -151,23 +154,17 @@ impl Merger { ctx.compressor = rs.meta.get_compressor(); ctx.digester = rs.meta.get_digester(); ctx.explicit_uidgid = rs.meta.explicit_uidgid(); + if config.as_ref().unwrap().is_tarfs_mode { + ctx.conversion_type = ConversionType::TarToTarfs; + ctx.blob_features |= BlobFeatures::TARFS; + } let mut parent_blob_added = false; let blobs = &rs.superblock.get_blob_infos(); for blob in blobs { let mut blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?; - if let Some(chunk_size) = chunk_size { - ensure!( - chunk_size == blob_ctx.chunk_size, - "can not merge bootstraps with inconsistent chunk size, current bootstrap {:?} with chunk size {:x}, expected {:x}", - bootstrap_path, - blob_ctx.chunk_size, - chunk_size, - ); - } else { - chunk_size = Some(blob_ctx.chunk_size); - } - if chunk_dict_blobs.get(&blob.blob_id()).is_none() { + chunk_size = Some(blob_ctx.chunk_size); + if chunk_dict_blobs.contains(&blob.blob_id()) { // It is assumed that the `nydus-image create` at each layer and `nydus-image merge` commands // use the same chunk dict bootstrap. So the parent bootstrap includes multiple blobs, but // only at most one new blob, the other blobs should be from the chunk dict image. @@ -207,8 +204,8 @@ impl Merger { } } - if !blob_idx_map.contains_key(&blob.blob_id()) { - blob_idx_map.insert(blob.blob_id().clone(), blob_mgr.len()); + if let Entry::Vacant(e) = blob_idx_map.entry(blob.blob_id()) { + e.insert(blob_mgr.len()); blob_mgr.add_blob(blob_ctx); } } @@ -235,11 +232,15 @@ impl Merger { } // Set node's layer index to distinguish same inode number (from bootstrap) // between different layers. - node.layer_idx = u16::try_from(layer_idx).context(format!( + let idx = u16::try_from(layer_idx).context(format!( "too many layers {}, limited to {}", layer_idx, u16::MAX - ))? + parent_layers as u16; + ))?; + if parent_layers + idx as usize > u16::MAX as usize { + bail!("too many layers {}, limited to {}", layer_idx, u16::MAX); + } + node.layer_idx = idx + parent_layers as u16; node.overlay = Overlay::UpperAddition; match node.whiteout_type(WhiteoutSpec::Oci) { // Insert whiteouts at the head, so they will be handled first when @@ -254,8 +255,16 @@ impl Merger { tree.apply(node, true, WhiteoutSpec::Oci)?; } } else { - let mut dict = HashChunkDict::new(rs.meta.get_digester()); - tree = Some(Tree::from_bootstrap(&rs, &mut dict)?); + tree = Some(Tree::from_bootstrap(&rs, &mut ())?); + } + } + + if ctx.conversion_type == ConversionType::TarToTarfs { + if parent_layers > 0 { + bail!("merging RAFS in TARFS mode conflicts with `--parent-bootstrap`"); + } + if !chunk_dict_blobs.is_empty() { + bail!("merging RAFS in TARFS mode conflicts with `--chunk-dict`"); } } diff --git a/rafs/src/metadata/chunk.rs b/rafs/src/metadata/chunk.rs index 35e2a644356..88fe42dd34d 100644 --- a/rafs/src/metadata/chunk.rs +++ b/rafs/src/metadata/chunk.rs @@ -15,7 +15,7 @@ use nydus_utils::digest::RafsDigest; use crate::metadata::cached_v5::CachedChunkInfoV5; use crate::metadata::direct_v5::DirectChunkInfoV5; -use crate::metadata::direct_v6::DirectChunkInfoV6; +use crate::metadata::direct_v6::{DirectChunkInfoV6, TarfsChunkInfo}; use crate::metadata::layout::v5::RafsV5ChunkInfo; use crate::metadata::{RafsStore, RafsVersion}; use crate::RafsIoWrite; @@ -331,10 +331,12 @@ impl ChunkWrapper { *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); + } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { + *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { - *self = Self::V6(to_rafs_v5_chunk_info(cki_v5)); + *self = Self::V5(to_rafs_v5_chunk_info(cki_v5)); } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { - *self = Self::V6(to_rafs_v5_chunk_info(cki_v5)); + *self = Self::V5(to_rafs_v5_chunk_info(cki_v5)); } else { panic!("unknown chunk information struct"); } @@ -347,6 +349,8 @@ fn as_blob_v5_chunk_info(cki: &dyn BlobChunkInfo) -> &dyn BlobV5ChunkInfo { cki_v6 } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { cki_v6 + } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { + cki_v6 } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { cki_v5 } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { diff --git a/rafs/src/metadata/direct_v6.rs b/rafs/src/metadata/direct_v6.rs index 72cb2eff836..45ce48e22ca 100644 --- a/rafs/src/metadata/direct_v6.rs +++ b/rafs/src/metadata/direct_v6.rs @@ -1466,3 +1466,21 @@ impl BlobChunkInfo for TarfsChunkInfo { self } } + +impl BlobV5ChunkInfo for TarfsChunkInfo { + fn index(&self) -> u32 { + self.chunk_index + } + + fn file_offset(&self) -> u64 { + 0 + } + + fn flags(&self) -> BlobChunkFlags { + BlobChunkFlags::empty() + } + + fn as_base(&self) -> &dyn BlobChunkInfo { + self + } +} diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs index a8ffb02bba1..dd9f0006b61 100644 --- a/rafs/src/metadata/mod.rs +++ b/rafs/src/metadata/mod.rs @@ -369,6 +369,8 @@ pub struct RafsSuperConfig { pub chunk_size: u32, /// Whether `explicit_uidgid` enabled or not. pub explicit_uidgid: bool, + /// RAFS in TARFS mode. + pub is_tarfs_mode: bool, } impl RafsSuperConfig { @@ -405,6 +407,11 @@ impl RafsSuperConfig { ))); } + let is_tarfs_mode = meta.flags.contains(RafsSuperFlags::TARTFS_MODE); + if is_tarfs_mode != self.is_tarfs_mode { + return Err(einval!(format!("Using inconsistent RAFS TARFS mode"))); + } + Ok(()) } } @@ -519,6 +526,7 @@ impl RafsSuperMeta { digester: self.get_digester(), chunk_size: self.chunk_size, explicit_uidgid: self.explicit_uidgid(), + is_tarfs_mode: self.flags.contains(RafsSuperFlags::TARTFS_MODE), } } } diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 17bbacfc280..6b4cc4a6ae3 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -883,6 +883,7 @@ impl Command { digester, chunk_size, explicit_uidgid: !repeatable, + is_tarfs_mode: false, }; let rafs_config = Arc::new(build_ctx.configuration.as_ref().clone()); // The separate chunk dict bootstrap doesn't support blob accessible.