From 578fe7254950f68bbd83a3923e6957790e53b91d Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Fri, 17 Mar 2023 09:54:47 +0800 Subject: [PATCH] rafe: enhance builder/merger to support RAFS in TARFS mode Enhance builder/merger to support RAFS in TARFS mode, so we can merge multiple RAFS filesystems in TARFS mode into one. Signed-off-by: Jiang Liu --- rafs/src/builder/core/bootstrap.rs | 3 ++- rafs/src/builder/core/chunk_dict.rs | 1 + rafs/src/builder/merge.rs | 16 +++++++++++++++- rafs/src/metadata/chunk.rs | 10 +++++++--- rafs/src/metadata/direct_v6.rs | 18 ++++++++++++++++++ rafs/src/metadata/mod.rs | 8 ++++++++ src/bin/nydus-image/main.rs | 1 + 7 files changed, 52 insertions(+), 5 deletions(-) diff --git a/rafs/src/builder/core/bootstrap.rs b/rafs/src/builder/core/bootstrap.rs index f889a812fc7..74b9f67ee9a 100644 --- a/rafs/src/builder/core/bootstrap.rs +++ b/rafs/src/builder/core/bootstrap.rs @@ -15,7 +15,7 @@ use crate::builder::{ ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, Tree, }; use crate::metadata::layout::{RafsBlobTable, RAFS_V5_ROOT_INODE}; -use crate::metadata::{RafsSuper, RafsSuperConfig}; +use crate::metadata::{RafsSuper, RafsSuperConfig, RafsSuperFlags}; pub(crate) const STARGZ_DEFAULT_BLOCK_SIZE: u32 = 4 << 20; @@ -295,6 +295,7 @@ impl Bootstrap { chunk_size: ctx.chunk_size, explicit_uidgid: ctx.explicit_uidgid, version: ctx.fs_version, + is_tarfs_mode: rs.meta.flags.contains(RafsSuperFlags::TARTFS_MODE), }; config.check_compatibility(&rs.meta)?; diff --git a/rafs/src/builder/core/chunk_dict.rs b/rafs/src/builder/core/chunk_dict.rs index 06f9e1a3a17..b83cb9f2d7d 100644 --- a/rafs/src/builder/core/chunk_dict.rs +++ b/rafs/src/builder/core/chunk_dict.rs @@ -264,6 +264,7 @@ mod tests { digester: digest::Algorithm::Blake3, chunk_size: 0x100000, explicit_uidgid: true, + is_tarfs_mode: false, }; let dict = HashChunkDict::from_commandline_arg(path, Arc::new(ConfigV2::default()), &rafs_config) diff --git a/rafs/src/builder/merge.rs b/rafs/src/builder/merge.rs index 35ec5876dba..4cbe7cd0f80 100644 --- a/rafs/src/builder/merge.rs +++ b/rafs/src/builder/merge.rs @@ -15,7 +15,8 @@ use nydus_storage::device::{BlobFeatures, BlobInfo}; use super::{ ArtifactStorage, BlobContext, BlobManager, Bootstrap, BootstrapContext, BuildContext, - BuildOutput, ChunkSource, HashChunkDict, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec, + BuildOutput, ChunkSource, ConversionType, HashChunkDict, MetadataTreeBuilder, Overlay, Tree, + WhiteoutSpec, }; use crate::metadata::{RafsInodeExt, RafsSuper, RafsVersion}; @@ -151,6 +152,10 @@ impl Merger { ctx.compressor = rs.meta.get_compressor(); ctx.digester = rs.meta.get_digester(); ctx.explicit_uidgid = rs.meta.explicit_uidgid(); + if config.as_ref().unwrap().is_tarfs_mode { + ctx.conversion_type = ConversionType::TarToTarfs; + ctx.blob_features |= BlobFeatures::TARFS; + } let mut parent_blob_added = false; let blobs = &rs.superblock.get_blob_infos(); @@ -259,6 +264,15 @@ impl Merger { } } + if ctx.conversion_type == ConversionType::TarToTarfs { + if parent_layers > 0 { + bail!("merging RAFS in TARFS mode conflicts with `--parent-bootstrap`"); + } + if !chunk_dict_blobs.is_empty() { + bail!("merging RAFS in TARFS mode conflicts with `--chunk-dict`"); + } + } + // Safe to unwrap because there is at least one source bootstrap. let tree = tree.unwrap(); ctx.fs_version = fs_version; diff --git a/rafs/src/metadata/chunk.rs b/rafs/src/metadata/chunk.rs index 35e2a644356..88fe42dd34d 100644 --- a/rafs/src/metadata/chunk.rs +++ b/rafs/src/metadata/chunk.rs @@ -15,7 +15,7 @@ use nydus_utils::digest::RafsDigest; use crate::metadata::cached_v5::CachedChunkInfoV5; use crate::metadata::direct_v5::DirectChunkInfoV5; -use crate::metadata::direct_v6::DirectChunkInfoV6; +use crate::metadata::direct_v6::{DirectChunkInfoV6, TarfsChunkInfo}; use crate::metadata::layout::v5::RafsV5ChunkInfo; use crate::metadata::{RafsStore, RafsVersion}; use crate::RafsIoWrite; @@ -331,10 +331,12 @@ impl ChunkWrapper { *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); + } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { + *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { - *self = Self::V6(to_rafs_v5_chunk_info(cki_v5)); + *self = Self::V5(to_rafs_v5_chunk_info(cki_v5)); } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { - *self = Self::V6(to_rafs_v5_chunk_info(cki_v5)); + *self = Self::V5(to_rafs_v5_chunk_info(cki_v5)); } else { panic!("unknown chunk information struct"); } @@ -347,6 +349,8 @@ fn as_blob_v5_chunk_info(cki: &dyn BlobChunkInfo) -> &dyn BlobV5ChunkInfo { cki_v6 } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { cki_v6 + } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { + cki_v6 } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { cki_v5 } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { diff --git a/rafs/src/metadata/direct_v6.rs b/rafs/src/metadata/direct_v6.rs index 72cb2eff836..45ce48e22ca 100644 --- a/rafs/src/metadata/direct_v6.rs +++ b/rafs/src/metadata/direct_v6.rs @@ -1466,3 +1466,21 @@ impl BlobChunkInfo for TarfsChunkInfo { self } } + +impl BlobV5ChunkInfo for TarfsChunkInfo { + fn index(&self) -> u32 { + self.chunk_index + } + + fn file_offset(&self) -> u64 { + 0 + } + + fn flags(&self) -> BlobChunkFlags { + BlobChunkFlags::empty() + } + + fn as_base(&self) -> &dyn BlobChunkInfo { + self + } +} diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs index a8ffb02bba1..dd9f0006b61 100644 --- a/rafs/src/metadata/mod.rs +++ b/rafs/src/metadata/mod.rs @@ -369,6 +369,8 @@ pub struct RafsSuperConfig { pub chunk_size: u32, /// Whether `explicit_uidgid` enabled or not. pub explicit_uidgid: bool, + /// RAFS in TARFS mode. + pub is_tarfs_mode: bool, } impl RafsSuperConfig { @@ -405,6 +407,11 @@ impl RafsSuperConfig { ))); } + let is_tarfs_mode = meta.flags.contains(RafsSuperFlags::TARTFS_MODE); + if is_tarfs_mode != self.is_tarfs_mode { + return Err(einval!(format!("Using inconsistent RAFS TARFS mode"))); + } + Ok(()) } } @@ -519,6 +526,7 @@ impl RafsSuperMeta { digester: self.get_digester(), chunk_size: self.chunk_size, explicit_uidgid: self.explicit_uidgid(), + is_tarfs_mode: self.flags.contains(RafsSuperFlags::TARTFS_MODE), } } } diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 17bbacfc280..6b4cc4a6ae3 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -883,6 +883,7 @@ impl Command { digester, chunk_size, explicit_uidgid: !repeatable, + is_tarfs_mode: false, }; let rafs_config = Arc::new(build_ctx.configuration.as_ref().clone()); // The separate chunk dict bootstrap doesn't support blob accessible.