From 9707185cdbb940dcdfbb743ac7f8428cca4d2c2d Mon Sep 17 00:00:00 2001 From: zyfjeff Date: Mon, 13 Jun 2022 07:00:03 +0000 Subject: [PATCH 1/3] nydus-image: supports multiple versions of merge Signed-off-by: zyfjeff --- src/bin/nydus-image/core/context.rs | 12 +++++ src/bin/nydus-image/merge.rs | 71 ++++++++++++++++------------- 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/src/bin/nydus-image/core/context.rs b/src/bin/nydus-image/core/context.rs index 9e990116933..02cdcf0b458 100644 --- a/src/bin/nydus-image/core/context.rs +++ b/src/bin/nydus-image/core/context.rs @@ -24,6 +24,7 @@ use nydus_utils::{compress, digest, div_round_up, round_down_4k}; use rafs::metadata::layout::v5::RafsV5BlobTable; use rafs::metadata::layout::v6::{RafsV6BlobTable, EROFS_BLOCK_SIZE, EROFS_INODE_SLOT_SIZE}; use rafs::metadata::layout::RafsBlobTable; +use rafs::metadata::layout::{RAFS_SUPER_VERSION_V5, RAFS_SUPER_VERSION_V6}; use rafs::metadata::RafsSuperFlags; use rafs::metadata::{Inode, RAFS_DEFAULT_CHUNK_SIZE, RAFS_MAX_CHUNK_SIZE}; use rafs::{RafsIoReader, RafsIoWrite}; @@ -51,6 +52,17 @@ impl Default for RafsVersion { } } +impl From for RafsVersion { + fn from(version: u32) -> Self { + if version == RAFS_SUPER_VERSION_V5 { + return RafsVersion::V5; + } else if version == RAFS_SUPER_VERSION_V6 { + return RafsVersion::V6; + } + RafsVersion::V5 + } +} + impl RafsVersion { #[allow(dead_code)] pub fn is_v5(&self) -> bool { diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs index e2ac82916f2..9bc77f0f3b2 100644 --- a/src/bin/nydus-image/merge.rs +++ b/src/bin/nydus-image/merge.rs @@ -10,7 +10,6 @@ use anyhow::{Context, Result}; use nydus_utils::compress; use nydus_utils::digest::{self}; -use rafs::metadata::layout::RAFS_ROOT_INODE; use rafs::metadata::{RafsInode, RafsMode, RafsSuper, RafsSuperMeta}; use crate::core::bootstrap::Bootstrap; @@ -81,11 +80,11 @@ impl Merger { chunk_dict_blobs.insert(blob.blob_id().to_string()); } } - + let mut fs_versions = HashSet::new(); for (layer_idx, bootstrap_path) in sources.iter().enumerate() { let rs = RafsSuper::load_from_metadata(bootstrap_path, RafsMode::Direct, true) .context(format!("load bootstrap {:?}", bootstrap_path))?; - + fs_versions.insert(rs.meta.version); let current_flags = Flags::from_meta(&rs.meta); if let Some(flags) = &flags { if flags != ¤t_flags { @@ -127,36 +126,42 @@ impl Merger { if let Some(tree) = &mut tree { let mut nodes = Vec::new(); - rs.walk_dir(RAFS_ROOT_INODE, None, &mut |inode: &dyn RafsInode, - path: &Path| - -> Result<()> { - let mut node = MetadataTreeBuilder::parse_node(&rs, inode, path.to_path_buf()) - .context(format!("parse node from bootstrap {:?}", bootstrap_path))?; - for chunk in &mut node.chunks { - let origin_blob_index = chunk.inner.blob_index() as usize; - // Set the blob index of chunk to real index in blob table of final bootstrap. - chunk.inner.set_blob_index(blob_idx_map[origin_blob_index]); - } - // Set node's layer index to distinguish same inode number (from bootstrap) - // between different layers. - node.layer_idx = u16::try_from(layer_idx).context(format!( - "too many layers {}, limited to {}", - layer_idx, - u16::MAX - ))?; - node.overlay = Overlay::UpperAddition; - match node.whiteout_type(WhiteoutSpec::Oci) { - Some(_) => { - // Insert whiteouts at the head, so they will be handled first when - // applying to lower layer. - nodes.insert(0, node); + rs.walk_dir( + rs.superblock.root_ino(), + None, + &mut |inode: &dyn RafsInode, path: &Path| -> Result<()> { + let mut node = + MetadataTreeBuilder::parse_node(&rs, inode, path.to_path_buf()) + .context(format!( + "parse node from bootstrap {:?}", + bootstrap_path + ))?; + for chunk in &mut node.chunks { + let origin_blob_index = chunk.inner.blob_index() as usize; + // Set the blob index of chunk to real index in blob table of final bootstrap. + chunk.inner.set_blob_index(blob_idx_map[origin_blob_index]); } - _ => { - nodes.push(node); + // Set node's layer index to distinguish same inode number (from bootstrap) + // between different layers. + node.layer_idx = u16::try_from(layer_idx).context(format!( + "too many layers {}, limited to {}", + layer_idx, + u16::MAX + ))?; + node.overlay = Overlay::UpperAddition; + match node.whiteout_type(WhiteoutSpec::Oci) { + Some(_) => { + // Insert whiteouts at the head, so they will be handled first when + // applying to lower layer. + nodes.insert(0, node); + } + _ => { + nodes.push(node); + } } - } - Ok(()) - })?; + Ok(()) + }, + )?; for node in &nodes { tree.apply(node, true, WhiteoutSpec::Oci)?; } @@ -166,6 +171,10 @@ impl Merger { } } + assert!(fs_versions.len() == 1); + for v in fs_versions.drain() { + ctx.fs_version = v.into(); + } // Safe to unwrap because there is at least one source bootstrap. let mut tree = tree.unwrap(); let mut bootstrap = Bootstrap::new()?; From 61dadb6a257be35d52459bc0214d1e86c5e4e3a8 Mon Sep 17 00:00:00 2001 From: zyfjeff Date: Mon, 13 Jun 2022 07:42:39 +0000 Subject: [PATCH 2/3] Graceful handle inconsistent fs version between layers Signed-off-by: zyfjeff --- src/bin/nydus-image/merge.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs index 9bc77f0f3b2..83521d1661d 100644 --- a/src/bin/nydus-image/merge.rs +++ b/src/bin/nydus-image/merge.rs @@ -80,11 +80,22 @@ impl Merger { chunk_dict_blobs.insert(blob.blob_id().to_string()); } } - let mut fs_versions = HashSet::new(); + let mut fs_version = None; for (layer_idx, bootstrap_path) in sources.iter().enumerate() { let rs = RafsSuper::load_from_metadata(bootstrap_path, RafsMode::Direct, true) .context(format!("load bootstrap {:?}", bootstrap_path))?; - fs_versions.insert(rs.meta.version); + + match fs_version { + Some(version) => { + if version != rs.meta.version { + bail!( + "inconsistent fs version between layers, expected version {}", + version + ); + } + } + None => fs_version = Some(rs.meta.version), + } let current_flags = Flags::from_meta(&rs.meta); if let Some(flags) = &flags { if flags != ¤t_flags { @@ -171,10 +182,8 @@ impl Merger { } } - assert!(fs_versions.len() == 1); - for v in fs_versions.drain() { - ctx.fs_version = v.into(); - } + // Safe to unwrap because a valid version must exist + ctx.fs_version = fs_version.unwrap().into(); // Safe to unwrap because there is at least one source bootstrap. let mut tree = tree.unwrap(); let mut bootstrap = Bootstrap::new()?; From fc615e86382d32b1cb755cd487cd05bc0459e43f Mon Sep 17 00:00:00 2001 From: zyfjeff Date: Mon, 13 Jun 2022 07:59:11 +0000 Subject: [PATCH 3/3] image-create: use try_from replace from for handle exception Signed-off-by: zyfjeff --- src/bin/nydus-image/core/context.rs | 14 +++++++------- src/bin/nydus-image/merge.rs | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bin/nydus-image/core/context.rs b/src/bin/nydus-image/core/context.rs index 02cdcf0b458..fa43d22f8c6 100644 --- a/src/bin/nydus-image/core/context.rs +++ b/src/bin/nydus-image/core/context.rs @@ -23,8 +23,7 @@ use vmm_sys_util::tempfile::TempFile; use nydus_utils::{compress, digest, div_round_up, round_down_4k}; use rafs::metadata::layout::v5::RafsV5BlobTable; use rafs::metadata::layout::v6::{RafsV6BlobTable, EROFS_BLOCK_SIZE, EROFS_INODE_SLOT_SIZE}; -use rafs::metadata::layout::RafsBlobTable; -use rafs::metadata::layout::{RAFS_SUPER_VERSION_V5, RAFS_SUPER_VERSION_V6}; +use rafs::metadata::layout::{RafsBlobTable, RAFS_SUPER_VERSION_V5, RAFS_SUPER_VERSION_V6}; use rafs::metadata::RafsSuperFlags; use rafs::metadata::{Inode, RAFS_DEFAULT_CHUNK_SIZE, RAFS_MAX_CHUNK_SIZE}; use rafs::{RafsIoReader, RafsIoWrite}; @@ -52,14 +51,15 @@ impl Default for RafsVersion { } } -impl From for RafsVersion { - fn from(version: u32) -> Self { +impl TryFrom for RafsVersion { + type Error = Error; + fn try_from(version: u32) -> Result { if version == RAFS_SUPER_VERSION_V5 { - return RafsVersion::V5; + return Ok(RafsVersion::V5); } else if version == RAFS_SUPER_VERSION_V6 { - return RafsVersion::V6; + return Ok(RafsVersion::V6); } - RafsVersion::V5 + Err(anyhow!("invalid version {}", version)) } } diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs index 83521d1661d..bbb23b462ca 100644 --- a/src/bin/nydus-image/merge.rs +++ b/src/bin/nydus-image/merge.rs @@ -14,7 +14,7 @@ use rafs::metadata::{RafsInode, RafsMode, RafsSuper, RafsSuperMeta}; use crate::core::bootstrap::Bootstrap; use crate::core::chunk_dict::HashChunkDict; -use crate::core::context::ArtifactStorage; +use crate::core::context::{ArtifactStorage, RafsVersion}; use crate::core::context::{BlobContext, BlobManager, BootstrapContext, BuildContext}; use crate::core::node::{ChunkSource, Overlay, WhiteoutSpec}; use crate::core::tree::{MetadataTreeBuilder, Tree}; @@ -183,7 +183,7 @@ impl Merger { } // Safe to unwrap because a valid version must exist - ctx.fs_version = fs_version.unwrap().into(); + ctx.fs_version = RafsVersion::try_from(fs_version.unwrap())?; // Safe to unwrap because there is at least one source bootstrap. let mut tree = tree.unwrap(); let mut bootstrap = Bootstrap::new()?;