Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

nydus-image: supports multiple versions of merge #489

Merged
merged 3 commits into from
Jun 13, 2022
Merged

nydus-image: supports multiple versions of merge #489

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9707185
nydus-image: supports multiple versions of merge
zyfjeff Jun 13, 2022
61dadb6
Graceful handle inconsistent fs version between layers
zyfjeff Jun 13, 2022
fc615e8
image-create: use try_from replace from for handle exception
zyfjeff Jun 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/bin/nydus-image/core/context.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ use nydus_utils::{compress, digest, div_round_up, round_down_4k};
use rafs::metadata::layout::v5::RafsV5BlobTable;
use rafs::metadata::layout::v6::{RafsV6BlobTable, EROFS_BLOCK_SIZE, EROFS_INODE_SLOT_SIZE};
use rafs::metadata::layout::RafsBlobTable;
use rafs::metadata::layout::{RAFS_SUPER_VERSION_V5, RAFS_SUPER_VERSION_V6};
use rafs::metadata::RafsSuperFlags;
use rafs::metadata::{Inode, RAFS_DEFAULT_CHUNK_SIZE, RAFS_MAX_CHUNK_SIZE};
use rafs::{RafsIoReader, RafsIoWrite};
Expand Down Expand Up @@ -51,6 +52,17 @@ impl Default for RafsVersion {
}
}

impl From<u32> for RafsVersion {
fn from(version: u32) -> Self {
if version == RAFS_SUPER_VERSION_V5 {
return RafsVersion::V5;
} else if version == RAFS_SUPER_VERSION_V6 {
return RafsVersion::V6;
}
RafsVersion::V5
}
}

impl RafsVersion {
#[allow(dead_code)]
pub fn is_v5(&self) -> bool {
Expand Down
78 changes: 48 additions & 30 deletions src/bin/nydus-image/merge.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ use anyhow::{Context, Result};

use nydus_utils::compress;
use nydus_utils::digest::{self};
use rafs::metadata::layout::RAFS_ROOT_INODE;
use rafs::metadata::{RafsInode, RafsMode, RafsSuper, RafsSuperMeta};

use crate::core::bootstrap::Bootstrap;
Expand Down Expand Up @@ -81,11 +80,22 @@ impl Merger {
chunk_dict_blobs.insert(blob.blob_id().to_string());
}
}

let mut fs_version = None;
for (layer_idx, bootstrap_path) in sources.iter().enumerate() {
let rs = RafsSuper::load_from_metadata(bootstrap_path, RafsMode::Direct, true)
.context(format!("load bootstrap {:?}", bootstrap_path))?;

match fs_version {
Some(version) => {
if version != rs.meta.version {
bail!(
"inconsistent fs version between layers, expected version {}",
version
);
}
}
None => fs_version = Some(rs.meta.version),
}
let current_flags = Flags::from_meta(&rs.meta);
if let Some(flags) = &flags {
if flags != &current_flags {
Expand Down Expand Up @@ -127,36 +137,42 @@ impl Merger {

if let Some(tree) = &mut tree {
let mut nodes = Vec::new();
rs.walk_dir(RAFS_ROOT_INODE, None, &mut |inode: &dyn RafsInode,
path: &Path|
-> Result<()> {
let mut node = MetadataTreeBuilder::parse_node(&rs, inode, path.to_path_buf())
.context(format!("parse node from bootstrap {:?}", bootstrap_path))?;
for chunk in &mut node.chunks {
let origin_blob_index = chunk.inner.blob_index() as usize;
// Set the blob index of chunk to real index in blob table of final bootstrap.
chunk.inner.set_blob_index(blob_idx_map[origin_blob_index]);
}
// Set node's layer index to distinguish same inode number (from bootstrap)
// between different layers.
node.layer_idx = u16::try_from(layer_idx).context(format!(
"too many layers {}, limited to {}",
layer_idx,
u16::MAX
))?;
node.overlay = Overlay::UpperAddition;
match node.whiteout_type(WhiteoutSpec::Oci) {
Some(_) => {
// Insert whiteouts at the head, so they will be handled first when
// applying to lower layer.
nodes.insert(0, node);
rs.walk_dir(
rs.superblock.root_ino(),
None,
&mut |inode: &dyn RafsInode, path: &Path| -> Result<()> {
let mut node =
MetadataTreeBuilder::parse_node(&rs, inode, path.to_path_buf())
.context(format!(
"parse node from bootstrap {:?}",
bootstrap_path
))?;
for chunk in &mut node.chunks {
let origin_blob_index = chunk.inner.blob_index() as usize;
// Set the blob index of chunk to real index in blob table of final bootstrap.
chunk.inner.set_blob_index(blob_idx_map[origin_blob_index]);
}
_ => {
nodes.push(node);
// Set node's layer index to distinguish same inode number (from bootstrap)
// between different layers.
node.layer_idx = u16::try_from(layer_idx).context(format!(
"too many layers {}, limited to {}",
layer_idx,
u16::MAX
))?;
node.overlay = Overlay::UpperAddition;
match node.whiteout_type(WhiteoutSpec::Oci) {
Some(_) => {
// Insert whiteouts at the head, so they will be handled first when
// applying to lower layer.
nodes.insert(0, node);
}
_ => {
nodes.push(node);
}
}
}
Ok(())
})?;
Ok(())
},
)?;
for node in &nodes {
tree.apply(node, true, WhiteoutSpec::Oci)?;
}
Expand All @@ -166,6 +182,8 @@ impl Merger {
}
}

// Safe to unwrap because a valid version must exist
ctx.fs_version = fs_version.unwrap().into();
// Safe to unwrap because there is at least one source bootstrap.
let mut tree = tree.unwrap();
let mut bootstrap = Bootstrap::new()?;
Expand Down