Skip to content

Commit

Permalink
rafe: enhance builder/merger to support RAFS in TARFS mode
Browse files Browse the repository at this point in the history 
Enhance builder/merger to support RAFS in TARFS mode, so we can merge
multiple RAFS filesystems in TARFS mode into one.

Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>
  • Loading branch information
@jiangliu
jiangliu committed Mar 16, 2023
1 parent 2a55d3e commit 1012164
Show file tree
Hide file tree
Showing 7 changed files with 82 additions and 40 deletions.
3 changes: 2 additions & 1 deletion rafs/src/builder/core/bootstrap.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use crate::builder::{
ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, Tree,
};
use crate::metadata::layout::{RafsBlobTable, RAFS_V5_ROOT_INODE};
use crate::metadata::{RafsSuper, RafsSuperConfig};
use crate::metadata::{RafsSuper, RafsSuperConfig, RafsSuperFlags};

pub(crate) const STARGZ_DEFAULT_BLOCK_SIZE: u32 = 4 << 20;

Expand Down Expand Up @@ -295,6 +295,7 @@ impl Bootstrap {
chunk_size: ctx.chunk_size,
explicit_uidgid: ctx.explicit_uidgid,
version: ctx.fs_version,
is_tarfs_mode: rs.meta.flags.contains(RafsSuperFlags::TARTFS_MODE),
};
config.check_compatibility(&rs.meta)?;

Expand Down
1 change: 1 addition & 0 deletions rafs/src/builder/core/chunk_dict.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ mod tests {
digester: digest::Algorithm::Blake3,
chunk_size: 0x100000,
explicit_uidgid: true,
is_tarfs_mode: false,
};
let dict =
HashChunkDict::from_commandline_arg(path, Arc::new(ConfigV2::default()), &rafs_config)
Expand Down
81 changes: 45 additions & 36 deletions rafs/src/builder/merge.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0

use std::collections::btree_map::Entry;
use std::collections::HashMap;
use std::collections::HashSet;
use std::convert::TryFrom;
Expand All @@ -15,7 +16,7 @@ use nydus_storage::device::{BlobFeatures, BlobInfo};

use super::{
ArtifactStorage, BlobContext, BlobManager, Bootstrap, BootstrapContext, BuildContext,
BuildOutput, ChunkSource, HashChunkDict, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec,
BuildOutput, ChunkSource, ConversionType, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec,
};
use crate::metadata::{RafsInodeExt, RafsSuper, RafsVersion};

Expand Down Expand Up @@ -50,7 +51,7 @@ impl Merger {
})
}

/// Generate the merged RAFS bootstrap for an image from per layer RAFS bootstraps.
/// Overlay multiple RAFS filesystems into a merged RAFS filesystem.
///
/// # Arguments
/// - sources: contains one or more per layer bootstraps in order of lower to higher.
Expand Down Expand Up @@ -79,14 +80,6 @@ impl Merger {
sources.len(),
);
}
if let Some(toc_digests) = blob_toc_digests.as_ref() {
ensure!(
toc_digests.len() == sources.len(),
"number of toc digest entries {} doesn't match number of sources {}",
toc_digests.len(),
sources.len(),
);
}
if let Some(sizes) = blob_sizes.as_ref() {
ensure!(
sizes.len() == sources.len(),
Expand All @@ -95,6 +88,14 @@ impl Merger {
sources.len(),
);
}
if let Some(toc_digests) = blob_toc_digests.as_ref() {
ensure!(
toc_digests.len() == sources.len(),
"number of toc digest entries {} doesn't match number of sources {}",
toc_digests.len(),
sources.len(),
);
}
if let Some(sizes) = blob_toc_sizes.as_ref() {
ensure!(
sizes.len() == sources.len(),
Expand All @@ -106,14 +107,18 @@ impl Merger {

let mut tree: Option<Tree> = None;
let mut blob_mgr = BlobManager::new(ctx.digester);

// Load parent bootstrap
let mut blob_idx_map = HashMap::new();
let mut chunk_size = None;
let mut config = None;
let mut fs_version = RafsVersion::V6;
let mut parent_layers = 0;

// Load parent bootstrap
if let Some(parent_bootstrap_path) = &parent_bootstrap_path {
let (rs, _) =
RafsSuper::load_from_file(parent_bootstrap_path, config_v2.clone(), false, false)
.context(format!("load parent bootstrap {:?}", parent_bootstrap_path))?;
config = Some(rs.meta.get_config());
tree = Some(Tree::from_bootstrap(&rs, &mut ())?);
let blobs = rs.superblock.get_blob_infos();
for blob in &blobs {
Expand All @@ -124,22 +129,20 @@ impl Merger {
parent_layers = blobs.len();
}

// Get the blobs come from chunk dict bootstrap.
// Get the blobs come from chunk dictionary.
let mut chunk_dict_blobs = HashSet::new();
let mut config = None;
if let Some(chunk_dict_path) = &chunk_dict {
let (rs, _) =
RafsSuper::load_from_file(chunk_dict_path, config_v2.clone(), true, false)
.context(format!("load chunk dict bootstrap {:?}", chunk_dict_path))?;
config = Some(rs.meta.get_config());
config
.get_or_insert_with(|| rs.meta.get_config())
.check_compatibility(&rs.meta)?;
for blob in rs.superblock.get_blob_infos() {
chunk_dict_blobs.insert(blob.blob_id().to_string());
}
}

let mut fs_version = RafsVersion::V6;
let mut chunk_size = None;

for (layer_idx, bootstrap_path) in sources.iter().enumerate() {
let (rs, _) = RafsSuper::load_from_file(bootstrap_path, config_v2.clone(), true, false)
.context(format!("load bootstrap {:?}", bootstrap_path))?;
Expand All @@ -151,23 +154,17 @@ impl Merger {
ctx.compressor = rs.meta.get_compressor();
ctx.digester = rs.meta.get_digester();
ctx.explicit_uidgid = rs.meta.explicit_uidgid();
if config.as_ref().unwrap().is_tarfs_mode {
ctx.conversion_type = ConversionType::TarToTarfs;
ctx.blob_features |= BlobFeatures::TARFS;
}

let mut parent_blob_added = false;
let blobs = &rs.superblock.get_blob_infos();
for blob in blobs {
let mut blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?;
if let Some(chunk_size) = chunk_size {
ensure!(
chunk_size == blob_ctx.chunk_size,
"can not merge bootstraps with inconsistent chunk size, current bootstrap {:?} with chunk size {:x}, expected {:x}",
bootstrap_path,
blob_ctx.chunk_size,
chunk_size,
);
} else {
chunk_size = Some(blob_ctx.chunk_size);
}
if chunk_dict_blobs.get(&blob.blob_id()).is_none() {
chunk_size = Some(blob_ctx.chunk_size);
if chunk_dict_blobs.contains(&blob.blob_id()) {
// It is assumed that the `nydus-image create` at each layer and `nydus-image merge` commands
// use the same chunk dict bootstrap. So the parent bootstrap includes multiple blobs, but
// only at most one new blob, the other blobs should be from the chunk dict image.
Expand Down Expand Up @@ -207,8 +204,8 @@ impl Merger {
}
}

if !blob_idx_map.contains_key(&blob.blob_id()) {
blob_idx_map.insert(blob.blob_id().clone(), blob_mgr.len());
if let Entry::Vacant(e) = blob_idx_map.entry(blob.blob_id()) {
e.insert(blob_mgr.len());
blob_mgr.add_blob(blob_ctx);
}
}
Expand All @@ -235,11 +232,15 @@ impl Merger {
}
// Set node's layer index to distinguish same inode number (from bootstrap)
// between different layers.
node.layer_idx = u16::try_from(layer_idx).context(format!(
let idx = u16::try_from(layer_idx).context(format!(
"too many layers {}, limited to {}",
layer_idx,
u16::MAX
))? + parent_layers as u16;
))?;
if parent_layers + idx as usize > u16::MAX as usize {
bail!("too many layers {}, limited to {}", layer_idx, u16::MAX);
}
node.layer_idx = idx + parent_layers as u16;
node.overlay = Overlay::UpperAddition;
match node.whiteout_type(WhiteoutSpec::Oci) {
// Insert whiteouts at the head, so they will be handled first when
Expand All @@ -254,8 +255,16 @@ impl Merger {
tree.apply(node, true, WhiteoutSpec::Oci)?;
}
} else {
let mut dict = HashChunkDict::new(rs.meta.get_digester());
tree = Some(Tree::from_bootstrap(&rs, &mut dict)?);
tree = Some(Tree::from_bootstrap(&rs, &mut ())?);
}
}

if ctx.conversion_type == ConversionType::TarToTarfs {
if parent_layers > 0 {
bail!("merging RAFS in TARFS mode conflicts with `--parent-bootstrap`");
}
if !chunk_dict_blobs.is_empty() {
bail!("merging RAFS in TARFS mode conflicts with `--chunk-dict`");
}
}

Expand Down
10 changes: 7 additions & 3 deletions rafs/src/metadata/chunk.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use nydus_utils::digest::RafsDigest;

use crate::metadata::cached_v5::CachedChunkInfoV5;
use crate::metadata::direct_v5::DirectChunkInfoV5;
use crate::metadata::direct_v6::DirectChunkInfoV6;
use crate::metadata::direct_v6::{DirectChunkInfoV6, TarfsChunkInfo};
use crate::metadata::layout::v5::RafsV5ChunkInfo;
use crate::metadata::{RafsStore, RafsVersion};
use crate::RafsIoWrite;
Expand Down Expand Up @@ -331,10 +331,12 @@ impl ChunkWrapper {
*self = Self::V6(to_rafs_v5_chunk_info(cki_v6));
} else if let Some(cki_v6) = cki.as_any().downcast_ref::<DirectChunkInfoV6>() {
*self = Self::V6(to_rafs_v5_chunk_info(cki_v6));
} else if let Some(cki_v6) = cki.as_any().downcast_ref::<TarfsChunkInfo>() {
*self = Self::V6(to_rafs_v5_chunk_info(cki_v6));
} else if let Some(cki_v5) = cki.as_any().downcast_ref::<CachedChunkInfoV5>() {
*self = Self::V6(to_rafs_v5_chunk_info(cki_v5));
*self = Self::V5(to_rafs_v5_chunk_info(cki_v5));
} else if let Some(cki_v5) = cki.as_any().downcast_ref::<DirectChunkInfoV5>() {
*self = Self::V6(to_rafs_v5_chunk_info(cki_v5));
*self = Self::V5(to_rafs_v5_chunk_info(cki_v5));
} else {
panic!("unknown chunk information struct");
}
Expand All @@ -347,6 +349,8 @@ fn as_blob_v5_chunk_info(cki: &dyn BlobChunkInfo) -> &dyn BlobV5ChunkInfo {
cki_v6
} else if let Some(cki_v6) = cki.as_any().downcast_ref::<DirectChunkInfoV6>() {
cki_v6
} else if let Some(cki_v6) = cki.as_any().downcast_ref::<TarfsChunkInfo>() {
cki_v6
} else if let Some(cki_v5) = cki.as_any().downcast_ref::<CachedChunkInfoV5>() {
cki_v5
} else if let Some(cki_v5) = cki.as_any().downcast_ref::<DirectChunkInfoV5>() {
Expand Down
18 changes: 18 additions & 0 deletions rafs/src/metadata/direct_v6.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1466,3 +1466,21 @@ impl BlobChunkInfo for TarfsChunkInfo {
self
}
}

impl BlobV5ChunkInfo for TarfsChunkInfo {
fn index(&self) -> u32 {
self.chunk_index
}

fn file_offset(&self) -> u64 {
0
}

fn flags(&self) -> BlobChunkFlags {
BlobChunkFlags::empty()
}

fn as_base(&self) -> &dyn BlobChunkInfo {
self
}
}
8 changes: 8 additions & 0 deletions rafs/src/metadata/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,8 @@ pub struct RafsSuperConfig {
pub chunk_size: u32,
/// Whether `explicit_uidgid` enabled or not.
pub explicit_uidgid: bool,
/// RAFS in TARFS mode.
pub is_tarfs_mode: bool,
}

impl RafsSuperConfig {
Expand Down Expand Up @@ -405,6 +407,11 @@ impl RafsSuperConfig {
)));
}

let is_tarfs_mode = meta.flags.contains(RafsSuperFlags::TARTFS_MODE);
if is_tarfs_mode != self.is_tarfs_mode {
return Err(einval!(format!("Using inconsistent RAFS TARFS mode")));
}

Ok(())
}
}
Expand Down Expand Up @@ -519,6 +526,7 @@ impl RafsSuperMeta {
digester: self.get_digester(),
chunk_size: self.chunk_size,
explicit_uidgid: self.explicit_uidgid(),
is_tarfs_mode: self.flags.contains(RafsSuperFlags::TARTFS_MODE),
}
}
}
Expand Down
1 change: 1 addition & 0 deletions src/bin/nydus-image/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -883,6 +883,7 @@ impl Command {
digester,
chunk_size,
explicit_uidgid: !repeatable,
is_tarfs_mode: false,
};
let rafs_config = Arc::new(build_ctx.configuration.as_ref().clone());
// The separate chunk dict bootstrap doesn't support blob accessible.
Expand Down

0 comments on commit 1012164

Please sign in to comment.