Skip to content

Commit

Permalink
Merge pull request #1147 from jiangliu/tarfs
Browse files Browse the repository at this point in the history 
Introduce new tarfs mode to Nydus
  • Loading branch information
@imeoer
imeoer authored Mar 16, 2023
2 parents 6638eee + 3891a51 commit c9d9b43
Show file tree
Hide file tree
Showing 22 changed files with 658 additions and 219 deletions.
4 changes: 4 additions & 0 deletions rafs/src/builder/core/blob.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ impl Blob {
Self::finalize_blob_data(ctx, blob_mgr, blob_writer)?;
}
ConversionType::TarToRef
| ConversionType::TarToTarfs
| ConversionType::TargzToRef
| ConversionType::EStargzToRef => {
// Use `sha256(tarball)` as `blob_id` for ref-type conversions.
Expand All @@ -68,6 +69,9 @@ impl Blob {
}
} else if let Some(tar_reader) = &ctx.blob_tar_reader {
blob_ctx.compressed_blob_size = tar_reader.position();
if ctx.conversion_type == ConversionType::TarToTarfs {
blob_ctx.uncompressed_blob_size = blob_ctx.compressed_blob_size;
}
if blob_ctx.blob_id.is_empty() {
let hash = tar_reader.get_hash_object();
blob_ctx.blob_id = format!("{:x}", hash.finalize());
Expand Down
6 changes: 4 additions & 2 deletions rafs/src/builder/core/bootstrap.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ impl Bootstrap {
let index = nodes.len() as u32 + 1;
let parent = &mut nodes[tree.node.index as usize - 1];
let parent_ino = parent.inode.ino();
let block_size = ctx.v6_block_size();

// Maybe the parent is not a directory in multi-layers build scenario, so we check here.
if parent.is_dir() {
Expand All @@ -162,7 +163,8 @@ impl Bootstrap {
parent.inode.set_child_index(index);
parent.inode.set_child_count(tree.children.len() as u32);
if ctx.fs_version.is_v6() {
parent.v6_set_dir_offset(bootstrap_ctx, tree.node.v6_dirent_size(tree)?)?;
let d_size = tree.node.v6_dirent_size(ctx, tree)?;
parent.v6_set_dir_offset(bootstrap_ctx, d_size, block_size)?;
}
}

Expand Down Expand Up @@ -216,7 +218,7 @@ impl Bootstrap {
if !child.node.is_dir() && ctx.fs_version.is_v6() {
child
.node
.v6_set_offset(bootstrap_ctx, v6_hardlink_offset)?;
.v6_set_offset(bootstrap_ctx, v6_hardlink_offset, block_size)?;
}

// Store node for bootstrap & blob dump.
Expand Down
40 changes: 28 additions & 12 deletions rafs/src/builder/core/context.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ use nydus_storage::meta::{
BlobMetaChunkInfo, ZranContextGenerator,
};
use nydus_utils::digest::DigestData;
use nydus_utils::{compress, digest, div_round_up, round_down_4k, BufReaderInfo};
use nydus_utils::{compress, digest, div_round_up, round_down, BufReaderInfo};

use super::node::{ChunkSource, Node};
use crate::builder::{
Expand Down Expand Up @@ -62,6 +62,7 @@ pub enum ConversionType {
TarToStargz,
TarToRafs,
TarToRef,
TarToTarfs,
}

impl Default for ConversionType {
Expand All @@ -85,6 +86,7 @@ impl FromStr for ConversionType {
"targz-ref" => Ok(Self::TargzToRef),
"tar-rafs" => Ok(Self::TarToRafs),
"tar-stargz" => Ok(Self::TarToStargz),
"tar-tarfs" => Ok(Self::TarToTarfs),
// kept for backward compatibility
"directory" => Ok(Self::DirectoryToRafs),
"stargz_index" => Ok(Self::EStargzIndexToRef),
Expand All @@ -106,8 +108,9 @@ impl fmt::Display for ConversionType {
ConversionType::TargzToStargz => write!(f, "targz-ref"),
ConversionType::TargzToRef => write!(f, "targz-ref"),
ConversionType::TarToRafs => write!(f, "tar-rafs"),
ConversionType::TarToStargz => write!(f, "tar-stargz"),
ConversionType::TarToRef => write!(f, "tar-ref"),
ConversionType::TarToStargz => write!(f, "tar-stargz"),
ConversionType::TarToTarfs => write!(f, "tar-tarfs"),
}
}
}
Expand All @@ -120,6 +123,7 @@ impl ConversionType {
| ConversionType::EStargzIndexToRef
| ConversionType::TargzToRef
| ConversionType::TarToRef
| ConversionType::TarToTarfs
)
}
}
Expand Down Expand Up @@ -478,6 +482,9 @@ impl BlobContext {
blob_ctx
.blob_meta_header
.set_cap_tar_toc(features.contains(BlobFeatures::CAP_TAR_TOC));
blob_ctx
.blob_meta_header
.set_tarfs(features.contains(BlobFeatures::TARFS));

blob_ctx
}
Expand Down Expand Up @@ -703,6 +710,7 @@ impl BlobContext {
/// Get offset of compressed blob, since current_compressed_offset
/// is always >= compressed_blob_size, we can safely subtract here.
pub fn compressed_offset(&self) -> u64 {
assert!(self.current_compressed_offset >= self.compressed_blob_size);
self.current_compressed_offset - self.compressed_blob_size
}
}
Expand Down Expand Up @@ -745,7 +753,9 @@ impl BlobManager {
ctx.digester,
);
blob_ctx.set_chunk_size(ctx.chunk_size);
blob_ctx.set_meta_info_enabled(ctx.fs_version == RafsVersion::V6);
blob_ctx.set_meta_info_enabled(
ctx.fs_version == RafsVersion::V6 && ctx.conversion_type != ConversionType::TarToTarfs,
);

Ok(blob_ctx)
}
Expand Down Expand Up @@ -990,19 +1000,22 @@ impl BootstrapContext {
// Try to find an used block with no less than `size` space left.
// If found it, return the offset where we can store data.
// If not, return 0.
pub(crate) fn allocate_available_block(&mut self, size: u64) -> u64 {
if size >= EROFS_BLOCK_SIZE_4096 {
pub(crate) fn allocate_available_block(&mut self, size: u64, block_size: u64) -> u64 {
if size >= block_size {
return 0;
}

let min_idx = div_round_up(size, EROFS_INODE_SLOT_SIZE as u64) as usize;
let max_idx = div_round_up(EROFS_BLOCK_SIZE_4096, EROFS_INODE_SLOT_SIZE as u64) as usize;
let max_idx = div_round_up(block_size, EROFS_INODE_SLOT_SIZE as u64) as usize;

for idx in min_idx..max_idx {
let blocks = &mut self.v6_available_blocks[idx];
if let Some(mut offset) = blocks.pop_front() {
offset += EROFS_BLOCK_SIZE_4096 - (idx * EROFS_INODE_SLOT_SIZE) as u64;
self.append_available_block(offset + (min_idx * EROFS_INODE_SLOT_SIZE) as u64);
offset += block_size - (idx * EROFS_INODE_SLOT_SIZE) as u64;
self.append_available_block(
offset + (min_idx * EROFS_INODE_SLOT_SIZE) as u64,
block_size,
);
return offset;
}
}
Expand All @@ -1011,11 +1024,11 @@ impl BootstrapContext {
}

// Append the block that `offset` belongs to corresponding deque.
pub(crate) fn append_available_block(&mut self, offset: u64) {
if offset % EROFS_BLOCK_SIZE_4096 != 0 {
let avail = EROFS_BLOCK_SIZE_4096 - offset % EROFS_BLOCK_SIZE_4096;
pub(crate) fn append_available_block(&mut self, offset: u64, block_size: u64) {
if offset % block_size != 0 {
let avail = block_size - offset % block_size;
let idx = avail as usize / EROFS_INODE_SLOT_SIZE;
self.v6_available_blocks[idx].push_back(round_down_4k(offset));
self.v6_available_blocks[idx].push_back(round_down(offset, block_size));
}
}
}
Expand Down Expand Up @@ -1114,6 +1127,9 @@ impl BuildContext {
blob_features |= BlobFeatures::HAS_TOC;
blob_features |= BlobFeatures::HAS_TAR_HEADER;
}
if conversion_type == ConversionType::TarToTarfs {
blob_features |= BlobFeatures::TARFS;
}

BuildContext {
blob_id,
Expand Down
52 changes: 34 additions & 18 deletions rafs/src/builder/core/node.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ use nydus_utils::digest::{DigestHasher, RafsDigest};
use nydus_utils::{div_round_up, event_tracer, root_tracer, try_round_up_4k, ByteSize};
use sha2::digest::Digest;

use crate::builder::{ArtifactWriter, BlobContext, BlobManager, BuildContext, ChunkDict, Overlay};
use crate::builder::{
ArtifactWriter, BlobContext, BlobManager, BuildContext, ChunkDict, ConversionType, Overlay,
};
use crate::metadata::chunk::ChunkWrapper;
use crate::metadata::inode::InodeWrapper;
use crate::metadata::layout::v6::EROFS_INODE_FLAT_PLAIN;
Expand Down Expand Up @@ -270,35 +272,45 @@ impl Node {
};

let chunk_data = &mut data_buf[0..uncompressed_size as usize];
let (chunk, chunk_info) = self.read_file_chunk(ctx, reader, chunk_data)?;
let (mut chunk, chunk_info) = self.read_file_chunk(ctx, reader, chunk_data)?;
if let Some(h) = inode_hasher.as_mut() {
h.digest_update(chunk.id().as_ref());
}

let mut chunk = match self.deduplicate_chunk(
ctx,
blob_mgr,
file_offset,
uncompressed_size,
chunk,
)? {
None => continue,
Some(c) => c,
};
// No need to perform chunk deduplication for tar-tarfs case.
if ctx.conversion_type != ConversionType::TarToTarfs {
chunk = match self.deduplicate_chunk(
ctx,
blob_mgr,
file_offset,
uncompressed_size,
chunk,
)? {
None => continue,
Some(c) => c,
};
}

let (blob_index, blob_ctx) = blob_mgr.get_or_create_current_blob(ctx)?;
let chunk_index = blob_ctx.alloc_chunk_index()?;
chunk.set_blob_index(blob_index);
chunk.set_index(chunk_index);
chunk.set_file_offset(file_offset);
self.dump_file_chunk(ctx, blob_ctx, blob_writer, chunk_data, &mut chunk)?;
if ctx.conversion_type == ConversionType::TarToTarfs {
chunk.set_uncompressed_offset(chunk.compressed_offset());
chunk.set_uncompressed_size(chunk.compressed_size());
} else {
self.dump_file_chunk(ctx, blob_ctx, blob_writer, chunk_data, &mut chunk)?;
}

let chunk = Arc::new(chunk);
blob_size += chunk.compressed_size() as u64;
blob_ctx.add_chunk_meta_info(&chunk, chunk_info)?;
blob_mgr
.layered_chunk_dict
.add_chunk(chunk.clone(), ctx.digester);
if ctx.conversion_type != ConversionType::TarToTarfs {
blob_ctx.add_chunk_meta_info(&chunk, chunk_info)?;
blob_mgr
.layered_chunk_dict
.add_chunk(chunk.clone(), ctx.digester);
}
self.chunks.push(NodeChunk {
source: ChunkSource::Build,
inner: chunk,
Expand Down Expand Up @@ -347,7 +359,11 @@ impl Node {
.with_context(|| format!("failed to read node file {:?}", self.path()))?;
}

chunk.set_id(RafsDigest::from_buf(buf, ctx.digester));
// For tar-tarfs case, no need to compute chunk id.
if ctx.conversion_type != ConversionType::TarToTarfs {
chunk.set_id(RafsDigest::from_buf(buf, ctx.digester));
}

Ok((chunk, chunk_info))
}

Expand Down
Loading

0 comments on commit c9d9b43

Please sign in to comment.