Skip to content

Commit

Permalink
Merge pull request #1144 from jiangliu/prepare-tarfs
Browse files Browse the repository at this point in the history 
Refine builder and rafs to prepare for tarfs
  • Loading branch information
@imeoer
imeoer authored Mar 13, 2023
2 parents 3f33613 + bd837d0 commit 3213eb7
Show file tree
Hide file tree
Showing 12 changed files with 196 additions and 156 deletions.
16 changes: 8 additions & 8 deletions rafs/src/builder/core/context.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ use crate::builder::{
};
use crate::metadata::chunk::ChunkWrapper;
use crate::metadata::layout::v5::RafsV5BlobTable;
use crate::metadata::layout::v6::{RafsV6BlobTable, EROFS_BLOCK_SIZE, EROFS_INODE_SLOT_SIZE};
use crate::metadata::layout::v6::{RafsV6BlobTable, EROFS_BLOCK_SIZE_4096, EROFS_INODE_SLOT_SIZE};
use crate::metadata::layout::RafsBlobTable;
use crate::metadata::{Inode, RAFS_DEFAULT_CHUNK_SIZE};
use crate::metadata::{RafsSuperFlags, RafsVersion};
Expand Down Expand Up @@ -970,11 +970,11 @@ impl BootstrapContext {
layered,
inode_map: HashMap::new(),
nodes: Vec::new(),
offset: EROFS_BLOCK_SIZE,
offset: EROFS_BLOCK_SIZE_4096,
writer,
v6_available_blocks: vec![
VecDeque::new();
EROFS_BLOCK_SIZE as usize / EROFS_INODE_SLOT_SIZE
EROFS_BLOCK_SIZE_4096 as usize / EROFS_INODE_SLOT_SIZE
],
})
}
Expand All @@ -991,17 +991,17 @@ impl BootstrapContext {
// If found it, return the offset where we can store data.
// If not, return 0.
pub(crate) fn allocate_available_block(&mut self, size: u64) -> u64 {
if size >= EROFS_BLOCK_SIZE {
if size >= EROFS_BLOCK_SIZE_4096 {
return 0;
}

let min_idx = div_round_up(size, EROFS_INODE_SLOT_SIZE as u64) as usize;
let max_idx = div_round_up(EROFS_BLOCK_SIZE, EROFS_INODE_SLOT_SIZE as u64) as usize;
let max_idx = div_round_up(EROFS_BLOCK_SIZE_4096, EROFS_INODE_SLOT_SIZE as u64) as usize;

for idx in min_idx..max_idx {
let blocks = &mut self.v6_available_blocks[idx];
if let Some(mut offset) = blocks.pop_front() {
offset += EROFS_BLOCK_SIZE - (idx * EROFS_INODE_SLOT_SIZE) as u64;
offset += EROFS_BLOCK_SIZE_4096 - (idx * EROFS_INODE_SLOT_SIZE) as u64;
self.append_available_block(offset + (min_idx * EROFS_INODE_SLOT_SIZE) as u64);
return offset;
}
Expand All @@ -1012,8 +1012,8 @@ impl BootstrapContext {

// Append the block that `offset` belongs to corresponding deque.
pub(crate) fn append_available_block(&mut self, offset: u64) {
if offset % EROFS_BLOCK_SIZE != 0 {
let avail = EROFS_BLOCK_SIZE - offset % EROFS_BLOCK_SIZE;
if offset % EROFS_BLOCK_SIZE_4096 != 0 {
let avail = EROFS_BLOCK_SIZE_4096 - offset % EROFS_BLOCK_SIZE_4096;
let idx = avail as usize / EROFS_INODE_SLOT_SIZE;
self.v6_available_blocks[idx].push_back(round_down_4k(offset));
}
Expand Down
31 changes: 9 additions & 22 deletions rafs/src/builder/core/node.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,15 +235,19 @@ impl Node {
return Ok(0);
} else if self.is_symlink() {
if let Some(symlink) = self.info.symlink.as_ref() {
self.inode
.set_digest(RafsDigest::from_buf(symlink.as_bytes(), ctx.digester));
if self.inode.is_v5() {
self.inode
.set_digest(RafsDigest::from_buf(symlink.as_bytes(), ctx.digester));
}
return Ok(0);
} else {
return Err(Error::msg("inode's symblink is invalid."));
}
} else if self.is_special() {
self.inode
.set_digest(RafsDigest::hasher(ctx.digester).digest_finalize());
if self.inode.is_v5() {
self.inode
.set_digest(RafsDigest::hasher(ctx.digester).digest_finalize());
}
return Ok(0);
}

Expand Down Expand Up @@ -581,7 +585,7 @@ impl Node {
// calculate it later by ourself.
if !self.is_dir() {
self.inode.set_size(meta.st_size());
self.set_inode_blocks();
self.v5_set_inode_blocks();
}
self.info = Arc::new(info);

Expand Down Expand Up @@ -736,23 +740,6 @@ impl Node {
&self.info.target
}

/// Calculate and set `i_blocks` for inode.
///
/// In order to support repeatable build, we can't reuse `i_blocks` from source filesystems,
/// so let's calculate it by ourself for stable `i_block`.
///
/// Normal filesystem includes the space occupied by Xattr into the directory size,
/// let's follow the normal behavior.
pub fn set_inode_blocks(&mut self) {
// Set inode blocks for RAFS v5 inode, v6 will calculate it at runtime.
if let InodeWrapper::V5(_) = self.inode {
self.inode.set_blocks(div_round_up(
self.inode.size() + self.info.xattrs.aligned_size_v5() as u64,
512,
));
}
}

/// Set symlink target for the node.
pub fn set_symlink(&mut self, symlink: OsString) {
let mut info = self.info.deref().clone();
Expand Down
21 changes: 19 additions & 2 deletions rafs/src/builder/core/v5.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use std::mem::size_of;

use anyhow::{bail, Context, Result};
use nydus_utils::digest::{DigestHasher, RafsDigest};
use nydus_utils::{root_tracer, timing_tracer, try_round_up_4k};
use nydus_utils::{div_round_up, root_tracer, timing_tracer, try_round_up_4k};

use super::bootstrap::STARGZ_DEFAULT_BLOCK_SIZE;
use super::node::Node;
Expand Down Expand Up @@ -102,7 +102,24 @@ impl Node {
// Safe to unwrap() because we have u32 for child count.
self.inode.set_size(try_round_up_4k(d_size).unwrap());
}
self.set_inode_blocks();
self.v5_set_inode_blocks();
}

/// Calculate and set `i_blocks` for inode.
///
/// In order to support repeatable build, we can't reuse `i_blocks` from source filesystems,
/// so let's calculate it by ourself for stable `i_block`.
///
/// Normal filesystem includes the space occupied by Xattr into the directory size,
/// let's follow the normal behavior.
pub fn v5_set_inode_blocks(&mut self) {
// Set inode blocks for RAFS v5 inode, v6 will calculate it at runtime.
if let InodeWrapper::V5(_) = self.inode {
self.inode.set_blocks(div_round_up(
self.inode.size() + self.info.xattrs.aligned_size_v5() as u64,
512,
));
}
}
}

Expand Down
37 changes: 19 additions & 18 deletions rafs/src/builder/core/v6.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ use crate::metadata::inode::new_v6_inode;
use crate::metadata::layout::v6::{
align_offset, calculate_nid, RafsV6BlobTable, RafsV6Device, RafsV6Dirent, RafsV6InodeChunkAddr,
RafsV6InodeChunkHeader, RafsV6OndiskInode, RafsV6SuperBlock, RafsV6SuperBlockExt,
EROFS_BLOCK_SIZE, EROFS_DEVTABLE_OFFSET, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE,
EROFS_BLOCK_SIZE_4096, EROFS_DEVTABLE_OFFSET, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE,
EROFS_INODE_FLAT_PLAIN, EROFS_INODE_SLOT_SIZE, EROFS_SUPER_BLOCK_SIZE, EROFS_SUPER_OFFSET,
};
use crate::metadata::RafsStore;
Expand Down Expand Up @@ -162,8 +162,8 @@ impl Node {
for child in tree.children.iter() {
let len = child.node.name().as_bytes().len() + size_of::<RafsV6Dirent>();
// erofs disk format requires dirent to be aligned with 4096.
if (d_size % EROFS_BLOCK_SIZE) + len as u64 > EROFS_BLOCK_SIZE {
d_size = div_round_up(d_size as u64, EROFS_BLOCK_SIZE) * EROFS_BLOCK_SIZE;
if (d_size % EROFS_BLOCK_SIZE_4096) + len as u64 > EROFS_BLOCK_SIZE_4096 {
d_size = div_round_up(d_size as u64, EROFS_BLOCK_SIZE_4096) * EROFS_BLOCK_SIZE_4096;
}
d_size += len as u64;
}
Expand Down Expand Up @@ -217,7 +217,7 @@ impl Node {
//
//
let inode_size = self.v6_size_with_xattr();
let tail: u64 = d_size % EROFS_BLOCK_SIZE;
let tail: u64 = d_size % EROFS_BLOCK_SIZE_4096;

// We use a simple inline strategy here:
// If the inode size with xattr + tail data size <= EROFS_BLOCK_SIZE,
Expand All @@ -229,7 +229,7 @@ impl Node {
// since it contain only single blocks with some unused space, the available space can only
// be smaller than EROFS_BLOCK_SIZE, therefore we can't use our used blocks to store the
// inode plus the tail data bigger than EROFS_BLOCK_SIZE.
let should_inline = tail != 0 && (inode_size + tail) <= EROFS_BLOCK_SIZE;
let should_inline = tail != 0 && (inode_size + tail) <= EROFS_BLOCK_SIZE_4096;

// If should inline, we first try to allocate space for the inode together with tail data
// using used blocks.
Expand All @@ -240,10 +240,11 @@ impl Node {
self.v6_datalayout = if should_inline {
self.v6_offset = bootstrap_ctx.allocate_available_block(inode_size + tail);
if self.v6_offset == 0 {
let available = EROFS_BLOCK_SIZE - bootstrap_ctx.offset % EROFS_BLOCK_SIZE;
let available =
EROFS_BLOCK_SIZE_4096 - bootstrap_ctx.offset % EROFS_BLOCK_SIZE_4096;
if available < inode_size + tail {
bootstrap_ctx.append_available_block(bootstrap_ctx.offset);
bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE);
bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE_4096);
}

self.v6_offset = bootstrap_ctx.offset;
Expand All @@ -252,7 +253,7 @@ impl Node {

if d_size != tail {
bootstrap_ctx.append_available_block(bootstrap_ctx.offset);
bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE);
bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE_4096);
}
self.v6_dirents_offset = bootstrap_ctx.offset;
bootstrap_ctx.offset += round_down_4k(d_size);
Expand All @@ -269,10 +270,10 @@ impl Node {
}

bootstrap_ctx.append_available_block(bootstrap_ctx.offset);
bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE);
bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE_4096);
self.v6_dirents_offset = bootstrap_ctx.offset;
bootstrap_ctx.offset += d_size;
bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE);
bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE_4096);

EROFS_INODE_FLAT_PLAIN
};
Expand Down Expand Up @@ -336,7 +337,7 @@ impl Node {
for (offset, name, file_type) in self.v6_dirents.iter() {
let len = name.len() + size_of::<RafsV6Dirent>();
// write to bootstrap when it will exceed EROFS_BLOCK_SIZE
if used + len as u64 > EROFS_BLOCK_SIZE {
if used + len as u64 > EROFS_BLOCK_SIZE_4096 {
for (entry, name) in dirents.iter_mut() {
trace!("{:?} nameoff {}", name, nameoff);
entry.set_name_offset(nameoff as u16);
Expand All @@ -363,7 +364,7 @@ impl Node {
nameoff = 0;
used = 0;
// track where we're going to write.
dirent_off += EROFS_BLOCK_SIZE;
dirent_off += EROFS_BLOCK_SIZE_4096;
}

trace!(
Expand Down Expand Up @@ -527,7 +528,7 @@ impl Node {

impl BuildContext {
pub fn v6_block_size(&self) -> u64 {
EROFS_BLOCK_SIZE
EROFS_BLOCK_SIZE_4096
}

pub fn v6_block_addr(&self, offset: u64) -> Result<u32> {
Expand Down Expand Up @@ -609,7 +610,7 @@ impl Bootstrap {
let blob_table_size = blob_table.size() as u64;
let blob_table_offset = align_offset(
(EROFS_DEVTABLE_OFFSET as u64) + devtable_len as u64,
EROFS_BLOCK_SIZE as u64,
EROFS_BLOCK_SIZE_4096 as u64,
);
let blob_table_entries = blobs.len();
assert!(blob_table_entries < u8::MAX as usize);
Expand Down Expand Up @@ -639,11 +640,11 @@ impl Bootstrap {
// When using nid 0 as root nid,
// the root directory will not be shown by glibc's getdents/readdir.
// Because in some OS, ino == 0 represents corresponding file is deleted.
let orig_meta_addr = bootstrap_ctx.nodes[0].v6_offset - EROFS_BLOCK_SIZE;
let orig_meta_addr = bootstrap_ctx.nodes[0].v6_offset - EROFS_BLOCK_SIZE_4096;
let meta_addr = if blob_table_size > 0 {
align_offset(
blob_table_offset + blob_table_size + prefetch_table_size as u64,
EROFS_BLOCK_SIZE as u64,
EROFS_BLOCK_SIZE_4096 as u64,
)
} else {
orig_meta_addr
Expand Down Expand Up @@ -733,7 +734,7 @@ impl Bootstrap {
.writer
.seek_to_end()
.context("failed to seek to bootstrap's end for chunk table")?;
assert_eq!(pos % EROFS_BLOCK_SIZE, 0);
assert_eq!(pos % EROFS_BLOCK_SIZE_4096, 0);
let mut devtable: Vec<RafsV6Device> = Vec::new();
let mut block_count = 0u32;
let mut inlined_chunk_digest = true;
Expand Down Expand Up @@ -834,7 +835,7 @@ impl Bootstrap {
.writer
.seek_to_end()
.context("failed to seek to bootstrap's end for chunk table")?;
let padding = align_offset(pos, EROFS_BLOCK_SIZE as u64) - pos;
let padding = align_offset(pos, EROFS_BLOCK_SIZE_4096 as u64) - pos;
bootstrap_ctx
.writer
.write_all(&WRITE_PADDING_DATA[0..padding as usize])
Expand Down
Loading

0 comments on commit 3213eb7

Please sign in to comment.