diff --git a/rafs/src/lib.rs b/rafs/src/lib.rs index c5f293c7112..d81d3fb0483 100644 --- a/rafs/src/lib.rs +++ b/rafs/src/lib.rs @@ -79,7 +79,7 @@ impl RafsIoRead for File {} pub type RafsIoWriter = Box; /// A helper trait for RafsIoWriter. -pub trait RafsIoWrite: Write + Seek { +pub trait RafsIoWrite: Write + Seek + 'static { fn as_any(&self) -> &dyn Any; fn validate_alignment(&mut self, size: usize, alignment: usize) -> Result { diff --git a/rafs/src/metadata/cached_v5.rs b/rafs/src/metadata/cached_v5.rs index 04712cc90a4..857ada16884 100644 --- a/rafs/src/metadata/cached_v5.rs +++ b/rafs/src/metadata/cached_v5.rs @@ -733,7 +733,7 @@ mod cached_tests { }; use crate::metadata::layout::{RafsXAttrs, RAFS_ROOT_INODE}; use crate::metadata::{RafsInode, RafsStore, RafsSuperMeta}; - use crate::{RafsIoReader, RafsIoWriter}; + use crate::{BufWriter, RafsIoReader}; #[test] fn test_load_inode() { @@ -744,7 +744,7 @@ mod cached_tests { .read(true) .open("/tmp/buf_1") .unwrap(); - let mut writer = Box::new(f.try_clone().unwrap()) as RafsIoWriter; + let mut writer = BufWriter::new(f.try_clone().unwrap()); let mut reader = Box::new(f.try_clone().unwrap()) as RafsIoReader; let mut ondisk_inode = RafsV5Inode::new(); @@ -815,7 +815,7 @@ mod cached_tests { .read(true) .open("/tmp/buf_2") .unwrap(); - let mut writer = Box::new(f.try_clone().unwrap()) as RafsIoWriter; + let mut writer = BufWriter::new(f.try_clone().unwrap()); let mut reader = Box::new(f.try_clone().unwrap()) as RafsIoReader; let file_name = OsString::from("c_inode_2"); let symlink_name = OsString::from("c_inode_1"); @@ -856,7 +856,7 @@ mod cached_tests { .read(true) .open("/tmp/buf_3") .unwrap(); - let mut writer = Box::new(f.try_clone().unwrap()) as RafsIoWriter; + let mut writer = BufWriter::new(f.try_clone().unwrap()); let mut reader = Box::new(f.try_clone().unwrap()) as RafsIoReader; let file_name = OsString::from("c_inode_3"); let mut ondisk_inode = RafsV5Inode::new(); diff --git a/rafs/src/metadata/layout/v5.rs b/rafs/src/metadata/layout/v5.rs index a1021d14c83..6922a7b4307 100644 --- a/rafs/src/metadata/layout/v5.rs +++ b/rafs/src/metadata/layout/v5.rs @@ -51,7 +51,7 @@ use crate::metadata::layout::{bytes_to_os_str, MetaRange, RafsXAttrs, RAFS_SUPER use crate::metadata::{ Inode, RafsInode, RafsStore, RafsSuperFlags, RAFS_DEFAULT_CHUNK_SIZE, RAFS_MAX_CHUNK_SIZE, }; -use crate::{impl_bootstrap_converter, impl_pub_getter_setter, RafsIoReader, RafsIoWriter}; +use crate::{impl_bootstrap_converter, impl_pub_getter_setter, RafsIoReader, RafsIoWrite}; // With Rafs v5, the storage manager needs to access file system metadata to decompress the // compressed blob file. To avoid circular dependency, the following Rafs v5 metadata structures @@ -354,7 +354,7 @@ impl RafsV5SuperBlock { } impl RafsStore for RafsV5SuperBlock { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { w.write_all(self.as_ref())?; w.validate_alignment(self.as_ref().len(), RAFSV5_ALIGNMENT) } @@ -464,7 +464,7 @@ impl RafsV5InodeTable { } impl RafsStore for RafsV5InodeTable { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { let (_, data, _) = unsafe { self.data.align_to::() }; w.write_all(data)?; @@ -516,7 +516,7 @@ impl RafsV5PrefetchTable { } /// Store the inode prefetch table to a writer. - pub fn store(&mut self, w: &mut RafsIoWriter) -> Result { + pub fn store(&mut self, w: &mut dyn RafsIoWrite) -> Result { // Sort prefetch table by inode index, hopefully, it can save time when mounting rafs // Because file data is dumped in the order of inode index. self.inodes.sort_unstable(); @@ -710,13 +710,13 @@ impl RafsV5BlobTable { } /// Store the extended blob information array. - pub fn store_extended(&self, w: &mut RafsIoWriter) -> Result { + pub fn store_extended(&self, w: &mut dyn RafsIoWrite) -> Result { self.extended.store(w) } } impl RafsStore for RafsV5BlobTable { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { let mut size = 0; self.entries .iter() @@ -858,7 +858,7 @@ impl RafsV5ExtBlobTable { } impl RafsStore for RafsV5ExtBlobTable { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { let mut size = 0; // Store the list of entries @@ -1043,7 +1043,7 @@ pub struct RafsV5InodeWrapper<'a> { } impl<'a> RafsStore for RafsV5InodeWrapper<'a> { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { let mut size: usize = 0; let inode_data = self.inode.as_ref(); @@ -1109,7 +1109,7 @@ impl RafsV5ChunkInfo { } impl RafsStore for RafsV5ChunkInfo { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { w.write_all(self.as_ref())?; w.validate_alignment(self.as_ref().len(), RAFSV5_ALIGNMENT) } @@ -1175,7 +1175,7 @@ impl RafsXAttrs { rafsv5_align(self.size()) } - pub fn store_v5(&self, w: &mut RafsIoWriter) -> Result { + pub fn store_v5(&self, w: &mut dyn RafsIoWrite) -> Result { let mut size = 0; if !self.pairs.is_empty() { @@ -1417,7 +1417,7 @@ pub mod tests { use super::*; use crate::metadata::RafsStore; - use crate::{RafsIoRead, RafsIoReader, RafsIoWrite}; + use crate::{RafsIoRead, RafsIoReader}; use std::any::Any; use std::str::FromStr; @@ -1535,7 +1535,7 @@ pub mod tests { .write(true) .open(tmp_file.as_path()) .unwrap(); - let mut writer = Box::new(BufWriter::new(file)) as Box; + let mut writer = BufWriter::new(file); table.store(&mut writer).unwrap(); // Load extended blob table @@ -1817,7 +1817,7 @@ pub mod tests { .write(true) .open(tmp_file.as_path()) .unwrap(); - let mut writer = Box::new(BufWriter::new(file)) as Box; + let mut writer = BufWriter::new(file); writer.write_all(&[0u8; 8]).unwrap(); assert_eq!(table.store(&mut writer).unwrap(), 8); writer.flush().unwrap(); @@ -1880,7 +1880,7 @@ pub mod tests { .write(true) .open(tmp_file.as_path()) .unwrap(); - let mut writer = Box::new(BufWriter::new(file)) as Box; + let mut writer = BufWriter::new(file); assert_eq!(inode_wrapper.store(&mut writer).unwrap(), 144); writer.flush().unwrap(); diff --git a/rafs/src/metadata/layout/v6.rs b/rafs/src/metadata/layout/v6.rs index b27c42b03ce..9e2745edcc9 100644 --- a/rafs/src/metadata/layout/v6.rs +++ b/rafs/src/metadata/layout/v6.rs @@ -7,7 +7,7 @@ use lazy_static::lazy_static; use std::convert::{TryFrom, TryInto}; use std::ffi::OsStr; use std::fmt::Debug; -use std::io::{Read, Result, Write}; +use std::io::{Read, Result}; use std::mem::size_of; use std::os::unix::ffi::OsStrExt; use std::sync::Arc; @@ -18,7 +18,7 @@ use storage::meta::BlobMetaHeaderOndisk; use storage::{compress, RAFS_MAX_CHUNK_SIZE}; use crate::metadata::{layout::RafsXAttrs, RafsStore, RafsSuperFlags}; -use crate::{impl_bootstrap_converter, impl_pub_getter_setter, RafsIoReader, RafsIoWriter}; +use crate::{impl_bootstrap_converter, impl_pub_getter_setter, RafsIoReader, RafsIoWrite}; /// EROFS metadata slot size. pub const EROFS_INODE_SLOT_SIZE: usize = 1 << EROFS_INODE_SLOT_BITS; @@ -211,7 +211,7 @@ impl RafsV6SuperBlock { impl RafsStore for RafsV6SuperBlock { // This method must be called before RafsV6SuperBlockExt::store(), otherwise data written by // RafsV6SuperBlockExt::store() will be overwritten. - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { debug_assert!(((EROFS_SUPER_OFFSET + EROFS_SUPER_BLOCK_SIZE) as u64) < EROFS_BLOCK_SIZE); w.write_all(&[0u8; EROFS_SUPER_OFFSET as usize])?; w.write_all(self.as_ref())?; @@ -349,7 +349,7 @@ impl RafsV6SuperBlockExt { } impl RafsStore for RafsV6SuperBlockExt { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { w.seek_to_offset((EROFS_SUPER_OFFSET + EROFS_SUPER_BLOCK_SIZE) as u64)?; w.write_all(self.as_ref())?; w.seek_to_offset(EROFS_BLOCK_SIZE as u64)?; @@ -520,7 +520,7 @@ impl RafsV6InodeExtended { impl_bootstrap_converter!(RafsV6InodeExtended); impl RafsStore for RafsV6InodeExtended { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { // TODO: need to write xattr as well. w.write_all(self.as_ref())?; Ok(self.as_ref().len()) @@ -583,7 +583,7 @@ impl RafsV6Dirent { } impl RafsStore for RafsV6Dirent { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { w.write_all(self.as_ref())?; Ok(self.as_ref().len()) @@ -700,7 +700,7 @@ impl RafsV6InodeChunkAddr { impl_bootstrap_converter!(RafsV6InodeChunkAddr); impl RafsStore for RafsV6InodeChunkAddr { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { w.write_all(self.as_ref())?; Ok(self.as_ref().len()) @@ -776,7 +776,7 @@ impl RafsV6Device { impl_bootstrap_converter!(RafsV6Device); impl RafsStore for RafsV6Device { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { w.write_all(self.as_ref())?; Ok(self.as_ref().len()) @@ -1094,7 +1094,7 @@ impl RafsV6BlobTable { } impl RafsStore for RafsV6BlobTable { - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { for blob_info in self.entries.iter() { let blob: RafsV6Blob = RafsV6Blob::from_blob_info(blob_info)?; trace!( @@ -1279,7 +1279,7 @@ impl RafsXAttrs { } /// Write Xattr to rafsv6 ondisk inode. - pub fn store_v6(&self, w: &mut RafsIoWriter) -> Result { + pub fn store_v6(&self, w: &mut dyn RafsIoWrite) -> Result { // TODO: check count of shared. let header = RafsV6XattrIbodyHeader::new(); w.write_all(header.as_ref())?; @@ -1329,7 +1329,7 @@ impl RafsXAttrs { #[cfg(test)] mod tests { use super::*; - use crate::{RafsIoRead, RafsIoWrite}; + use crate::{BufWriter, RafsIoRead}; use std::ffi::OsString; use std::fs::OpenOptions; use vmm_sys_util::tempfile::TempFile; @@ -1348,7 +1348,7 @@ mod tests { .write(false) .open(temp.as_path()) .unwrap(); - let mut writer: Box = Box::new(w); + let mut writer = BufWriter::new(w); let mut reader: Box = Box::new(r); sb.s_blocks = 0x1000; @@ -1382,7 +1382,7 @@ mod tests { .write(false) .open(temp.as_path()) .unwrap(); - let mut writer: Box = Box::new(w); + let mut writer = BufWriter::new(w); let mut reader: Box = Box::new(r); let mut inode = RafsV6InodeExtended::new(); @@ -1447,7 +1447,7 @@ mod tests { .write(false) .open(temp.as_path()) .unwrap(); - let mut writer: Box = Box::new(w); + let mut writer = BufWriter::new(w); let mut reader: Box = Box::new(r); let mut chunk = RafsV6InodeChunkAddr::new(); @@ -1476,7 +1476,7 @@ mod tests { .write(false) .open(temp.as_path()) .unwrap(); - let mut writer: Box = Box::new(w); + let mut writer = BufWriter::new(w); let mut reader: Box = Box::new(r); let id = [0xa5u8; 32]; @@ -1535,7 +1535,7 @@ mod tests { .write(false) .open(temp.as_path()) .unwrap(); - let mut writer: Box = Box::new(w); + let mut writer = BufWriter::new(w); let mut reader: Box = Box::new(r); let mut xattrs = RafsXAttrs::new(); diff --git a/rafs/src/metadata/md_v5.rs b/rafs/src/metadata/md_v5.rs index 232a7dce5a3..fe832ec9793 100644 --- a/rafs/src/metadata/md_v5.rs +++ b/rafs/src/metadata/md_v5.rs @@ -53,7 +53,7 @@ impl RafsSuper { Ok(true) } - pub(crate) fn store_v5(&self, w: &mut RafsIoWriter) -> Result { + pub(crate) fn store_v5(&self, w: &mut dyn RafsIoWrite) -> Result { let mut sb = RafsV5SuperBlock::new(); sb.set_magic(self.meta.magic); diff --git a/rafs/src/metadata/md_v6.rs b/rafs/src/metadata/md_v6.rs index 459bf143152..445727f0921 100644 --- a/rafs/src/metadata/md_v6.rs +++ b/rafs/src/metadata/md_v6.rs @@ -53,7 +53,7 @@ impl RafsSuper { mod tests { use super::*; use crate::metadata::RafsStore; - use crate::RafsIoWriter; + use crate::BufWriter; use std::fs::OpenOptions; use std::io::Write; use vmm_sys_util::tempfile::TempFile; @@ -107,7 +107,7 @@ mod tests { .open(t_file.as_path()) .unwrap(); let sb = RafsV6SuperBlock::new(); - let mut writer = Box::new(file) as RafsIoWriter; + let mut writer = BufWriter::new(file); sb.store(&mut writer).unwrap(); let file = OpenOptions::new() diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs index 0c467d7e182..c7c3275145a 100644 --- a/rafs/src/metadata/mod.rs +++ b/rafs/src/metadata/mod.rs @@ -28,7 +28,7 @@ use storage::device::{BlobChunkInfo, BlobInfo, BlobIoVec}; use self::layout::{XattrName, XattrValue, RAFS_SUPER_VERSION_V5, RAFS_SUPER_VERSION_V6}; use self::noop::NoopSuperBlock; use crate::fs::{RafsConfig, RAFS_DEFAULT_ATTR_TIMEOUT, RAFS_DEFAULT_ENTRY_TIMEOUT}; -use crate::{RafsError, RafsIoReader, RafsIoWriter, RafsResult}; +use crate::{RafsError, RafsIoReader, RafsIoWrite, RafsResult}; pub mod cached_v5; pub mod direct_v5; @@ -203,7 +203,7 @@ pub trait RafsInode: Any { /// Trait to store Rafs meta block and validate alignment. pub trait RafsStore { /// Write the Rafs filesystem metadata to a writer. - fn store(&self, w: &mut RafsIoWriter) -> Result; + fn store(&self, w: &mut dyn RafsIoWrite) -> Result; } bitflags! { @@ -470,7 +470,7 @@ impl RafsSuper { } /// Store RAFS metadata to backend storage. - pub fn store(&self, w: &mut RafsIoWriter) -> Result { + pub fn store(&self, w: &mut dyn RafsIoWrite) -> Result { if self.meta.is_v5() { return self.store_v5(w); } diff --git a/src/bin/nydus-image/builder/diff.rs b/src/bin/nydus-image/builder/diff.rs index 952cd5b3917..ac1fa2aa47b 100644 --- a/src/bin/nydus-image/builder/diff.rs +++ b/src/bin/nydus-image/builder/diff.rs @@ -56,7 +56,8 @@ use crate::core::blob::Blob; use crate::core::bootstrap::Bootstrap; use crate::core::chunk_dict::{ChunkDict, HashChunkDict}; use crate::core::context::{ - BlobContext, BlobManager, BlobStorage, BootstrapContext, BuildContext, RafsVersion, + ArtifactStorage, BlobContext, BlobManager, BootstrapContext, BootstrapManager, BuildContext, + RafsVersion, }; use crate::core::node::{ChunkWrapper, Node, Overlay}; use crate::core::tree::Tree; @@ -223,7 +224,7 @@ fn dump_blob( ctx: Arc, snapshot_idx: u32, blob_id: String, - blob_storage: Option, + blob_storage: Option, upper_nodes: UpperNodes, nodes: &mut Vec, chunk_dict: Arc, @@ -450,9 +451,11 @@ impl DiffBuilder { fn build_with_hint( &mut self, ctx: &mut BuildContext, - bootstrap_ctx: &mut BootstrapContext, + bootstrap_mgr: &mut BootstrapManager, blob_mgr: &mut BlobManager, ) -> Result<(Vec, u64)> { + let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; + let mut paths = vec![ctx.source_path.clone()]; paths.append(&mut self.extra_paths); @@ -501,7 +504,7 @@ impl DiffBuilder { self.build_bootstrap( ctx, - bootstrap_ctx, + &mut bootstrap_ctx, blob_mgr, workers, paths[base - 1].clone(), @@ -511,9 +514,10 @@ impl DiffBuilder { fn build_with_diff( &mut self, ctx: &mut BuildContext, - bootstrap_ctx: &mut BootstrapContext, + bootstrap_mgr: &mut BootstrapManager, blob_mgr: &mut BlobManager, ) -> Result<(Vec, u64)> { + let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; let mut paths = vec![None, Some(ctx.source_path.clone())]; let mut extra_paths: Vec<_> = self.extra_paths.iter().map(|p| Some(p.clone())).collect(); paths.append(&mut extra_paths); @@ -555,7 +559,7 @@ impl DiffBuilder { // Safe to unwrap because last snapshot path must be exists. self.build_bootstrap( ctx, - bootstrap_ctx, + &mut bootstrap_ctx, blob_mgr, workers, paths[base].clone().unwrap(), @@ -567,13 +571,13 @@ impl Builder for DiffBuilder { fn build( &mut self, ctx: &mut BuildContext, - bootstrap_ctx: &mut BootstrapContext, + bootstrap_mgr: &mut BootstrapManager, blob_mgr: &mut BlobManager, ) -> Result<(Vec, u64)> { if self.diff_hint { - self.build_with_hint(ctx, bootstrap_ctx, blob_mgr) + self.build_with_hint(ctx, bootstrap_mgr, blob_mgr) } else { - self.build_with_diff(ctx, bootstrap_ctx, blob_mgr) + self.build_with_diff(ctx, bootstrap_mgr, blob_mgr) } } } diff --git a/src/bin/nydus-image/builder/directory.rs b/src/bin/nydus-image/builder/directory.rs index 0555fac827c..9da293e2a7c 100644 --- a/src/bin/nydus-image/builder/directory.rs +++ b/src/bin/nydus-image/builder/directory.rs @@ -10,7 +10,9 @@ use anyhow::{Context, Result}; use crate::builder::Builder; use crate::core::blob::Blob; use crate::core::bootstrap::Bootstrap; -use crate::core::context::{BlobContext, BlobManager, BootstrapContext, BuildContext, RafsVersion}; +use crate::core::context::{ + BlobContext, BlobManager, BootstrapContext, BootstrapManager, BuildContext, RafsVersion, +}; use crate::core::node::{Node, Overlay}; use crate::core::tree::Tree; @@ -33,7 +35,6 @@ impl FilesystemTreeBuilder { return Ok(result); } - let layered = bootstrap_ctx.f_parent_bootstrap.is_some(); let children = fs::read_dir(parent.path()) .with_context(|| format!("failed to read dir {:?}", parent.path()))?; let children = children.collect::, std::io::Error>>()?; @@ -53,7 +54,7 @@ impl FilesystemTreeBuilder { // as per OCI spec, whiteout file should not be present within final image // or filesystem, only existed in layers. - if !layered + if !bootstrap_ctx.layered && child.whiteout_type(ctx.whiteout_spec).is_some() && !child.is_overlayfs_opaque(ctx.whiteout_spec) { @@ -106,21 +107,22 @@ impl Builder for DirectoryBuilder { fn build( &mut self, ctx: &mut BuildContext, - bootstrap_ctx: &mut BootstrapContext, + bootstrap_mgr: &mut BootstrapManager, blob_mgr: &mut BlobManager, ) -> Result<(Vec, u64)> { + let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; // Scan source directory to build upper layer tree. - let mut tree = self.build_tree_from_fs(ctx, bootstrap_ctx)?; + let mut tree = self.build_tree_from_fs(ctx, &mut bootstrap_ctx)?; let mut bootstrap = Bootstrap::new()?; - if bootstrap_ctx.f_parent_bootstrap.is_some() { + if bootstrap_ctx.layered { // Merge with lower layer if there's one, do not prepare `prefetch` list during merging. ctx.prefetch.disable(); - bootstrap.build(ctx, bootstrap_ctx, &mut tree)?; - tree = bootstrap.apply(ctx, bootstrap_ctx, blob_mgr, None)?; + bootstrap.build(ctx, &mut bootstrap_ctx, &mut tree)?; + tree = bootstrap.apply(ctx, &mut bootstrap_ctx, bootstrap_mgr, blob_mgr, None)?; } // Convert the hierarchy tree into an array, stored in `bootstrap_ctx.nodes`. timing_tracer!( - { bootstrap.build(ctx, bootstrap_ctx, &mut tree) }, + { bootstrap.build(ctx, &mut bootstrap_ctx, &mut tree) }, "build_bootstrap" )?; @@ -154,8 +156,8 @@ impl Builder for DirectoryBuilder { // Dump bootstrap file match ctx.fs_version { - RafsVersion::V5 => bootstrap.dump_rafsv5(ctx, bootstrap_ctx, blob_mgr), - RafsVersion::V6 => bootstrap.dump_rafsv6(ctx, bootstrap_ctx, blob_mgr), + RafsVersion::V5 => bootstrap.dump_rafsv5(ctx, &mut bootstrap_ctx, blob_mgr), + RafsVersion::V6 => bootstrap.dump_rafsv6(ctx, &mut bootstrap_ctx, blob_mgr), } } } diff --git a/src/bin/nydus-image/builder/mod.rs b/src/bin/nydus-image/builder/mod.rs index 59f62b8cd29..3a639e49cba 100644 --- a/src/bin/nydus-image/builder/mod.rs +++ b/src/bin/nydus-image/builder/mod.rs @@ -4,7 +4,7 @@ use anyhow::Result; -use crate::core::context::{BlobManager, BootstrapContext, BuildContext}; +use crate::core::context::{BlobManager, BootstrapManager, BuildContext}; pub(crate) use diff::DiffBuilder; pub(crate) use directory::DirectoryBuilder; @@ -18,7 +18,7 @@ pub(crate) trait Builder { fn build( &mut self, build_ctx: &mut BuildContext, - bootstrap_ctx: &mut BootstrapContext, + bootstrap_mgr: &mut BootstrapManager, blob_mgr: &mut BlobManager, ) -> Result<(Vec, u64)>; } diff --git a/src/bin/nydus-image/builder/stargz.rs b/src/bin/nydus-image/builder/stargz.rs index f294b466cc3..0d5bc7ae182 100644 --- a/src/bin/nydus-image/builder/stargz.rs +++ b/src/bin/nydus-image/builder/stargz.rs @@ -26,7 +26,9 @@ use storage::device::BlobChunkFlags; use crate::builder::Builder; use crate::core::bootstrap::Bootstrap; -use crate::core::context::{BlobContext, BlobManager, BootstrapContext, BuildContext, RafsVersion}; +use crate::core::context::{ + BlobContext, BlobManager, BootstrapContext, BootstrapManager, BuildContext, RafsVersion, +}; use crate::core::node::{ChunkWrapper, InodeWrapper, Node, Overlay}; use crate::core::tree::Tree; @@ -619,28 +621,29 @@ impl Builder for StargzBuilder { fn build( &mut self, ctx: &mut BuildContext, - bootstrap_ctx: &mut BootstrapContext, + bootstrap_mgr: &mut BootstrapManager, blob_mgr: &mut BlobManager, ) -> Result<(Vec, u64)> { + let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; // Build tree from source let mut tree = self.build_tree_from_index(ctx)?; let mut bootstrap = Bootstrap::new()?; - if bootstrap_ctx.f_parent_bootstrap.is_some() { + if bootstrap_mgr.f_parent_bootstrap.is_some() { // Merge with lower layer if there's one. - bootstrap.build(ctx, bootstrap_ctx, &mut tree)?; - tree = bootstrap.apply(ctx, bootstrap_ctx, blob_mgr, None)?; + bootstrap.build(ctx, &mut bootstrap_ctx, &mut tree)?; + tree = bootstrap.apply(ctx, &mut bootstrap_ctx, bootstrap_mgr, blob_mgr, None)?; } timing_tracer!( - { bootstrap.build(ctx, bootstrap_ctx, &mut tree) }, + { bootstrap.build(ctx, &mut bootstrap_ctx, &mut tree) }, "build_bootstrap" )?; // generate node chunks and digest - self.generate_nodes(ctx, bootstrap_ctx, blob_mgr)?; + self.generate_nodes(ctx, &mut bootstrap_ctx, blob_mgr)?; // Dump bootstrap file match ctx.fs_version { - RafsVersion::V5 => bootstrap.dump_rafsv5(ctx, bootstrap_ctx, blob_mgr), + RafsVersion::V5 => bootstrap.dump_rafsv5(ctx, &mut bootstrap_ctx, blob_mgr), RafsVersion::V6 => todo!(), } } diff --git a/src/bin/nydus-image/core/bootstrap.rs b/src/bin/nydus-image/core/bootstrap.rs index d2ea8022a8a..9d12a35d2f5 100644 --- a/src/bin/nydus-image/core/bootstrap.rs +++ b/src/bin/nydus-image/core/bootstrap.rs @@ -5,6 +5,7 @@ use std::convert::TryFrom; use std::convert::TryInto; use std::ffi::OsString; +use std::io::Write; use std::mem::size_of; use anyhow::{Context, Error, Result}; @@ -16,12 +17,15 @@ use rafs::metadata::layout::v6::{ align_offset, calculate_nid, RafsV6BlobTable, RafsV6Device, RafsV6SuperBlock, RafsV6SuperBlockExt, EROFS_BLOCK_SIZE, EROFS_DEVTABLE_OFFSET, EROFS_INODE_SLOT_SIZE, }; +use rafs::RafsIoWrite; use rafs::metadata::layout::RAFS_ROOT_INODE; use rafs::metadata::{RafsMode, RafsStore, RafsSuper, RafsSuperFlags}; use storage::device::BlobFeatures; -use super::context::{BlobManager, BootstrapContext, BuildContext, RafsVersion, SourceType}; +use super::context::{ + BlobManager, BootstrapContext, BootstrapManager, BuildContext, RafsVersion, SourceType, +}; use super::node::{Node, WhiteoutType, OVERLAYFS_WHITEOUT_OPAQUE}; use super::prefetch::PrefetchPolicy; use super::tree::Tree; @@ -94,12 +98,13 @@ impl Bootstrap { &mut self, ctx: &mut BuildContext, bootstrap_ctx: &mut BootstrapContext, + bootstrap_mgr: &mut BootstrapManager, blob_mgr: &mut BlobManager, tree: Option, ) -> Result { let mut tree = match tree { Some(v) => v, - None => self.load_parent_bootstrap(ctx, bootstrap_ctx, blob_mgr)?, + None => self.load_parent_bootstrap(ctx, bootstrap_mgr, blob_mgr)?, }; // Apply new node (upper layer) to node tree (lower layer) @@ -198,10 +203,10 @@ impl Bootstrap { // Put the whiteout file of upper layer in the front of node list for layered build, // so that it can be applied to the node tree of lower layer first than other files of upper layer. match ( - &bootstrap_ctx.f_parent_bootstrap, + bootstrap_ctx.layered, child.node.whiteout_type(ctx.whiteout_spec), ) { - (Some(_), Some(whiteout_type)) => { + (true, Some(whiteout_type)) => { // Insert removal operations at the head, so they will be handled first when // applying to lower layer. nodes.insert(0, child.node.clone()); @@ -214,7 +219,7 @@ impl Bootstrap { nodes.push(child.node.clone()); } } - (None, Some(whiteout_type)) => { + (false, Some(whiteout_type)) => { // Remove overlayfs opaque xattr for single layer build if whiteout_type == WhiteoutType::OverlayFsOpaque { child @@ -291,10 +296,10 @@ impl Bootstrap { fn load_parent_bootstrap( &mut self, ctx: &mut BuildContext, - bootstrap_ctx: &mut BootstrapContext, + bootstrap_mgr: &mut BootstrapManager, blob_mgr: &mut BlobManager, ) -> Result { - let rs = if let Some(r) = bootstrap_ctx.f_parent_bootstrap.as_mut() { + let rs = if let Some(r) = bootstrap_mgr.f_parent_bootstrap.as_mut() { let mut rs = RafsSuper { mode: RafsMode::Direct, validate_digest: true, @@ -453,46 +458,48 @@ impl Bootstrap { super_block.set_has_xattr(); } + let mut bootstrap_writer = bootstrap_ctx.create_writer()?; + // Dump super block super_block - .store(&mut bootstrap_ctx.f_bootstrap) + .store(&mut bootstrap_writer) .context("failed to store superblock")?; // Dump inode table inode_table - .store(&mut bootstrap_ctx.f_bootstrap) + .store(&mut bootstrap_writer) .context("failed to store inode table")?; // Dump prefetch table if let Some(mut prefetch_table) = ctx.prefetch.get_rafsv5_prefetch_table() { prefetch_table - .store(&mut bootstrap_ctx.f_bootstrap) + .store(&mut bootstrap_writer) .context("failed to store prefetch table")?; } // Dump blob table blob_table - .store(&mut bootstrap_ctx.f_bootstrap) + .store(&mut bootstrap_writer) .context("failed to store blob table")?; // Dump extended blob table blob_table - .store_extended(&mut bootstrap_ctx.f_bootstrap) + .store_extended(&mut bootstrap_writer) .context("failed to store extended blob table")?; // Dump inodes and chunks timing_tracer!( { - for node in &mut bootstrap_ctx.nodes { + for node in &bootstrap_ctx.nodes { if ctx.source_type == SourceType::StargzIndex { debug!("[{}]\t{}", node.overlay, node); if log::max_level() >= log::LevelFilter::Debug { - for chunk in node.chunks.iter_mut() { + for chunk in node.chunks.iter() { trace!("\t\tbuilding chunk: {}", chunk); } } } - node.dump_bootstrap_v5(&mut bootstrap_ctx.f_bootstrap) + node.dump_bootstrap_v5(&mut bootstrap_writer) .context("failed to dump bootstrap")?; } @@ -503,7 +510,7 @@ impl Bootstrap { )?; // Flush remaining data in BufWriter to file - bootstrap_ctx.f_bootstrap.flush()?; + bootstrap_writer.flush()?; let blob_ids: Vec = blob_table .entries @@ -534,6 +541,7 @@ impl Bootstrap { let blob_table = blob_mgr.to_blob_table_v6(ctx)?; let blob_table_size = blob_table.size() as u64; + let bootstrap_writer = &mut bootstrap_ctx.create_writer()? as &mut dyn RafsIoWrite; // get devt_slotoff let mut devtable: Vec = Vec::new(); @@ -582,8 +590,7 @@ impl Bootstrap { sb.set_extra_devices(blob_table_entries as u16); - sb.store(&mut bootstrap_ctx.f_bootstrap) - .context("failed to store SB")?; + sb.store(bootstrap_writer).context("failed to store SB")?; // Dump extended superblock let mut ext_sb = RafsV6SuperBlockExt::new(); @@ -594,39 +601,32 @@ impl Bootstrap { ext_sb.set_blob_table_size(blob_table_size as u32); ext_sb - .store(&mut bootstrap_ctx.f_bootstrap) + .store(bootstrap_writer) .context("failed to store extended SB")?; // dump devtslot - bootstrap_ctx - .f_bootstrap + bootstrap_writer .seek_to_offset(EROFS_DEVTABLE_OFFSET as u64) .context("failed to seek to devtslot")?; for slot in devtable.iter() { - slot.store(&mut bootstrap_ctx.f_bootstrap) + slot.store(bootstrap_writer) .context("failed to store device slot")?; } // Dump blob table - bootstrap_ctx - .f_bootstrap + bootstrap_writer .seek_to_offset(blob_table_offset as u64) .context("failed seek for extended blob table offset")?; blob_table - .store(&mut bootstrap_ctx.f_bootstrap) + .store(bootstrap_writer) .context("failed to store extended blob table")?; // Dump bootstrap timing_tracer!( { for node in &mut bootstrap_ctx.nodes { - node.dump_bootstrap_v6( - &mut bootstrap_ctx.f_bootstrap, - orig_meta_addr, - meta_addr, - ctx, - ) - .context("failed to dump bootstrap")?; + node.dump_bootstrap_v6(bootstrap_writer, orig_meta_addr, meta_addr, ctx) + .context("failed to dump bootstrap")?; } Ok(()) @@ -636,12 +636,10 @@ impl Bootstrap { )?; // Flush remaining data in BufWriter to file - bootstrap_ctx - .f_bootstrap + bootstrap_writer .flush() .context("failed to flush bootstrap")?; - let pos = bootstrap_ctx - .f_bootstrap + let pos = bootstrap_writer .seek_to_end() .context("failed to seek to bootstrap's end")?; debug!( @@ -649,8 +647,7 @@ impl Bootstrap { align_offset(pos, EROFS_BLOCK_SIZE as u64) ); let padding = align_offset(pos, EROFS_BLOCK_SIZE as u64) - pos; - bootstrap_ctx - .f_bootstrap + bootstrap_writer .write_all(&WRITE_PADDING_DATA[0..padding as usize]) .context("failed to write 0 to padding of bootstrap's end")?; diff --git a/src/bin/nydus-image/core/context.rs b/src/bin/nydus-image/core/context.rs index 98d675effca..d89d593d27a 100644 --- a/src/bin/nydus-image/core/context.rs +++ b/src/bin/nydus-image/core/context.rs @@ -4,6 +4,7 @@ //! Struct to maintain context information for the image builder. +use std::any::Any; use std::collections::HashMap; use std::convert::TryFrom; use std::fs::{remove_file, rename, File, OpenOptions}; @@ -18,7 +19,7 @@ use nydus_utils::digest; use nydus_utils::div_round_up; use rafs::metadata::layout::v6::EROFS_BLOCK_SIZE; use rafs::metadata::{Inode, RAFS_DEFAULT_CHUNK_SIZE, RAFS_MAX_CHUNK_SIZE}; -use rafs::{RafsIoReader, RafsIoWriter}; +use rafs::{RafsIoReader, RafsIoWrite}; use sha2::{Digest, Sha256}; use storage::compress; use storage::device::BlobInfo; @@ -72,31 +73,55 @@ impl FromStr for SourceType { } #[derive(Debug, Clone)] -pub enum BlobStorage { +pub enum ArtifactStorage { // Won't rename user's specification SingleFile(PathBuf), // Will rename it from tmp file as user didn't specify a name. - BlobsDir(PathBuf), + FileDir(PathBuf), } -impl Default for BlobStorage { +impl Default for ArtifactStorage { fn default() -> Self { Self::SingleFile(PathBuf::new()) } } -pub struct BlobBufferWriter { +/// ArtifactBufferWriter provides a writer to allow writing bootstrap +/// or blob data to a single file or in a directory. +pub struct ArtifactBufferWriter { file: BufWriter, - blob_stor: BlobStorage, + storage: ArtifactStorage, // Keep this because tmp file will be removed automatically when it is dropped. // But we will rename/link the tmp file before it is removed. tmp_file: Option, } -impl BlobBufferWriter { - pub fn new(blob_stor: BlobStorage) -> Result { - match blob_stor { - BlobStorage::SingleFile(ref p) => { +impl RafsIoWrite for ArtifactBufferWriter { + fn as_any(&self) -> &dyn Any { + self + } +} + +impl std::io::Write for ArtifactBufferWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.file.write(bytes) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.file.flush() + } +} + +impl std::io::Seek for ArtifactBufferWriter { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + self.file.seek(pos) + } +} + +impl ArtifactBufferWriter { + pub fn new(storage: ArtifactStorage) -> Result { + match storage { + ArtifactStorage::SingleFile(ref p) => { let b = BufWriter::with_capacity( BUF_WRITER_CAPACITY, OpenOptions::new() @@ -104,23 +129,23 @@ impl BlobBufferWriter { .create(true) .truncate(true) .open(p) - .with_context(|| format!("failed to open blob {:?}", p))?, + .with_context(|| format!("failed to open file {:?}", p))?, ); Ok(Self { file: b, - blob_stor, + storage, tmp_file: None, }) } - BlobStorage::BlobsDir(ref p) => { + ArtifactStorage::FileDir(ref p) => { // Better we can use open(2) O_TMPFILE, but for compatibility sake, we delay this job. // TODO: Blob dir existence? let tmp = TempFile::new_in(&p) - .with_context(|| format!("failed to create temp blob file in {:?}", p))?; + .with_context(|| format!("failed to create temp file in {:?}", p))?; let tmp2 = tmp.as_file().try_clone()?; Ok(Self { file: BufWriter::with_capacity(BUF_WRITER_CAPACITY, tmp2), - blob_stor, + storage, tmp_file: Some(tmp), }) } @@ -142,7 +167,7 @@ impl BlobBufferWriter { f.flush()?; if let Some(n) = name { - if let BlobStorage::BlobsDir(s) = &self.blob_stor { + if let ArtifactStorage::FileDir(s) = &self.storage { // NOTE: File with same name will be deleted ahead of time. // So each newly generated blob can be stored. let might_exist_path = Path::new(s).join(n); @@ -151,7 +176,7 @@ impl BlobBufferWriter { .with_context(|| format!("failed to remove blob {:?}", might_exist_path))?; } - // Safe to unwrap as `BlobsDir` must have `tmp_file` created. + // Safe to unwrap as `FileDir` must have `tmp_file` created. let tmp_file = self.tmp_file.unwrap(); rename(tmp_file.as_path(), &might_exist_path).with_context(|| { format!( @@ -161,7 +186,7 @@ impl BlobBufferWriter { ) })?; } - } else if let BlobStorage::SingleFile(s) = &self.blob_stor { + } else if let ArtifactStorage::SingleFile(s) = &self.storage { // `new_name` is None means no blob is really built, perhaps due to dedup. // We don't want to puzzle user, so delete it from here. // In the future, FIFO could be leveraged, don't remove it then. @@ -172,16 +197,6 @@ impl BlobBufferWriter { } } -impl Write for BlobBufferWriter { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.file.write(buf) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.file.flush() - } -} - /// BlobContext is used to hold the blob information of a layer during build. pub struct BlobContext { /// Blob id (user specified or sha256(blob)). @@ -216,13 +231,13 @@ pub struct BlobContext { pub chunk_dict: Arc, // Blob writer for writing to disk file. - pub writer: Option, + pub writer: Option, } impl BlobContext { - pub fn new(blob_id: String, blob_stor: Option) -> Result { + pub fn new(blob_id: String, blob_stor: Option) -> Result { let writer = if let Some(blob_stor) = blob_stor { - Some(BlobBufferWriter::new(blob_stor)?) + Some(ArtifactBufferWriter::new(blob_stor)?) } else { None }; @@ -230,7 +245,7 @@ impl BlobContext { Ok(Self::new_with_writer(blob_id, writer)) } - pub fn new_with_writer(blob_id: String, writer: Option) -> Self { + pub fn new_with_writer(blob_id: String, writer: Option) -> Self { let size = if writer.is_some() { RAFS_MAX_CHUNK_SIZE as usize } else { @@ -410,10 +425,7 @@ impl BlobManager { /// BootstrapContext is used to hold inmemory data of bootstrap during build. pub struct BootstrapContext { - /// Bootstrap file writer. - pub f_bootstrap: RafsIoWriter, - /// Parent bootstrap file reader. - pub f_parent_bootstrap: Option, + pub layered: bool, /// Cache node index for hardlinks, HashMap<(real_inode, dev), Vec>. pub lower_inode_map: HashMap<(Inode, u64), Vec>, pub upper_inode_map: HashMap<(Inode, u64), Vec>, @@ -421,18 +433,20 @@ pub struct BootstrapContext { pub nodes: Vec, /// current position to write in f_bootstrap pub offset: u64, + /// Bootstrap file writer. + storage: ArtifactStorage, } impl BootstrapContext { - pub fn new(f_bootstrap: RafsIoWriter, f_parent_bootstrap: Option) -> Self { - Self { - f_bootstrap, - f_parent_bootstrap, + pub fn new(storage: ArtifactStorage, layered: bool) -> Result { + Ok(Self { + layered, lower_inode_map: HashMap::new(), upper_inode_map: HashMap::new(), nodes: Vec::new(), offset: EROFS_BLOCK_SIZE, - } + storage, + }) } pub fn align_offset(&mut self, align_size: u64) { @@ -440,6 +454,37 @@ impl BootstrapContext { self.offset = div_round_up(self.offset, align_size) * align_size; } } + + pub fn create_writer(&self) -> Result { + ArtifactBufferWriter::new(self.storage.clone()) + } +} + +/// BootstrapManager is used to hold the parent bootstrap reader and create +/// new bootstrap context. +pub struct BootstrapManager { + /// Parent bootstrap file reader. + pub f_parent_bootstrap: Option, + bootstrap_storage: ArtifactStorage, +} + +impl BootstrapManager { + pub fn new( + bootstrap_storage: ArtifactStorage, + f_parent_bootstrap: Option, + ) -> Self { + Self { + f_parent_bootstrap, + bootstrap_storage, + } + } + + pub fn create_ctx(&self) -> Result { + BootstrapContext::new( + self.bootstrap_storage.clone(), + self.f_parent_bootstrap.is_some(), + ) + } } #[derive(Default, Clone)] @@ -476,7 +521,7 @@ pub struct BuildContext { pub prefetch: Prefetch, /// Storage writing blob to single file or a directory. - pub blob_storage: Option, + pub blob_storage: Option, } impl BuildContext { @@ -491,7 +536,7 @@ impl BuildContext { source_type: SourceType, source_path: PathBuf, prefetch: Prefetch, - blob_storage: Option, + blob_storage: Option, ) -> Self { BuildContext { blob_id, diff --git a/src/bin/nydus-image/core/node.rs b/src/bin/nydus-image/core/node.rs index d165a6f05d1..534aab3e4b1 100644 --- a/src/bin/nydus-image/core/node.rs +++ b/src/bin/nydus-image/core/node.rs @@ -8,7 +8,6 @@ use std::ffi::{OsStr, OsString}; use std::fmt::{self, Display, Formatter}; use std::fs::{self, File}; -use std::io::prelude::*; use std::io::Read; use std::io::SeekFrom; use std::mem::size_of; @@ -38,7 +37,7 @@ use rafs::metadata::layout::v6::{ }; use rafs::metadata::layout::RafsXAttrs; use rafs::metadata::{Inode, RafsInode, RafsStore}; -use rafs::RafsIoWriter; +use rafs::RafsIoWrite; use storage::compress; use storage::device::v5::BlobV5ChunkInfo; use storage::device::{BlobChunkFlags, BlobChunkInfo}; @@ -414,7 +413,7 @@ impl Node { Ok(blob_size) } - pub fn dump_bootstrap_v5(&mut self, f_bootstrap: &mut RafsIoWriter) -> Result { + pub fn dump_bootstrap_v5(&self, f_bootstrap: &mut dyn RafsIoWrite) -> Result { let mut node_size = 0; if let InodeWrapper::V5(raw_inode) = &self.inode { // Dump inode info @@ -443,7 +442,7 @@ impl Node { bail!("invalid chunks count {}: {}", self.chunks.len(), self); } - for chunk in &mut self.chunks { + for chunk in &self.chunks { let chunk_size = chunk .store(f_bootstrap) .context("failed to dump chunk info to bootstrap")?; @@ -458,7 +457,7 @@ impl Node { pub fn dump_bootstrap_v6( &mut self, - f_bootstrap: &mut RafsIoWriter, + f_bootstrap: &mut dyn RafsIoWrite, orig_meta_addr: u64, meta_addr: u64, ctx: &BuildContext, @@ -1600,7 +1599,7 @@ impl ChunkWrapper { } } - fn store(&self, w: &mut RafsIoWriter) -> Result { + fn store(&self, w: &mut dyn RafsIoWrite) -> Result { match self { ChunkWrapper::V5(c) => c.store(w).context("failed to store rafs v5 chunk"), ChunkWrapper::V6(c) => c.store(w).context("failed to store rafs v5 chunk"), @@ -1671,11 +1670,9 @@ fn to_rafsv5_chunk_info(cki: &dyn BlobV5ChunkInfo) -> RafsV5ChunkInfo { #[cfg(test)] mod tests { use super::*; - use crate::core::context::{BootstrapContext, BUF_WRITER_CAPACITY}; + use crate::core::context::{ArtifactStorage, BootstrapContext}; use rafs::metadata::layout::v6::EROFS_INODE_CHUNK_BASED; use rafs::metadata::RAFS_DEFAULT_CHUNK_SIZE; - use std::fs::OpenOptions; - use std::io::BufWriter; use std::os::unix::fs; use std::path::Path; use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; @@ -1695,16 +1692,8 @@ mod tests { .unwrap(); let bootstrap_path = TempFile::new().unwrap(); - let f_bootstrap = Box::new(BufWriter::with_capacity( - BUF_WRITER_CAPACITY, - OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(bootstrap_path.as_path()) - .unwrap(), - )); - let mut bootstrap_ctx = BootstrapContext::new(f_bootstrap, None); + let storage = ArtifactStorage::SingleFile(bootstrap_path.as_path().to_path_buf()); + let mut bootstrap_ctx = BootstrapContext::new(storage, false).unwrap(); bootstrap_ctx.offset = 1; // reg file. diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index de362df03e1..ff17ecb5d87 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -16,7 +16,6 @@ extern crate serde_json; extern crate lazy_static; use std::fs::{self, metadata, DirEntry, OpenOptions}; -use std::io::BufWriter; use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; @@ -32,8 +31,7 @@ use storage::{compress, RAFS_DEFAULT_CHUNK_SIZE}; use crate::builder::{Builder, DiffBuilder, DirectoryBuilder, StargzBuilder}; use crate::core::chunk_dict::import_chunk_dict; use crate::core::context::{ - BlobManager, BlobStorage, BootstrapContext, BuildContext, RafsVersion, SourceType, - BUF_WRITER_CAPACITY, + ArtifactStorage, BlobManager, BootstrapManager, BuildContext, RafsVersion, SourceType, }; use crate::core::node::{self, WhiteoutSpec}; use crate::core::prefetch::Prefetch; @@ -444,17 +442,6 @@ impl Command { .parse()?; let prefetch = Prefetch::new(prefetch_policy)?; - let bootstrap = Box::new(BufWriter::with_capacity( - BUF_WRITER_CAPACITY, - OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(bootstrap_path) - .with_context(|| format!("failed to create bootstrap file {:?}", bootstrap_path))?, - )); - let mut bootstrap_ctx = BootstrapContext::new(bootstrap, parent_bootstrap); - let mut build_ctx = BuildContext::new( blob_id, aligned_chunk, @@ -471,7 +458,6 @@ impl Command { build_ctx.set_chunk_size(chunk_size); let mut blob_mgr = BlobManager::new(); - if let Some(chunk_dict_arg) = matches.value_of("chunk-dict") { blob_mgr.set_chunk_dict(timing_tracer!( { import_chunk_dict(chunk_dict_arg) }, @@ -479,6 +465,20 @@ impl Command { )?); } + let mut bootstrap_mgr = if source_type == SourceType::Diff { + // TODO: with diff build scenario, use ArtifactStorage::FileDir + // to dump bootstrap for every layer into a directory. + BootstrapManager::new( + ArtifactStorage::SingleFile(PathBuf::from(bootstrap_path)), + None, + ) + } else { + BootstrapManager::new( + ArtifactStorage::SingleFile(PathBuf::from(bootstrap_path)), + parent_bootstrap, + ) + }; + let diff_overlay_hint = matches.is_present("diff-overlay-hint"); let mut builder: Box = match source_type { SourceType::Directory => Box::new(DirectoryBuilder::new()), @@ -488,7 +488,7 @@ impl Command { let (blob_ids, blob_size) = timing_tracer!( { builder - .build(&mut build_ctx, &mut bootstrap_ctx, &mut blob_mgr) + .build(&mut build_ctx, &mut bootstrap_mgr, &mut blob_mgr) .context("build failed") }, "total_build" @@ -604,21 +604,21 @@ impl Command { fn get_blob_storage( matches: &clap::ArgMatches, source_type: SourceType, - ) -> Result> { + ) -> Result> { // Must specify a path to blob file. // For cli/binary interface compatibility sake, keep option `backend-config`, but // it only receives "localfs" backend type and it will be REMOVED in the future let blob_stor = if source_type == SourceType::Directory || source_type == SourceType::Diff { if let Some(p) = matches .value_of("blob") - .map(|b| BlobStorage::SingleFile(b.into())) + .map(|b| ArtifactStorage::SingleFile(b.into())) { Some(p) } else if let Some(d) = matches.value_of("blob-dir").map(PathBuf::from) { if !d.exists() { bail!("Directory holding blobs does not exist") } - Some(BlobStorage::BlobsDir(d)) + Some(ArtifactStorage::FileDir(d)) } else { // Safe because `backend-type` must be specified if `blob` is not with `Directory` source // and `backend-config` must be provided as per clap restriction. @@ -631,7 +631,7 @@ impl Command { if let Some(bf) = config.get("blob_file") { // Even unwrap, it is caused by invalid json. Image creation just can't start. let b: PathBuf = bf.as_str().unwrap().to_string().into(); - Some(BlobStorage::SingleFile(b)) + Some(ArtifactStorage::SingleFile(b)) } else { error!("Wrong backend config input!"); return Err(anyhow!("invalid backend config"));