Add --blob-cache-dir arg use to generate raw blob cache and meta

generate blob cache and blob meta through the —-blob-cache-dir parameters,
so that nydusd can be started directly from these two files without
going to the backend to download. this can improve the performance
of data loading in localfs mode.

Signed-off-by: zyfjeff <zyfjeff@linux.alibaba.com>

builder/src/compact.rs

@@ -21,6 +21,8 @@ use nydus_utils::{digest, try_round_up_4k};
 use serde::{Deserialize, Serialize};
 use sha2::Digest;
 
+use crate::core::context::Artifact;
+
 use super::core::blob::Blob;
 use super::core::bootstrap::Bootstrap;
 use super::{

builder/src/core/blob.rs

@@ -3,7 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::borrow::Cow;
-use std::io::Write;
 use std::slice;
 
 use anyhow::{Context, Result};
@@ -16,9 +15,8 @@ use sha2::digest::Digest;
 
 use super::layout::BlobLayout;
 use super::node::Node;
-use crate::{
-    ArtifactWriter, BlobContext, BlobManager, BuildContext, ConversionType, Feature, Tree,
-};
+use crate::core::context::Artifact;
+use crate::{BlobContext, BlobManager, BuildContext, ConversionType, Feature, Tree};
 
 /// Generator for RAFS data blob.
 pub(crate) struct Blob {}
@@ -29,7 +27,7 @@ impl Blob {
         ctx: &BuildContext,
         tree: &Tree,
         blob_mgr: &mut BlobManager,
-        blob_writer: &mut ArtifactWriter,
+        blob_writer: &mut dyn Artifact,
     ) -> Result<()> {
         match ctx.conversion_type {
             ConversionType::DirectoryToRafs => {
@@ -101,7 +99,7 @@ impl Blob {
     fn finalize_blob_data(
         ctx: &BuildContext,
         blob_mgr: &mut BlobManager,
-        blob_writer: &mut ArtifactWriter,
+        blob_writer: &mut dyn Artifact,
     ) -> Result<()> {
         // Dump buffered batch chunk data if exists.
         if let Some(ref batch) = ctx.blob_batch_generator {
@@ -159,7 +157,7 @@ impl Blob {
     pub(crate) fn dump_meta_data(
         ctx: &BuildContext,
         blob_ctx: &mut BlobContext,
-        blob_writer: &mut ArtifactWriter,
+        blob_writer: &mut dyn Artifact,
     ) -> Result<()> {
         // Dump blob meta for v6 when it has chunks or bootstrap is to be inlined.
         if !blob_ctx.blob_meta_info_enabled || blob_ctx.uncompressed_blob_size == 0 {
@@ -194,7 +192,6 @@ impl Blob {
         } else if ctx.blob_tar_reader.is_some() {
             header.set_separate_blob(true);
         };
-
         let mut compressor = Self::get_compression_algorithm_for_meta(ctx);
         let (compressed_data, compressed) = compress::compress(ci_data, compressor)
             .with_context(|| "failed to compress blob chunk info array".to_string())?;
@@ -223,6 +220,9 @@ impl Blob {
         }
 
         blob_ctx.blob_meta_header = header;
+        if let Some(blob_cache) = ctx.blob_cache_generator.as_ref() {
+            blob_cache.write_blob_meta(ci_data, &header)?;
+        }
         let encrypted_header =
             crypt::encrypt_with_context(header.as_bytes(), cipher_obj, cipher_ctx, encrypt)?;
         let header_size = encrypted_header.len();

builder/src/core/context.rs

@@ -10,6 +10,7 @@ use std::collections::{HashMap, VecDeque};
 use std::convert::TryFrom;
 use std::fs::{remove_file, rename, File, OpenOptions};
 use std::io::{BufWriter, Cursor, Read, Seek, Write};
+use std::mem::size_of;
 use std::os::unix::fs::FileTypeExt;
 use std::path::{Display, Path, PathBuf};
 use std::str::FromStr;
@@ -40,7 +41,7 @@ use nydus_storage::meta::{
     BlobMetaChunkArray, BlobMetaChunkInfo, ZranContextGenerator,
 };
 use nydus_utils::digest::DigestData;
-use nydus_utils::{compress, digest, div_round_up, round_down, BufReaderInfo};
+use nydus_utils::{compress, digest, div_round_up, round_down, try_round_up_4k, BufReaderInfo};
 
 use super::node::ChunkSource;
 use crate::core::tree::TreeNode;
@@ -193,7 +194,13 @@ impl Write for ArtifactMemoryWriter {
     }
 }
 
-struct ArtifactFileWriter(ArtifactWriter);
+struct ArtifactFileWriter(pub ArtifactWriter);
+
+impl ArtifactFileWriter {
+    pub fn finalize(&mut self, name: Option<String>) -> Result<()> {
+        self.0.finalize(name)
+    }
+}
 
 impl RafsIoWrite for ArtifactFileWriter {
     fn as_any(&self) -> &dyn Any {
@@ -215,6 +222,12 @@ impl RafsIoWrite for ArtifactFileWriter {
     }
 }
 
+impl ArtifactFileWriter {
+    pub fn set_len(&mut self, s: u64) -> std::io::Result<()> {
+        self.0.file.get_mut().set_len(s)
+    }
+}
+
 impl Seek for ArtifactFileWriter {
     fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
         self.0.file.seek(pos)
@@ -231,6 +244,37 @@ impl Write for ArtifactFileWriter {
     }
 }
 
+pub trait Artifact: Write {
+    fn pos(&self) -> Result<u64>;
+    fn finalize(&mut self, name: Option<String>) -> Result<()>;
+}
+
+#[derive(Default)]
+pub struct NoopArtifactWriter {
+    pos: usize,
+}
+
+impl Write for NoopArtifactWriter {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.pos += buf.len();
+        Ok(buf.len())
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+impl Artifact for NoopArtifactWriter {
+    fn pos(&self) -> Result<u64> {
+        Ok(self.pos as u64)
+    }
+
+    fn finalize(&mut self, _name: Option<String>) -> Result<()> {
+        Ok(())
+    }
+}
+
 /// ArtifactWriter provides a writer to allow writing bootstrap
 /// or blob data to a single file or in a directory.
 pub struct ArtifactWriter {
@@ -308,36 +352,18 @@ impl ArtifactWriter {
             }
         }
     }
+}
 
+impl Artifact for ArtifactWriter {
     /// Get the current write position.
-    pub fn pos(&self) -> Result<u64> {
+    fn pos(&self) -> Result<u64> {
         Ok(self.pos as u64)
     }
 
-    // The `inline-bootstrap` option merges the blob and bootstrap into one
-    // file. We need some header to index the location of the blob and bootstrap,
-    // write_tar_header uses tar header that arranges the data as follows:
-    // data | tar_header | data | tar_header
-    // This is a tar-like structure, except that we put the tar header after the
-    // data. The advantage is that we do not need to determine the size of the data
-    // first, so that we can write the blob data by stream without seek to improve
-    // the performance of the blob dump by using fifo.
-    fn write_tar_header(&mut self, name: &str, size: u64) -> Result<Header> {
-        let mut header = Header::new_gnu();
-        header.set_path(Path::new(name))?;
-        header.set_entry_type(EntryType::Regular);
-        header.set_size(size);
-        // The checksum must be set to ensure that the tar reader implementation
-        // in golang can correctly parse the header.
-        header.set_cksum();
-        self.write_all(header.as_bytes())?;
-        Ok(header)
-    }
-
     /// Finalize the metadata/data blob.
     ///
     /// When `name` is None, it means that the blob is empty and should be removed.
-    pub fn finalize(&mut self, name: Option<String>) -> Result<()> {
+    fn finalize(&mut self, name: Option<String>) -> Result<()> {
         self.file.flush()?;
 
         if let Some(n) = name {
@@ -367,6 +393,72 @@ impl ArtifactWriter {
     }
 }
 
+pub struct BlobCacheGenerator {
+    blob_data: Mutex<ArtifactFileWriter>,
+    blob_meta: Mutex<ArtifactFileWriter>,
+}
+
+impl BlobCacheGenerator {
+    pub fn new(storage: ArtifactStorage) -> Result<Self> {
+        Ok(BlobCacheGenerator {
+            blob_data: Mutex::new(ArtifactFileWriter(ArtifactWriter::new(storage.clone())?)),
+            blob_meta: Mutex::new(ArtifactFileWriter(ArtifactWriter::new(storage)?)),
+        })
+    }
+
+    pub fn write_blob_meta(
+        &self,
+        data: &[u8],
+        header: &BlobCompressionContextHeader,
+    ) -> Result<()> {
+        let mut guard = self.blob_meta.lock().unwrap();
+        let aligned_uncompressed_size = try_round_up_4k(data.len() as u64).ok_or(anyhow!(
+            format!("invalid input {} for try_round_up_4k", data.len())
+        ))?;
+        guard.set_len(
+            aligned_uncompressed_size + size_of::<BlobCompressionContextHeader>() as u64,
+        )?;
+        guard
+            .write_all(data)
+            .context("failed to write blob meta data")?;
+        guard.seek(std::io::SeekFrom::Start(aligned_uncompressed_size))?;
+        guard
+            .write_all(header.as_bytes())
+            .context("failed to write blob meta header")?;
+        Ok(())
+    }
+
+    pub fn write_blob_data(
+        &self,
+        chunk_data: &[u8],
+        chunk_info: &ChunkWrapper,
+        aligned_d_size: u32,
+    ) -> Result<()> {
+        let mut guard = self.blob_data.lock().unwrap();
+        let curr_pos = guard.seek(std::io::SeekFrom::End(0))?;
+        if curr_pos < chunk_info.uncompressed_offset() + aligned_d_size as u64 {
+            guard.set_len(chunk_info.uncompressed_offset() + aligned_d_size as u64)?;
+        }
+
+        guard.seek(std::io::SeekFrom::Start(chunk_info.uncompressed_offset()))?;
+        guard
+            .write_all(&chunk_data)
+            .context("failed to write blob cache")?;
+        Ok(())
+    }
+
+    pub fn finalize(&self, name: &str) -> Result<()> {
+        let blob_data_name = format!("{}.blob.data", name);
+        let mut guard = self.blob_data.lock().unwrap();
+        guard.finalize(Some(blob_data_name))?;
+        drop(guard);
+
+        let blob_meta_name = format!("{}.blob.meta", name);
+        let mut guard = self.blob_meta.lock().unwrap();
+        guard.finalize(Some(blob_meta_name))
+    }
+}
+
 /// BlobContext is used to hold the blob information of a layer during build.
 pub struct BlobContext {
     /// Blob id (user specified or sha256(blob)).
@@ -731,7 +823,7 @@ impl BlobContext {
     }
 
     /// Helper to write data to blob and update blob hash.
-    pub fn write_data(&mut self, blob_writer: &mut ArtifactWriter, data: &[u8]) -> Result<()> {
+    pub fn write_data(&mut self, blob_writer: &mut dyn Artifact, data: &[u8]) -> Result<()> {
         blob_writer.write_all(data)?;
         self.blob_hash.update(data);
         Ok(())
@@ -740,11 +832,28 @@ impl BlobContext {
     /// Helper to write a tar header to blob and update blob hash.
     pub fn write_tar_header(
         &mut self,
-        blob_writer: &mut ArtifactWriter,
+        blob_writer: &mut dyn Artifact,
         name: &str,
         size: u64,
     ) -> Result<Header> {
-        let header = blob_writer.write_tar_header(name, size)?;
+        // The `inline-bootstrap` option merges the blob and bootstrap into one
+        // file. We need some header to index the location of the blob and bootstrap,
+        // write_tar_header uses tar header that arranges the data as follows:
+        // data | tar_header | data | tar_header
+        // This is a tar-like structure, except that we put the tar header after the
+        // data. The advantage is that we do not need to determine the size of the data
+        // first, so that we can write the blob data by stream without seek to improve
+        // the performance of the blob dump by using fifo.
+
+        let mut header = Header::new_gnu();
+        header.set_path(Path::new(name))?;
+        header.set_entry_type(EntryType::Regular);
+        header.set_size(size);
+        // The checksum must be set to ensure that the tar reader implementation
+        // in golang can correctly parse the header.
+        header.set_cksum();
+
+        blob_writer.write_all(header.as_bytes())?;
         self.blob_hash.update(header.as_bytes());
         Ok(header)
     }
@@ -1182,6 +1291,8 @@ pub struct BuildContext {
 
     pub features: Features,
     pub configuration: Arc<ConfigV2>,
+    /// Generate the blob cache and blob meta
+    pub blob_cache_generator: Option<BlobCacheGenerator>,
 }
 
 impl BuildContext {
@@ -1221,7 +1332,6 @@ impl BuildContext {
         } else {
             crypt::Algorithm::None
         };
-
         BuildContext {
             blob_id,
             aligned_chunk,
@@ -1250,6 +1360,7 @@ impl BuildContext {
 
             features,
             configuration: Arc::new(ConfigV2::default()),
+            blob_cache_generator: None,
         }
     }
 
@@ -1299,6 +1410,7 @@ impl Default for BuildContext {
             blob_inline_meta: false,
             features: Features::new(),
             configuration: Arc::new(ConfigV2::default()),
+            blob_cache_generator: None,
         }
     }
 }