rafe: enhance builder/merger to support RAFS in TARFS mode

Enhance builder/merger to support RAFS in TARFS mode, so we can merge
multiple RAFS filesystems in TARFS mode into one.

Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>

rafs/src/builder/core/bootstrap.rs

@@ -15,7 +15,7 @@ use crate::builder::{
     ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, Tree,
 };
 use crate::metadata::layout::{RafsBlobTable, RAFS_V5_ROOT_INODE};
-use crate::metadata::{RafsSuper, RafsSuperConfig};
+use crate::metadata::{RafsSuper, RafsSuperConfig, RafsSuperFlags};
 
 pub(crate) const STARGZ_DEFAULT_BLOCK_SIZE: u32 = 4 << 20;
 
@@ -295,6 +295,7 @@ impl Bootstrap {
             chunk_size: ctx.chunk_size,
             explicit_uidgid: ctx.explicit_uidgid,
             version: ctx.fs_version,
+            is_tarfs_mode: rs.meta.flags.contains(RafsSuperFlags::TARTFS_MODE),
         };
         config.check_compatibility(&rs.meta)?;
 

rafs/src/builder/core/chunk_dict.rs

@@ -264,6 +264,7 @@ mod tests {
             digester: digest::Algorithm::Blake3,
             chunk_size: 0x100000,
             explicit_uidgid: true,
+            is_tarfs_mode: false,
         };
         let dict =
             HashChunkDict::from_commandline_arg(path, Arc::new(ConfigV2::default()), &rafs_config)

rafs/src/builder/merge.rs

@@ -15,7 +15,8 @@ use nydus_storage::device::{BlobFeatures, BlobInfo};
 
 use super::{
     ArtifactStorage, BlobContext, BlobManager, Bootstrap, BootstrapContext, BuildContext,
-    BuildOutput, ChunkSource, HashChunkDict, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec,
+    BuildOutput, ChunkSource, ConversionType, HashChunkDict, MetadataTreeBuilder, Overlay, Tree,
+    WhiteoutSpec,
 };
 use crate::metadata::{RafsInodeExt, RafsSuper, RafsVersion};
 
@@ -151,6 +152,10 @@ impl Merger {
             ctx.compressor = rs.meta.get_compressor();
             ctx.digester = rs.meta.get_digester();
             ctx.explicit_uidgid = rs.meta.explicit_uidgid();
+            if config.as_ref().unwrap().is_tarfs_mode {
+                ctx.conversion_type = ConversionType::TarToTarfs;
+                ctx.blob_features |= BlobFeatures::TARFS;
+            }
 
             let mut parent_blob_added = false;
             let blobs = &rs.superblock.get_blob_infos();
@@ -259,6 +264,15 @@ impl Merger {
             }
         }
 
+        if ctx.conversion_type == ConversionType::TarToTarfs {
+            if parent_layers > 0 {
+                bail!("merging RAFS in TARFS mode conflicts with `--parent-bootstrap`");
+            }
+            if !chunk_dict_blobs.is_empty() {
+                bail!("merging RAFS in TARFS mode conflicts with `--chunk-dict`");
+            }
+        }
+
         // Safe to unwrap because there is at least one source bootstrap.
         let tree = tree.unwrap();
         ctx.fs_version = fs_version;

rafs/src/metadata/chunk.rs

@@ -15,7 +15,7 @@ use nydus_utils::digest::RafsDigest;
 
 use crate::metadata::cached_v5::CachedChunkInfoV5;
 use crate::metadata::direct_v5::DirectChunkInfoV5;
-use crate::metadata::direct_v6::DirectChunkInfoV6;
+use crate::metadata::direct_v6::{DirectChunkInfoV6, TarfsChunkInfo};
 use crate::metadata::layout::v5::RafsV5ChunkInfo;
 use crate::metadata::{RafsStore, RafsVersion};
 use crate::RafsIoWrite;
@@ -331,10 +331,12 @@ impl ChunkWrapper {
                 *self = Self::V6(to_rafs_v5_chunk_info(cki_v6));
             } else if let Some(cki_v6) = cki.as_any().downcast_ref::<DirectChunkInfoV6>() {
                 *self = Self::V6(to_rafs_v5_chunk_info(cki_v6));
+            } else if let Some(cki_v6) = cki.as_any().downcast_ref::<TarfsChunkInfo>() {
+                *self = Self::V6(to_rafs_v5_chunk_info(cki_v6));
             } else if let Some(cki_v5) = cki.as_any().downcast_ref::<CachedChunkInfoV5>() {
-                *self = Self::V6(to_rafs_v5_chunk_info(cki_v5));
+                *self = Self::V5(to_rafs_v5_chunk_info(cki_v5));
             } else if let Some(cki_v5) = cki.as_any().downcast_ref::<DirectChunkInfoV5>() {
-                *self = Self::V6(to_rafs_v5_chunk_info(cki_v5));
+                *self = Self::V5(to_rafs_v5_chunk_info(cki_v5));
             } else {
                 panic!("unknown chunk information struct");
             }
@@ -347,6 +349,8 @@ fn as_blob_v5_chunk_info(cki: &dyn BlobChunkInfo) -> &dyn BlobV5ChunkInfo {
         cki_v6
     } else if let Some(cki_v6) = cki.as_any().downcast_ref::<DirectChunkInfoV6>() {
         cki_v6
+    } else if let Some(cki_v6) = cki.as_any().downcast_ref::<TarfsChunkInfo>() {
+        cki_v6
     } else if let Some(cki_v5) = cki.as_any().downcast_ref::<CachedChunkInfoV5>() {
         cki_v5
     } else if let Some(cki_v5) = cki.as_any().downcast_ref::<DirectChunkInfoV5>() {

rafs/src/metadata/direct_v6.rs

@@ -1466,3 +1466,21 @@ impl BlobChunkInfo for TarfsChunkInfo {
         self
     }
 }
+
+impl BlobV5ChunkInfo for TarfsChunkInfo {
+    fn index(&self) -> u32 {
+        self.chunk_index
+    }
+
+    fn file_offset(&self) -> u64 {
+        0
+    }
+
+    fn flags(&self) -> BlobChunkFlags {
+        BlobChunkFlags::empty()
+    }
+
+    fn as_base(&self) -> &dyn BlobChunkInfo {
+        self
+    }
+}

rafs/src/metadata/mod.rs

@@ -369,6 +369,8 @@ pub struct RafsSuperConfig {
     pub chunk_size: u32,
     /// Whether `explicit_uidgid` enabled or not.
     pub explicit_uidgid: bool,
+    /// RAFS in TARFS mode.
+    pub is_tarfs_mode: bool,
 }
 
 impl RafsSuperConfig {
@@ -405,6 +407,11 @@ impl RafsSuperConfig {
             )));
         }
 
+        let is_tarfs_mode = meta.flags.contains(RafsSuperFlags::TARTFS_MODE);
+        if is_tarfs_mode != self.is_tarfs_mode {
+            return Err(einval!(format!("Using inconsistent RAFS TARFS mode")));
+        }
+
         Ok(())
     }
 }
@@ -519,6 +526,7 @@ impl RafsSuperMeta {
             digester: self.get_digester(),
             chunk_size: self.chunk_size,
             explicit_uidgid: self.explicit_uidgid(),
+            is_tarfs_mode: self.flags.contains(RafsSuperFlags::TARTFS_MODE),
         }
     }
 }

src/bin/nydus-image/main.rs

@@ -883,6 +883,7 @@ impl Command {
                 digester,
                 chunk_size,
                 explicit_uidgid: !repeatable,
+                is_tarfs_mode: false,
             };
             let rafs_config = Arc::new(build_ctx.configuration.as_ref().clone());
             // The separate chunk dict bootstrap doesn't support blob accessible.