Merge pull request dragonflyoss#538 from ccx1024cc/morgan/nydus-tar-t…

…o-tar

feat: support convert Nydus image layers to tar files

.github/workflows/it.yml

@@ -100,12 +100,13 @@ jobs:
             "target": "musl"
         }
         EOF
-    - name: run test_api
+    - name: run integration tests
       run: |
         cd /home/runner/work/image-service/image-service/contrib/nydus-test
         sudo mkdir -p /blobdir
         sudo python3 nydus_test_config.py --dist fs_structure.yaml
-        sudo pytest -vs --durations=0 --pdb functional-test/test_api.py::test_detect_io_hang \
-                                  functional-test/test_api.py::test_access_pattern \
-                                  functional-test/test_api.py::test_api_mount_with_prefetch \
-                                  functional-test/test_api.py::test_daemon_info
+        # sudo pytest -vs --durations=0 --pdb functional-test/test_api.py::test_detect_io_hang \
+        #                           functional-test/test_api.py::test_access_pattern \
+        #                           functional-test/test_api.py::test_api_mount_with_prefetch \
+        #                           functional-test/test_api.py::test_daemon_info
+        sudo pytest -vs --durations=0 --pdb functional-test/test_stargz.py

contrib/nydus-test/framework/workload_gen.py

@@ -179,20 +179,24 @@ def __verify_one_level(self, path_queue, conn):
                 source_path = os.path.join(self.verify_dir, relpath)
 
                 if os.path.islink(cur_path):
+                    logging.debug("Verifying link file.");
                     if os.readlink(cur_path) != os.readlink(source_path):
                         err_cnt += 1
                         logging.error("Symlink mismatch, %s", cur_path)
                 elif os.path.isfile(cur_path):
                     # TODO: How to verify special files?
+                    logging.debug("Verifying regular file.");
                     cur_md5 = WorkloadGen.calc_file_md5(cur_path)
                     source_md5 = WorkloadGen.calc_file_md5(source_path)
                     if cur_md5 != source_md5:
                         err_cnt += 1
                         logging.error("Verification error. File %s", cur_path)
                         assert False
                 elif stat.S_ISBLK(os.stat(cur_path).st_mode):
+                    logging.debug("Verifying block device.");
                     assert os.stat(cur_path).st_rdev == os.stat(source_path).st_rdev
                 elif stat.S_ISCHR(os.stat(cur_path).st_mode):
+                    logging.debug("Verifying char device.");
                     assert os.stat(cur_path).st_rdev == os.stat(source_path).st_rdev
                 elif stat.S_ISFIFO(os.stat(cur_path).st_mode):
                     pass

contrib/nydus-test/functional-test/test_stargz.py

@@ -28,15 +28,16 @@ def test_stargz(
     """
     intermediator = "tmp.tar.gz"
     stargz_image = "tmp.stargz"
-
-    dist = Distributor(nydus_scratch_image.rootfs(), 4, 4)
-    dist.generate_tree()
-    dirs = dist.put_directories(20)
-    dist.put_multiple_files(100, Size(64, Unit.KB))
-    dist.put_symlinks(30)
-    dist.put_multiple_files(10, Size(4, Unit.MB))
-    dist.put_hardlinks(20)
-    dist.put_single_file(Size(3, Unit.MB), name="test")
+    cmd = ["mkdir", "wq-test"]
+    utils.execute(cmd)
+    dist = Distributor("./wq-test", 4, 4)
+    # dist.generate_tree()
+    # dirs = dist.put_directories(20)
+    # dist.put_multiple_files(1, Size(64, Unit.KB))
+    # dist.put_symlinks(30)
+    # dist.put_multiple_files(1, Size(4, Unit.MB))
+    # dist.put_hardlinks(20)
+    dist.put_single_file(Size(1, Unit.MB), name="test")
     try:
         shutil.rmtree("origin")
     except Exception:
@@ -46,6 +47,8 @@ def test_stargz(
 
     cmd = ["framework/bin/stargzify", f"file:{intermediator}", stargz_image]
     utils.execute(cmd)
+    cmd = ["tar", "zxvf", stargz_image]
+    utils.execute(cmd)
 
     toc = utils.parse_stargz(stargz_image)
     image = RafsImage(
@@ -71,10 +74,10 @@ def test_stargz(
 
     wg = WorkloadGen(nydus_anchor.mount_point, "origin")
 
-    wg.verify_entire_fs()
+    assert wg.verify_entire_fs()
 
-    wg.setup_workload_generator()
-    wg.torture_read(4, 4)
+    # wg.setup_workload_generator()
+    # wg.torture_read(4, 4)
 
-    wg.finish_torture_read()
-    assert not wg.io_error
+    # wg.finish_torture_read()
+    # assert not wg.io_error

src/bin/nydus-image/main.rs

@@ -44,6 +44,7 @@ use crate::core::prefetch::Prefetch;
 use crate::core::tree;
 use crate::merge::Merger;
 use crate::trace::{EventTracerClass, TimingTracerClass, TraceClass};
+use crate::unpack::{OCIUnpacker, Unpacker};
 use crate::validator::Validator;
 
 #[macro_use]
@@ -53,6 +54,7 @@ mod core;
 mod inspect;
 mod merge;
 mod stat;
+mod unpack;
 mod validator;
 
 const BLOB_ID_MAXIMUM_LENGTH: usize = 255;
@@ -509,6 +511,33 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> {
                         .help("path to JSON output file")
                         .takes_value(true))
         )
+        .subcommand(
+            SubCommand::with_name("unpack")
+            .about("Unpack nydus image layer to a tar file")
+            .arg(
+                Arg::with_name("bootstrap")
+                .long("bootstrap")
+                .short("B")
+                .help("path to bootstrap file")
+                .required(true)
+                .takes_value(true))
+            .arg(
+                Arg::with_name("blob")
+                .long("blob")
+                .short("b")
+                .help("path to blob file")
+                .required(false)
+                .takes_value(true)
+                )
+            .arg(
+                Arg::with_name("output")
+                .long("output")
+                .short("o")
+                .help("path to output tar file")
+                .required(true)
+                .takes_value(true)
+                )
+        )
         .arg(
             Arg::with_name("log-file")
                 .long("log-file")
@@ -567,6 +596,8 @@ fn main() -> Result<()> {
         Command::stat(matches)
     } else if let Some(matches) = cmd.subcommand_matches("compact") {
         Command::compact(matches, &build_info)
+    } else if let Some(matches) = cmd.subcommand_matches("unpack") {
+        Command::unpack(matches)
     } else {
         println!("{}", cmd.usage());
         Ok(())
@@ -765,6 +796,17 @@ impl Command {
         Ok(())
     }
 
+    fn unpack(args: &clap::ArgMatches) -> Result<()> {
+        let bootstrap = args.value_of("bootstrap").expect("pass in bootstrap");
+        let blob = args.value_of("blob");
+        let output = args.value_of("output").expect("pass in output");
+
+        let unpacker =
+            OCIUnpacker::new(bootstrap, blob, output).with_context(|| "fail to create unpacker")?;
+
+        unpacker.unpack().with_context(|| "fail to unpack")
+    }
+
     fn check(matches: &clap::ArgMatches, build_info: &BuildTimeInfo) -> Result<()> {
         let bootstrap_path = Self::get_bootstrap(matches)?;
         let verbose = matches.is_present("verbose");