add basic nydusify conversion case

contrib/nydus-test/framework/nydusify.py

@@ -30,7 +30,8 @@ def work_dir(self, work_dir):
         return self.set_param("work-dir", work_dir)
 
     def fs_version(self, fs_version):
-        return self.set_param("fs-version", fs_version)
+        return self.set_param("fs-version", str(fs_version))
+
 
 class Nydusify(LinuxCommand):
     def __init__(self, anchor: NydusAnchor):
@@ -45,7 +46,7 @@ def __init__(self, anchor: NydusAnchor):
         self.cmd = NydusifyParam(self.nydusify_bin)
         self.cmd.nydus_image(self.image_builder).work_dir(self.work_dir)
 
-    def convert(self, source, suffix="_converted", target_ref=None, fs_version="5"):
+    def convert(self, source, suffix="_converted", target_ref=None, fs_version=5):
         """
         A reference to image looks like registry/namespace/repo:tag
         Before conversion begins, split the reference into those parts.
@@ -69,7 +70,9 @@ def convert(self, source, suffix="_converted", target_ref=None, fs_version="5"):
         self.target_ref = target_ref
 
         cmd = str(self.cmd)
-        with utils.timer("### Image convertion time including Pull and Push ###"):
+        with utils.timer(
+            f"### Rafs V{fs_version} Image conversion time including Pull and Push ###"
+        ):
             _, p = utils.run(
                 cmd,
                 False,
@@ -80,7 +83,7 @@ def convert(self, source, suffix="_converted", target_ref=None, fs_version="5"):
             p.wait()
             assert p.returncode == 0
 
-    def check(self, source, suffix="_converted", target_ref=None, fs_version="5"):
+    def check(self, source, suffix="_converted", target_ref=None, fs_version=5):
         """
         A reference to image looks like registry/namespace/repo:tag
         Before conversion begins, split the reference into those parts.
@@ -261,12 +264,12 @@ def pull_bootstrap(self, downloaded_dir, bootstrap_name, arch="amd64"):
 
         import requests
 
-        # Currently, we can handle auth
+        # Currently, we can not handle auth
         # OCI distribution spec: /v2/<name>/blobs/<digest>
         os.makedirs(downloaded_dir, exist_ok=True)
 
         reader = requests.get(
-            f"http://{self.registry_url}/v2/{self.original_repo}/blobs/{bootstrap_digest}",
+            f"http://{self.registry_url}/v2/{self.anchor.registry_namespace}/{self.original_repo}/blobs/{bootstrap_digest}",
             stream=True,
         )
         with utils.pushd(downloaded_dir):

contrib/nydus-test/functional-test/test_nydusify.py

@@ -15,8 +15,72 @@
 @pytest.mark.parametrize(
     "source",
     [
+        "python:3.7",
+        "docker.io/busybox:latest",
+    ],
+)
+@pytest.mark.parametrize("fs_version", [5, 6])
+def test_basic_conversion(
+    nydus_anchor: NydusAnchor,
+    rafs_conf: RafsConf,
+    source,
+    fs_version,
+    local_registry,
+    nydusify_converter,
+):
+    """
+    No need to locate where bootstrap is as we can directly pull it from registry
+    """
+    converter = Nydusify(nydus_anchor)
+
+    time.sleep(1)
+
+    converter.docker_v2().enable_multiplatfrom(False).convert(
+        source, fs_version=fs_version
+    )
+    assert converter.locate_bootstrap() is not None
+    pulled_bootstrap = converter.pull_bootstrap(
+        tempfile.TemporaryDirectory(
+            dir=nydus_anchor.workspace, suffix="bootstrap"
+        ).name,
+        "pulled_bootstrap",
+    )
+
+    # Skopeo does not support media type: "application/vnd.oci.image.layer.nydus.blob.v1",
+    # So can't download build cache like a oci image.
+
+    layers, base = converter.extract_source_layers_names_and_download()
+    nydus_anchor.mount_overlayfs(layers, base)
+
+    converted_layers = converter.extract_converted_layers_names()
+    converted_layers.sort()
+
+    rafs_conf.set_rafs_backend(
+        Backend.REGISTRY, repo=posixpath.basename(source).split(":")[0]
+    )
+    rafs_conf.enable_fs_prefetch()
+    rafs_conf.enable_rafs_blobcache()
+    rafs_conf.dump_rafs_conf()
+
+    rafs = RafsMount(nydus_anchor, None, rafs_conf)
+
+    # Use `nydus-image inspect` to compare blob table in bootstrap and manifest
+
+    workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
+    # No need to locate where bootstrap is as we can directly pull it from registry
+    rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount()
+
+    assert workload_gen.verify_entire_fs()
+    workload_gen.setup_workload_generator()
+    workload_gen.torture_read(4, 6, verify=True)
+    workload_gen.finish_torture_read()
+
+
+@pytest.mark.parametrize(
+    "source",
+    [
+        "python:3.7",
         "docker.io/busybox:latest",
-        "reg.docker.alibaba-inc.com/chge-mirrors/python:3.7",
     ],
 )
 @pytest.mark.parametrize("enable_multiplatform", [False])
@@ -29,7 +93,7 @@ def test_build_cache(
     nydusify_converter,
 ):
     """
-    No need to locate where boostrap is as we can directly pull it from registry
+    No need to locate where bootstrap is as we can directly pull it from registry
     """
     converter = Nydusify(nydus_anchor)
 
@@ -39,7 +103,7 @@ def test_build_cache(
         "localhost:5000/build_cache:000"
     ).enable_multiplatfrom(enable_multiplatform).convert(source)
 
-    # No need to locate where boostrap is as we can directly pull it from registry
+    # No need to locate where bootstrap is as we can directly pull it from registry
     bootstrap = converter.locate_bootstrap()
 
     converter.docker_v2().build_cache_ref("localhost:5000/build_cache:000").convert(
@@ -83,7 +147,7 @@ def test_build_cache(
     # Use `nydus-image inspect` to compare blob table in bootstrap and manifest
 
     workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
-    # No need to locate where boostrap is as we can directly pull it from registry
+    # No need to locate where bootstrap is as we can directly pull it from registry
     rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount()
 
     assert workload_gen.verify_entire_fs()
@@ -92,16 +156,11 @@ def test_build_cache(
     workload_gen.finish_torture_read()
 
 
+@pytest.mark.skip(reason="Get rid of OSS dependency!")
 @pytest.mark.parametrize(
     "source",
     [
-        # "reg.docker.alibaba-inc.com/antmesh/odp:1.5.6",
-        # "reg.docker.alibaba-inc.com/antfin/antcoderpc:20210702-2",
-        # "reg.docker.alibaba-inc.com/antfin/dpprecompilestream:20210702-1",
         "docker.io/busybox:latest",
-        "reg.docker.alibaba-inc.com/chge-mirrors/python:3.7",
-        "reg.docker.alibaba-inc.com/chge-mirrors/node:13.13.0",
-        "reg.docker.alibaba-inc.com/chge-mirrors/redis:5.0.5",
     ],
 )
 def test_upload_oss(
@@ -142,10 +201,8 @@ def test_upload_oss(
 
     # `check` deletes all files
     checker = Nydusify(nydus_anchor)
-    checker.backend_type(
-        "oss", oss_object_prefix=oss_prefix
-    ).with_new_work_dir(
-        nydus_anchor.nydusify_work_dir+'-check'
+    checker.backend_type("oss", oss_object_prefix=oss_prefix).with_new_work_dir(
+        nydus_anchor.nydusify_work_dir + "-check"
     ).check(source)
 
     converted_layers = converter.extract_converted_layers_names()
@@ -190,93 +247,10 @@ def test_upload_oss(
         oss.rm(b)
 
 
-@pytest.mark.parametrize(
-    "source,chunk_dict_arg",
-    [
-        (
-            "reg.docker.alibaba-inc.com/antsys/nydus-image:supportd-1026-1",
-            "bootstrap:registry:reg.docker.alibaba-inc.com/antsys/chunkdict_test:antfin_supportprod"
-        ),
-        (
-            "reg.docker.alibaba-inc.com/antsys/nydus-image:mirrorprod-1027-3",
-            "bootstrap:registry:reg.docker.alibaba-inc.com/antsys/chunkdict_test:antfin_mirrorprod"
-        )
-    ]
-)
-def test_chunk_dict(
-    nydus_anchor: NydusAnchor,
-    rafs_conf: RafsConf,
-    source,
-    chunk_dict_arg,
-    local_registry,
-    nydusify_converter,
-):
-    '''
-    only support oss backend now
-    - convert image with chunk-dict
-    - check new image
-    '''
-    converter = Nydusify(nydus_anchor)
-
-    time.sleep(1)
-
-    oss_prefix = "nydus_v2/"
-
-    converter.docker_v2().backend_type(
-        "oss", oss_object_prefix=oss_prefix
-    ).build_cache_ref(
-        "localhost:5000/build_cache:000"
-    ).force_push().chunk_dict(chunk_dict_arg).convert(source)
-
-    rafs_conf.set_rafs_backend(Backend.OSS, prefix=oss_prefix)
-    rafs_conf.enable_fs_prefetch()
-    rafs_conf.enable_rafs_blobcache()
-    rafs_conf.dump_rafs_conf()
-
-    bootstrap = converter.locate_bootstrap()
-
-    checker = Nydusify(nydus_anchor)
-    checker.backend_type(
-        "oss", oss_object_prefix=oss_prefix
-    ).with_new_work_dir(
-        nydus_anchor.nydusify_work_dir+'-check'
-    ).check(source)
-
-    converted_layers = converter.extract_converted_layers_names()
-
-    # With oss backend, ant useage, `layers` only has one member
-    records = converter.get_build_cache_records("localhost:5000/build_cache:000")
-    assert len(records) != 0
-    cached_layers = [c["digest"] for c in records]
-    assert cached_layers.sort() == converted_layers.sort()
-
-    pulled_bootstrap = converter.pull_bootstrap(
-        tempfile.TemporaryDirectory(
-            dir=nydus_anchor.workspace, suffix="bootstrap"
-        ).name,
-        "pulled_bootstrap",
-    )
-
-    # Mount source rootfs (ociv1)
-    layers, base = converter.extract_source_layers_names_and_download()
-    nydus_anchor.mount_overlayfs(layers, base)
-    # Mount rafs rootfs
-    rafs = RafsMount(nydus_anchor, None, rafs_conf)
-
-    workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
-    rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount()
-
-    assert workload_gen.verify_entire_fs()
-    workload_gen.setup_workload_generator()
-    workload_gen.torture_read(8, 12, verify=True)
-    workload_gen.finish_torture_read()
-
-
 @pytest.mark.parametrize(
     "source",
     [
-        "busybox:latest",  # From dockerhub, manifest list image foramt, image config includes os/arch
-        # "reg.docker.alibaba-inc.com/swiftimage/basementtask-assets:12b2b720210429235656909_aarch64",  # Single manifest from internal registry
+        "busybox:latest",  # From DockerHub, manifest list image format, image config includes os/arch
     ],
 )
 @pytest.mark.parametrize("arch", ["arm64", "amd64"])
@@ -370,79 +344,11 @@ def test_cross_platform_multiplatform(
 
     # Use `nydus-image inspect` to compare blob table in bootstrap and manifest
     workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
-    # No need to locate where boostrap is as we can directly pull it from registry
+    # No need to locate where bootstrap is as we can directly pull it from registry
     rafs = RafsMount(nydus_anchor, None, rafs_conf)
     rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount()
 
     assert workload_gen.verify_entire_fs()
     workload_gen.setup_workload_generator()
     workload_gen.torture_read(8, 12, verify=True)
     workload_gen.finish_torture_read()
-
-
-@pytest.mark.skipif(platform.machine() == "x86_64", reason="Only supported x86")
-@pytest.mark.parametrize(
-    "source",
-    [
-        "reg.docker.alibaba-inc.com/swiftimage/basementtask-assets:12b2b720210429235656909_aarch64",  # Single manifest from internal registry
-    ],
-)
-def test_arm64_single_manifest(
-    nydus_anchor: NydusAnchor,
-    rafs_conf: RafsConf,
-    source,
-    local_registry,
-    nydusify_converter,
-):
-
-    converter = Nydusify(nydus_anchor)
-    arch = "arm64"
-
-    converter.docker_v2().build_cache_ref("localhost:5000/build_cache:000").convert(
-        source
-    )
-
-    # TODO: configure registry backend from `local_registry` rather than anchor
-    rafs_conf.set_rafs_backend(
-        Backend.REGISTRY, repo=posixpath.basename(source).split(":")[0]
-    )
-    rafs_conf.enable_fs_prefetch()
-    rafs_conf.enable_rafs_blobcache()
-
-    pulled_bootstrap = converter.pull_bootstrap(
-        tempfile.TemporaryDirectory(
-            dir=nydus_anchor.workspace, suffix="bootstrap"
-        ).name,
-        "pulled_bootstrap",
-        arch,
-    )
-
-    # Skopeo does not support media type: "application/vnd.oci.image.layer.nydus.blob.v1",
-    # So can't download build cache like a oci image.
-    layers, base = converter.extract_source_layers_names_and_download(arch=arch)
-    nydus_anchor.mount_overlayfs(layers, base)
-
-    converted_layers = converter.extract_converted_layers_names(arch=arch)
-    converted_layers.sort()
-
-    records = converter.get_build_cache_records("localhost:5000/build_cache:000")
-    assert len(records) != 0
-    cached_layers = [c["digest"] for c in records]
-    cached_layers.sort()
-    # >       assert cached_layers == converted_layers
-    # E       AssertionError: assert None == ['sha256:3f18...af3234b4c257']
-    # E         +None
-    # E         -['sha256:3f18b27a912188108c8590684206bd9da7d81bbfd0e8325f3ef0af3234b4c257']
-    for l in converted_layers:
-        assert l in cached_layers
-
-    # Use `nydus-image inspect` to compare blob table in bootstrap and manifest
-    workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
-    # No need to locate where boostrap is as we can directly pull it from registry
-    rafs = RafsMount(nydus_anchor, None, rafs_conf)
-    rafs.thread_num(2).bootstrap(pulled_bootstrap).prefetch_files("/").mount()
-
-    assert workload_gen.verify_entire_fs()
-    workload_gen.setup_workload_generator()
-    workload_gen.torture_read(4, 4, verify=True)
-    workload_gen.finish_torture_read()

contrib/nydus-test/functional-test/test_stargz.py

@@ -51,7 +51,7 @@ def test_stargz(
     image = RafsImage(
         nydus_anchor,
         toc,
-        "boostrap_scratched",
+        "bootstrap_scratched",
         "blob_scratched",
         clear_from_oss=True,
     )
@@ -77,4 +77,4 @@ def test_stargz(
     wg.torture_read(4, 4)
 
     wg.finish_torture_read()
-    assert not wg.io_error
+    assert not wg.io_error

contrib/nydus-test/functional-test/test_stress.py

@@ -28,7 +28,7 @@ def setup_module(module):
     SCRATCH_IMAGE = RafsImage(
         ANCHOR,
         ANCHOR.scratch_dir,
-        "boostrap_scratched",
+        "bootstrap_scratched",
         "blob_scratched",
         clear_from_oss=True,
     )

src/bin/nydus-image/core/node.rs

@@ -569,6 +569,7 @@ impl Node {
 
         // update all the inodes's offset according to the new 'meta_addr'.
         self.offset = self.offset - orig_meta_addr + meta_addr;
+
         // Only dir and symlink file can be of EROFS_INODE_FLAT_INLINE layout,
         // so the `dirents_offset` is only used for these two types.
         // For other types, the `dirents_offset` will always be 0,