diff --git a/contrib/nydus-test/framework/nydusify.py b/contrib/nydus-test/framework/nydusify.py index fcf62fbc862..4a0f76e1e1a 100644 --- a/contrib/nydus-test/framework/nydusify.py +++ b/contrib/nydus-test/framework/nydusify.py @@ -30,7 +30,8 @@ def work_dir(self, work_dir): return self.set_param("work-dir", work_dir) def fs_version(self, fs_version): - return self.set_param("fs-version", fs_version) + return self.set_param("fs-version", str(fs_version)) + class Nydusify(LinuxCommand): def __init__(self, anchor: NydusAnchor): @@ -45,7 +46,7 @@ def __init__(self, anchor: NydusAnchor): self.cmd = NydusifyParam(self.nydusify_bin) self.cmd.nydus_image(self.image_builder).work_dir(self.work_dir) - def convert(self, source, suffix="_converted", target_ref=None, fs_version="5"): + def convert(self, source, suffix="_converted", target_ref=None, fs_version=5): """ A reference to image looks like registry/namespace/repo:tag Before conversion begins, split the reference into those parts. @@ -69,7 +70,9 @@ def convert(self, source, suffix="_converted", target_ref=None, fs_version="5"): self.target_ref = target_ref cmd = str(self.cmd) - with utils.timer("### Image convertion time including Pull and Push ###"): + with utils.timer( + f"### Rafs V{fs_version} Image conversion time including Pull and Push ###" + ): _, p = utils.run( cmd, False, @@ -80,7 +83,7 @@ def convert(self, source, suffix="_converted", target_ref=None, fs_version="5"): p.wait() assert p.returncode == 0 - def check(self, source, suffix="_converted", target_ref=None, fs_version="5"): + def check(self, source, suffix="_converted", target_ref=None, fs_version=5): """ A reference to image looks like registry/namespace/repo:tag Before conversion begins, split the reference into those parts. @@ -261,12 +264,12 @@ def pull_bootstrap(self, downloaded_dir, bootstrap_name, arch="amd64"): import requests - # Currently, we can handle auth + # Currently, we can not handle auth # OCI distribution spec: /v2//blobs/ os.makedirs(downloaded_dir, exist_ok=True) reader = requests.get( - f"http://{self.registry_url}/v2/{self.original_repo}/blobs/{bootstrap_digest}", + f"http://{self.registry_url}/v2/{self.anchor.registry_namespace}/{self.original_repo}/blobs/{bootstrap_digest}", stream=True, ) with utils.pushd(downloaded_dir): diff --git a/contrib/nydus-test/functional-test/test_nydusify.py b/contrib/nydus-test/functional-test/test_nydusify.py index d0cce192913..b2b5b581f47 100644 --- a/contrib/nydus-test/functional-test/test_nydusify.py +++ b/contrib/nydus-test/functional-test/test_nydusify.py @@ -15,8 +15,72 @@ @pytest.mark.parametrize( "source", [ + "python:3.7", + "docker.io/busybox:latest", + ], +) +@pytest.mark.parametrize("fs_version", [5, 6]) +def test_basic_conversion( + nydus_anchor: NydusAnchor, + rafs_conf: RafsConf, + source, + fs_version, + local_registry, + nydusify_converter, +): + """ + No need to locate where bootstrap is as we can directly pull it from registry + """ + converter = Nydusify(nydus_anchor) + + time.sleep(1) + + converter.docker_v2().enable_multiplatfrom(False).convert( + source, fs_version=fs_version + ) + assert converter.locate_bootstrap() is not None + pulled_bootstrap = converter.pull_bootstrap( + tempfile.TemporaryDirectory( + dir=nydus_anchor.workspace, suffix="bootstrap" + ).name, + "pulled_bootstrap", + ) + + # Skopeo does not support media type: "application/vnd.oci.image.layer.nydus.blob.v1", + # So can't download build cache like a oci image. + + layers, base = converter.extract_source_layers_names_and_download() + nydus_anchor.mount_overlayfs(layers, base) + + converted_layers = converter.extract_converted_layers_names() + converted_layers.sort() + + rafs_conf.set_rafs_backend( + Backend.REGISTRY, repo=posixpath.basename(source).split(":")[0] + ) + rafs_conf.enable_fs_prefetch() + rafs_conf.enable_rafs_blobcache() + rafs_conf.dump_rafs_conf() + + rafs = RafsMount(nydus_anchor, None, rafs_conf) + + # Use `nydus-image inspect` to compare blob table in bootstrap and manifest + + workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs) + # No need to locate where bootstrap is as we can directly pull it from registry + rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount() + + assert workload_gen.verify_entire_fs() + workload_gen.setup_workload_generator() + workload_gen.torture_read(4, 6, verify=True) + workload_gen.finish_torture_read() + + +@pytest.mark.parametrize( + "source", + [ + "python:3.7", "docker.io/busybox:latest", - "reg.docker.alibaba-inc.com/chge-mirrors/python:3.7", ], ) @pytest.mark.parametrize("enable_multiplatform", [False]) @@ -29,7 +93,7 @@ def test_build_cache( nydusify_converter, ): """ - No need to locate where boostrap is as we can directly pull it from registry + No need to locate where bootstrap is as we can directly pull it from registry """ converter = Nydusify(nydus_anchor) @@ -39,7 +103,7 @@ def test_build_cache( "localhost:5000/build_cache:000" ).enable_multiplatfrom(enable_multiplatform).convert(source) - # No need to locate where boostrap is as we can directly pull it from registry + # No need to locate where bootstrap is as we can directly pull it from registry bootstrap = converter.locate_bootstrap() converter.docker_v2().build_cache_ref("localhost:5000/build_cache:000").convert( @@ -83,7 +147,7 @@ def test_build_cache( # Use `nydus-image inspect` to compare blob table in bootstrap and manifest workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs) - # No need to locate where boostrap is as we can directly pull it from registry + # No need to locate where bootstrap is as we can directly pull it from registry rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount() assert workload_gen.verify_entire_fs() @@ -92,16 +156,11 @@ def test_build_cache( workload_gen.finish_torture_read() +@pytest.mark.skip(reason="Get rid of OSS dependency!") @pytest.mark.parametrize( "source", [ - # "reg.docker.alibaba-inc.com/antmesh/odp:1.5.6", - # "reg.docker.alibaba-inc.com/antfin/antcoderpc:20210702-2", - # "reg.docker.alibaba-inc.com/antfin/dpprecompilestream:20210702-1", "docker.io/busybox:latest", - "reg.docker.alibaba-inc.com/chge-mirrors/python:3.7", - "reg.docker.alibaba-inc.com/chge-mirrors/node:13.13.0", - "reg.docker.alibaba-inc.com/chge-mirrors/redis:5.0.5", ], ) def test_upload_oss( @@ -142,10 +201,8 @@ def test_upload_oss( # `check` deletes all files checker = Nydusify(nydus_anchor) - checker.backend_type( - "oss", oss_object_prefix=oss_prefix - ).with_new_work_dir( - nydus_anchor.nydusify_work_dir+'-check' + checker.backend_type("oss", oss_object_prefix=oss_prefix).with_new_work_dir( + nydus_anchor.nydusify_work_dir + "-check" ).check(source) converted_layers = converter.extract_converted_layers_names() @@ -190,93 +247,10 @@ def test_upload_oss( oss.rm(b) -@pytest.mark.parametrize( - "source,chunk_dict_arg", - [ - ( - "reg.docker.alibaba-inc.com/antsys/nydus-image:supportd-1026-1", - "bootstrap:registry:reg.docker.alibaba-inc.com/antsys/chunkdict_test:antfin_supportprod" - ), - ( - "reg.docker.alibaba-inc.com/antsys/nydus-image:mirrorprod-1027-3", - "bootstrap:registry:reg.docker.alibaba-inc.com/antsys/chunkdict_test:antfin_mirrorprod" - ) - ] -) -def test_chunk_dict( - nydus_anchor: NydusAnchor, - rafs_conf: RafsConf, - source, - chunk_dict_arg, - local_registry, - nydusify_converter, -): - ''' - only support oss backend now - - convert image with chunk-dict - - check new image - ''' - converter = Nydusify(nydus_anchor) - - time.sleep(1) - - oss_prefix = "nydus_v2/" - - converter.docker_v2().backend_type( - "oss", oss_object_prefix=oss_prefix - ).build_cache_ref( - "localhost:5000/build_cache:000" - ).force_push().chunk_dict(chunk_dict_arg).convert(source) - - rafs_conf.set_rafs_backend(Backend.OSS, prefix=oss_prefix) - rafs_conf.enable_fs_prefetch() - rafs_conf.enable_rafs_blobcache() - rafs_conf.dump_rafs_conf() - - bootstrap = converter.locate_bootstrap() - - checker = Nydusify(nydus_anchor) - checker.backend_type( - "oss", oss_object_prefix=oss_prefix - ).with_new_work_dir( - nydus_anchor.nydusify_work_dir+'-check' - ).check(source) - - converted_layers = converter.extract_converted_layers_names() - - # With oss backend, ant useage, `layers` only has one member - records = converter.get_build_cache_records("localhost:5000/build_cache:000") - assert len(records) != 0 - cached_layers = [c["digest"] for c in records] - assert cached_layers.sort() == converted_layers.sort() - - pulled_bootstrap = converter.pull_bootstrap( - tempfile.TemporaryDirectory( - dir=nydus_anchor.workspace, suffix="bootstrap" - ).name, - "pulled_bootstrap", - ) - - # Mount source rootfs (ociv1) - layers, base = converter.extract_source_layers_names_and_download() - nydus_anchor.mount_overlayfs(layers, base) - # Mount rafs rootfs - rafs = RafsMount(nydus_anchor, None, rafs_conf) - - workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs) - rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount() - - assert workload_gen.verify_entire_fs() - workload_gen.setup_workload_generator() - workload_gen.torture_read(8, 12, verify=True) - workload_gen.finish_torture_read() - - @pytest.mark.parametrize( "source", [ - "busybox:latest", # From dockerhub, manifest list image foramt, image config includes os/arch - # "reg.docker.alibaba-inc.com/swiftimage/basementtask-assets:12b2b720210429235656909_aarch64", # Single manifest from internal registry + "busybox:latest", # From DockerHub, manifest list image format, image config includes os/arch ], ) @pytest.mark.parametrize("arch", ["arm64", "amd64"]) @@ -370,7 +344,7 @@ def test_cross_platform_multiplatform( # Use `nydus-image inspect` to compare blob table in bootstrap and manifest workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs) - # No need to locate where boostrap is as we can directly pull it from registry + # No need to locate where bootstrap is as we can directly pull it from registry rafs = RafsMount(nydus_anchor, None, rafs_conf) rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount() @@ -378,71 +352,3 @@ def test_cross_platform_multiplatform( workload_gen.setup_workload_generator() workload_gen.torture_read(8, 12, verify=True) workload_gen.finish_torture_read() - - -@pytest.mark.skipif(platform.machine() == "x86_64", reason="Only supported x86") -@pytest.mark.parametrize( - "source", - [ - "reg.docker.alibaba-inc.com/swiftimage/basementtask-assets:12b2b720210429235656909_aarch64", # Single manifest from internal registry - ], -) -def test_arm64_single_manifest( - nydus_anchor: NydusAnchor, - rafs_conf: RafsConf, - source, - local_registry, - nydusify_converter, -): - - converter = Nydusify(nydus_anchor) - arch = "arm64" - - converter.docker_v2().build_cache_ref("localhost:5000/build_cache:000").convert( - source - ) - - # TODO: configure registry backend from `local_registry` rather than anchor - rafs_conf.set_rafs_backend( - Backend.REGISTRY, repo=posixpath.basename(source).split(":")[0] - ) - rafs_conf.enable_fs_prefetch() - rafs_conf.enable_rafs_blobcache() - - pulled_bootstrap = converter.pull_bootstrap( - tempfile.TemporaryDirectory( - dir=nydus_anchor.workspace, suffix="bootstrap" - ).name, - "pulled_bootstrap", - arch, - ) - - # Skopeo does not support media type: "application/vnd.oci.image.layer.nydus.blob.v1", - # So can't download build cache like a oci image. - layers, base = converter.extract_source_layers_names_and_download(arch=arch) - nydus_anchor.mount_overlayfs(layers, base) - - converted_layers = converter.extract_converted_layers_names(arch=arch) - converted_layers.sort() - - records = converter.get_build_cache_records("localhost:5000/build_cache:000") - assert len(records) != 0 - cached_layers = [c["digest"] for c in records] - cached_layers.sort() - # > assert cached_layers == converted_layers - # E AssertionError: assert None == ['sha256:3f18...af3234b4c257'] - # E +None - # E -['sha256:3f18b27a912188108c8590684206bd9da7d81bbfd0e8325f3ef0af3234b4c257'] - for l in converted_layers: - assert l in cached_layers - - # Use `nydus-image inspect` to compare blob table in bootstrap and manifest - workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs) - # No need to locate where boostrap is as we can directly pull it from registry - rafs = RafsMount(nydus_anchor, None, rafs_conf) - rafs.thread_num(2).bootstrap(pulled_bootstrap).prefetch_files("/").mount() - - assert workload_gen.verify_entire_fs() - workload_gen.setup_workload_generator() - workload_gen.torture_read(4, 4, verify=True) - workload_gen.finish_torture_read() diff --git a/contrib/nydus-test/functional-test/test_stargz.py b/contrib/nydus-test/functional-test/test_stargz.py index 79eda3afc26..16a166cf551 100644 --- a/contrib/nydus-test/functional-test/test_stargz.py +++ b/contrib/nydus-test/functional-test/test_stargz.py @@ -51,7 +51,7 @@ def test_stargz( image = RafsImage( nydus_anchor, toc, - "boostrap_scratched", + "bootstrap_scratched", "blob_scratched", clear_from_oss=True, ) @@ -77,4 +77,4 @@ def test_stargz( wg.torture_read(4, 4) wg.finish_torture_read() - assert not wg.io_error \ No newline at end of file + assert not wg.io_error diff --git a/contrib/nydus-test/functional-test/test_stress.py b/contrib/nydus-test/functional-test/test_stress.py index 45456a2a977..9cd652e6f7c 100644 --- a/contrib/nydus-test/functional-test/test_stress.py +++ b/contrib/nydus-test/functional-test/test_stress.py @@ -28,7 +28,7 @@ def setup_module(module): SCRATCH_IMAGE = RafsImage( ANCHOR, ANCHOR.scratch_dir, - "boostrap_scratched", + "bootstrap_scratched", "blob_scratched", clear_from_oss=True, ) diff --git a/src/bin/nydus-image/core/node.rs b/src/bin/nydus-image/core/node.rs index 6e8618feee1..9ea3f9ca50a 100644 --- a/src/bin/nydus-image/core/node.rs +++ b/src/bin/nydus-image/core/node.rs @@ -569,6 +569,7 @@ impl Node { // update all the inodes's offset according to the new 'meta_addr'. self.offset = self.offset - orig_meta_addr + meta_addr; + // Only dir and symlink file can be of EROFS_INODE_FLAT_INLINE layout, // so the `dirents_offset` is only used for these two types. // For other types, the `dirents_offset` will always be 0,