Skip to content

Commit

Permalink
Merge pull request #566 from changweige/impr-test
Browse files Browse the repository at this point in the history 
add basic nydusify conversion case
  • Loading branch information
@jiangliu
jiangliu authored Jul 6, 2022
2 parents 29e8a87 + 8402360 commit 65ebb7f
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 176 deletions.
15 changes: 9 additions & 6 deletions contrib/nydus-test/framework/nydusify.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ def work_dir(self, work_dir):
return self.set_param("work-dir", work_dir)

def fs_version(self, fs_version):
return self.set_param("fs-version", fs_version)
return self.set_param("fs-version", str(fs_version))


class Nydusify(LinuxCommand):
def __init__(self, anchor: NydusAnchor):
Expand All @@ -45,7 +46,7 @@ def __init__(self, anchor: NydusAnchor):
self.cmd = NydusifyParam(self.nydusify_bin)
self.cmd.nydus_image(self.image_builder).work_dir(self.work_dir)

def convert(self, source, suffix="_converted", target_ref=None, fs_version="5"):
def convert(self, source, suffix="_converted", target_ref=None, fs_version=5):
"""
A reference to image looks like registry/namespace/repo:tag
Before conversion begins, split the reference into those parts.
Expand All @@ -69,7 +70,9 @@ def convert(self, source, suffix="_converted", target_ref=None, fs_version="5"):
self.target_ref = target_ref

cmd = str(self.cmd)
with utils.timer("### Image convertion time including Pull and Push ###"):
with utils.timer(
f"### Rafs V{fs_version} Image conversion time including Pull and Push ###"
):
_, p = utils.run(
cmd,
False,
Expand All @@ -80,7 +83,7 @@ def convert(self, source, suffix="_converted", target_ref=None, fs_version="5"):
p.wait()
assert p.returncode == 0

def check(self, source, suffix="_converted", target_ref=None, fs_version="5"):
def check(self, source, suffix="_converted", target_ref=None, fs_version=5):
"""
A reference to image looks like registry/namespace/repo:tag
Before conversion begins, split the reference into those parts.
Expand Down Expand Up @@ -261,12 +264,12 @@ def pull_bootstrap(self, downloaded_dir, bootstrap_name, arch="amd64"):

import requests

# Currently, we can handle auth
# Currently, we can not handle auth
# OCI distribution spec: /v2/<name>/blobs/<digest>
os.makedirs(downloaded_dir, exist_ok=True)

reader = requests.get(
f"http://{self.registry_url}/v2/{self.original_repo}/blobs/{bootstrap_digest}",
f"http://{self.registry_url}/v2/{self.anchor.registry_namespace}/{self.original_repo}/blobs/{bootstrap_digest}",
stream=True,
)
with utils.pushd(downloaded_dir):
Expand Down
240 changes: 73 additions & 167 deletions contrib/nydus-test/functional-test/test_nydusify.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,72 @@
@pytest.mark.parametrize(
"source",
[
"python:3.7",
"docker.io/busybox:latest",
],
)
@pytest.mark.parametrize("fs_version", [5, 6])
def test_basic_conversion(
nydus_anchor: NydusAnchor,
rafs_conf: RafsConf,
source,
fs_version,
local_registry,
nydusify_converter,
):
"""
No need to locate where bootstrap is as we can directly pull it from registry
"""
converter = Nydusify(nydus_anchor)

time.sleep(1)

converter.docker_v2().enable_multiplatfrom(False).convert(
source, fs_version=fs_version
)
assert converter.locate_bootstrap() is not None
pulled_bootstrap = converter.pull_bootstrap(
tempfile.TemporaryDirectory(
dir=nydus_anchor.workspace, suffix="bootstrap"
).name,
"pulled_bootstrap",
)

# Skopeo does not support media type: "application/vnd.oci.image.layer.nydus.blob.v1",
# So can't download build cache like a oci image.

layers, base = converter.extract_source_layers_names_and_download()
nydus_anchor.mount_overlayfs(layers, base)

converted_layers = converter.extract_converted_layers_names()
converted_layers.sort()

rafs_conf.set_rafs_backend(
Backend.REGISTRY, repo=posixpath.basename(source).split(":")[0]
)
rafs_conf.enable_fs_prefetch()
rafs_conf.enable_rafs_blobcache()
rafs_conf.dump_rafs_conf()

rafs = RafsMount(nydus_anchor, None, rafs_conf)

# Use `nydus-image inspect` to compare blob table in bootstrap and manifest

workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
# No need to locate where bootstrap is as we can directly pull it from registry
rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount()

assert workload_gen.verify_entire_fs()
workload_gen.setup_workload_generator()
workload_gen.torture_read(4, 6, verify=True)
workload_gen.finish_torture_read()


@pytest.mark.parametrize(
"source",
[
"python:3.7",
"docker.io/busybox:latest",
"reg.docker.alibaba-inc.com/chge-mirrors/python:3.7",
],
)
@pytest.mark.parametrize("enable_multiplatform", [False])
Expand All @@ -29,7 +93,7 @@ def test_build_cache(
nydusify_converter,
):
"""
No need to locate where boostrap is as we can directly pull it from registry
No need to locate where bootstrap is as we can directly pull it from registry
"""
converter = Nydusify(nydus_anchor)

Expand All @@ -39,7 +103,7 @@ def test_build_cache(
"localhost:5000/build_cache:000"
).enable_multiplatfrom(enable_multiplatform).convert(source)

# No need to locate where boostrap is as we can directly pull it from registry
# No need to locate where bootstrap is as we can directly pull it from registry
bootstrap = converter.locate_bootstrap()

converter.docker_v2().build_cache_ref("localhost:5000/build_cache:000").convert(
Expand Down Expand Up @@ -83,7 +147,7 @@ def test_build_cache(
# Use `nydus-image inspect` to compare blob table in bootstrap and manifest

workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
# No need to locate where boostrap is as we can directly pull it from registry
# No need to locate where bootstrap is as we can directly pull it from registry
rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount()

assert workload_gen.verify_entire_fs()
Expand All @@ -92,16 +156,11 @@ def test_build_cache(
workload_gen.finish_torture_read()


@pytest.mark.skip(reason="Get rid of OSS dependency!")
@pytest.mark.parametrize(
"source",
[
# "reg.docker.alibaba-inc.com/antmesh/odp:1.5.6",
# "reg.docker.alibaba-inc.com/antfin/antcoderpc:20210702-2",
# "reg.docker.alibaba-inc.com/antfin/dpprecompilestream:20210702-1",
"docker.io/busybox:latest",
"reg.docker.alibaba-inc.com/chge-mirrors/python:3.7",
"reg.docker.alibaba-inc.com/chge-mirrors/node:13.13.0",
"reg.docker.alibaba-inc.com/chge-mirrors/redis:5.0.5",
],
)
def test_upload_oss(
Expand Down Expand Up @@ -142,10 +201,8 @@ def test_upload_oss(

# `check` deletes all files
checker = Nydusify(nydus_anchor)
checker.backend_type(
"oss", oss_object_prefix=oss_prefix
).with_new_work_dir(
nydus_anchor.nydusify_work_dir+'-check'
checker.backend_type("oss", oss_object_prefix=oss_prefix).with_new_work_dir(
nydus_anchor.nydusify_work_dir + "-check"
).check(source)

converted_layers = converter.extract_converted_layers_names()
Expand Down Expand Up @@ -190,93 +247,10 @@ def test_upload_oss(
oss.rm(b)


@pytest.mark.parametrize(
"source,chunk_dict_arg",
[
(
"reg.docker.alibaba-inc.com/antsys/nydus-image:supportd-1026-1",
"bootstrap:registry:reg.docker.alibaba-inc.com/antsys/chunkdict_test:antfin_supportprod"
),
(
"reg.docker.alibaba-inc.com/antsys/nydus-image:mirrorprod-1027-3",
"bootstrap:registry:reg.docker.alibaba-inc.com/antsys/chunkdict_test:antfin_mirrorprod"
)
]
)
def test_chunk_dict(
nydus_anchor: NydusAnchor,
rafs_conf: RafsConf,
source,
chunk_dict_arg,
local_registry,
nydusify_converter,
):
'''
only support oss backend now
- convert image with chunk-dict
- check new image
'''
converter = Nydusify(nydus_anchor)

time.sleep(1)

oss_prefix = "nydus_v2/"

converter.docker_v2().backend_type(
"oss", oss_object_prefix=oss_prefix
).build_cache_ref(
"localhost:5000/build_cache:000"
).force_push().chunk_dict(chunk_dict_arg).convert(source)

rafs_conf.set_rafs_backend(Backend.OSS, prefix=oss_prefix)
rafs_conf.enable_fs_prefetch()
rafs_conf.enable_rafs_blobcache()
rafs_conf.dump_rafs_conf()

bootstrap = converter.locate_bootstrap()

checker = Nydusify(nydus_anchor)
checker.backend_type(
"oss", oss_object_prefix=oss_prefix
).with_new_work_dir(
nydus_anchor.nydusify_work_dir+'-check'
).check(source)

converted_layers = converter.extract_converted_layers_names()

# With oss backend, ant useage, `layers` only has one member
records = converter.get_build_cache_records("localhost:5000/build_cache:000")
assert len(records) != 0
cached_layers = [c["digest"] for c in records]
assert cached_layers.sort() == converted_layers.sort()

pulled_bootstrap = converter.pull_bootstrap(
tempfile.TemporaryDirectory(
dir=nydus_anchor.workspace, suffix="bootstrap"
).name,
"pulled_bootstrap",
)

# Mount source rootfs (ociv1)
layers, base = converter.extract_source_layers_names_and_download()
nydus_anchor.mount_overlayfs(layers, base)
# Mount rafs rootfs
rafs = RafsMount(nydus_anchor, None, rafs_conf)

workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount()

assert workload_gen.verify_entire_fs()
workload_gen.setup_workload_generator()
workload_gen.torture_read(8, 12, verify=True)
workload_gen.finish_torture_read()


@pytest.mark.parametrize(
"source",
[
"busybox:latest", # From dockerhub, manifest list image foramt, image config includes os/arch
# "reg.docker.alibaba-inc.com/swiftimage/basementtask-assets:12b2b720210429235656909_aarch64", # Single manifest from internal registry
"busybox:latest", # From DockerHub, manifest list image format, image config includes os/arch
],
)
@pytest.mark.parametrize("arch", ["arm64", "amd64"])
Expand Down Expand Up @@ -370,79 +344,11 @@ def test_cross_platform_multiplatform(

# Use `nydus-image inspect` to compare blob table in bootstrap and manifest
workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
# No need to locate where boostrap is as we can directly pull it from registry
# No need to locate where bootstrap is as we can directly pull it from registry
rafs = RafsMount(nydus_anchor, None, rafs_conf)
rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount()

assert workload_gen.verify_entire_fs()
workload_gen.setup_workload_generator()
workload_gen.torture_read(8, 12, verify=True)
workload_gen.finish_torture_read()


@pytest.mark.skipif(platform.machine() == "x86_64", reason="Only supported x86")
@pytest.mark.parametrize(
"source",
[
"reg.docker.alibaba-inc.com/swiftimage/basementtask-assets:12b2b720210429235656909_aarch64", # Single manifest from internal registry
],
)
def test_arm64_single_manifest(
nydus_anchor: NydusAnchor,
rafs_conf: RafsConf,
source,
local_registry,
nydusify_converter,
):

converter = Nydusify(nydus_anchor)
arch = "arm64"

converter.docker_v2().build_cache_ref("localhost:5000/build_cache:000").convert(
source
)

# TODO: configure registry backend from `local_registry` rather than anchor
rafs_conf.set_rafs_backend(
Backend.REGISTRY, repo=posixpath.basename(source).split(":")[0]
)
rafs_conf.enable_fs_prefetch()
rafs_conf.enable_rafs_blobcache()

pulled_bootstrap = converter.pull_bootstrap(
tempfile.TemporaryDirectory(
dir=nydus_anchor.workspace, suffix="bootstrap"
).name,
"pulled_bootstrap",
arch,
)

# Skopeo does not support media type: "application/vnd.oci.image.layer.nydus.blob.v1",
# So can't download build cache like a oci image.
layers, base = converter.extract_source_layers_names_and_download(arch=arch)
nydus_anchor.mount_overlayfs(layers, base)

converted_layers = converter.extract_converted_layers_names(arch=arch)
converted_layers.sort()

records = converter.get_build_cache_records("localhost:5000/build_cache:000")
assert len(records) != 0
cached_layers = [c["digest"] for c in records]
cached_layers.sort()
# > assert cached_layers == converted_layers
# E AssertionError: assert None == ['sha256:3f18...af3234b4c257']
# E +None
# E -['sha256:3f18b27a912188108c8590684206bd9da7d81bbfd0e8325f3ef0af3234b4c257']
for l in converted_layers:
assert l in cached_layers

# Use `nydus-image inspect` to compare blob table in bootstrap and manifest
workload_gen = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
# No need to locate where boostrap is as we can directly pull it from registry
rafs = RafsMount(nydus_anchor, None, rafs_conf)
rafs.thread_num(2).bootstrap(pulled_bootstrap).prefetch_files("/").mount()

assert workload_gen.verify_entire_fs()
workload_gen.setup_workload_generator()
workload_gen.torture_read(4, 4, verify=True)
workload_gen.finish_torture_read()
4 changes: 2 additions & 2 deletions contrib/nydus-test/functional-test/test_stargz.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def test_stargz(
image = RafsImage(
nydus_anchor,
toc,
"boostrap_scratched",
"bootstrap_scratched",
"blob_scratched",
clear_from_oss=True,
)
Expand All @@ -77,4 +77,4 @@ def test_stargz(
wg.torture_read(4, 4)

wg.finish_torture_read()
assert not wg.io_error
assert not wg.io_error
2 changes: 1 addition & 1 deletion contrib/nydus-test/functional-test/test_stress.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def setup_module(module):
SCRATCH_IMAGE = RafsImage(
ANCHOR,
ANCHOR.scratch_dir,
"boostrap_scratched",
"bootstrap_scratched",
"blob_scratched",
clear_from_oss=True,
)
Expand Down
1 change: 1 addition & 0 deletions src/bin/nydus-image/core/node.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,7 @@ impl Node {

// update all the inodes's offset according to the new 'meta_addr'.
self.offset = self.offset - orig_meta_addr + meta_addr;

// Only dir and symlink file can be of EROFS_INODE_FLAT_INLINE layout,
// so the `dirents_offset` is only used for these two types.
// For other types, the `dirents_offset` will always be 0,
Expand Down

0 comments on commit 65ebb7f

Please sign in to comment.