From e714b16652619980ad55ddec05346a1ca2807018 Mon Sep 17 00:00:00 2001 From: Yadong Ding Date: Wed, 22 Nov 2023 18:13:56 +0800 Subject: [PATCH] smoke: add performance test Add performance test to make sure there don't have performance descend Signed-off-by: Yadong Ding --- Makefile | 3 + misc/performance/containerd_config.toml | 17 +++ misc/performance/nydus-snapshotter.service | 18 +++ misc/performance/nydusd_config.json | 26 ++++ misc/performance/prepare.sh | 23 +++ misc/performance/snapshotter_config.toml | 132 ++++++++++++++++++ smoke/Makefile | 6 + smoke/tests/performance_test.go | 102 ++++++++++++++ smoke/tests/tool/container.go | 154 +++++++++++++++++++++ smoke/tests/tool/util.go | 14 ++ 10 files changed, 495 insertions(+) create mode 100644 misc/performance/containerd_config.toml create mode 100644 misc/performance/nydus-snapshotter.service create mode 100644 misc/performance/nydusd_config.json create mode 100644 misc/performance/prepare.sh create mode 100644 misc/performance/snapshotter_config.toml create mode 100644 smoke/tests/performance_test.go create mode 100644 smoke/tests/tool/container.go diff --git a/Makefile b/Makefile index f026280d7ac..7b212981dcf 100644 --- a/Makefile +++ b/Makefile @@ -133,6 +133,9 @@ coverage-codecov: smoke-only: make -C smoke test +smoke-performance: + make -C smoke test-performance + smoke: release smoke-only docker-nydus-smoke: diff --git a/misc/performance/containerd_config.toml b/misc/performance/containerd_config.toml new file mode 100644 index 00000000000..e2bc6705352 --- /dev/null +++ b/misc/performance/containerd_config.toml @@ -0,0 +1,17 @@ +version = 2 +root = "/var/lib/containerd" +state = "/run/containerd" +oom_score = 0 + +[debug] + level = "debug" + +[plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "nydus" + disable_snapshot_annotations = false + +[proxy_plugins] + [proxy_plugins.nydus] + type = "snapshot" + address = "/run/containerd-nydus/containerd-nydus-grpc.sock" diff --git a/misc/performance/nydus-snapshotter.service b/misc/performance/nydus-snapshotter.service new file mode 100644 index 00000000000..3abdb555986 --- /dev/null +++ b/misc/performance/nydus-snapshotter.service @@ -0,0 +1,18 @@ +[Unit] +Description=nydus snapshotter +After=network.target +Before=containerd.service + +[Service] +Type=simple +Environment=HOME=/root +ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus/config.toml +Restart=always +RestartSec=1 +KillMode=process +OOMScoreAdjust=-999 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/misc/performance/nydusd_config.json b/misc/performance/nydusd_config.json new file mode 100644 index 00000000000..e3691c38400 --- /dev/null +++ b/misc/performance/nydusd_config.json @@ -0,0 +1,26 @@ +{ + "device": { + "backend": { + "type": "registry", + "config": { + "scheme": "http", + "host": "localhost:5000" + } + }, + "cache": { + "type": "blobcache", + "config": { + "work_dir": "/var/lib/containerd-nydus/cache" + } + } + }, + "mode": "direct", + "digest_validate": false, + "enable_xattr": true, + "iostats_files": false, + "access_pattern": false, + "latest_read_files": false, + "fs_prefetch": { + "enable": false + } +} diff --git a/misc/performance/prepare.sh b/misc/performance/prepare.sh new file mode 100644 index 00000000000..8ea57518af8 --- /dev/null +++ b/misc/performance/prepare.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +readonly SNAPSHOTTER_VERSION=0.13.3 +readonly NERDCTL_VERSION=1.7.0 +readonly CNI_PLUGINS_VERSION=1.3.0 + +# setup nerdctl and nydusd env +sudo install -D -m 755 contrib/nydusify/cmd/nydusify /usr/local/bin +sudo install -D -m 755 target/release/nydusd target/release/nydus-image /usr/local/bin +wget https://github.com/containerd/nydus-snapshotter/releases/download/v$SNAPSHOTTER_VERSION/nydus-snapshotter-v$SNAPSHOTTER_VERSION-x86_64.tgz +tar zxvf nydus-snapshotter-v$SNAPSHOTTER_VERSION-x86_64.tgz +sudo install -D -m 755 nydus-snapshotter/containerd-nydus-grpc /usr/local/bin +sudo wget https://github.com/containerd/nerdctl/releases/download/v$NERDCTL_VERSION/nerdctl-$NERDCTL_VERSION-linux-amd64.tar.gz +sudo tar -xzvf nerdctl-$NERDCTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin +sudo mkdir -p /opt/cni/bin +sudo wget https://github.com/containernetworking/plugins/releases/download/v$CNI_PLUGINS_VERSION/cni-plugins-linux-amd64-v$CNI_PLUGINS_VERSION.tgz +sudo tar -xzvf cni-plugins-linux-amd64-v$CNI_PLUGINS_VERSION.tgz -C /opt/cni/bin +sudo install -D misc/performance/containerd_config.toml /etc/containerd/config.toml +sudo systemctl restart containerd +sudo install -D misc/performance/nydusd_config.json /etc/nydus/nydusd-config.fusedev.json +sudo install -D misc/performance/snapshotter_config.toml /etc/nydus/config.toml +sudo install -D misc/performance/nydus-snapshotter.service /etc/systemd/system/nydus-snapshotter.service +sudo systemctl start nydus-snapshotter diff --git a/misc/performance/snapshotter_config.toml b/misc/performance/snapshotter_config.toml new file mode 100644 index 00000000000..e8d3c118d9c --- /dev/null +++ b/misc/performance/snapshotter_config.toml @@ -0,0 +1,132 @@ +version = 1 +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" +daemon_mode = "dedicated" +# Whether snapshotter should try to clean up resources when it is closed +cleanup_on_close = false + +[system] +# Snapshotter's debug and trace HTTP server interface +enable = true +# Unix domain socket path where system controller is listening on +address = "/run/containerd-nydus/system.sock" + +[system.debug] +# Snapshotter can profile the CPU utilization of each nydusd daemon when it is being started. +# This option specifies the profile duration when nydusd is downloading and uncomproessing data. +daemon_cpu_profile_duration_secs = 5 +# Enable by assigning an address, empty indicates pprof server is disabled +pprof_address = "" + +[daemon] +# Specify a configuration file for nydusd +nydusd_config = "/etc/nydus/nydusd-config.fusedev.json" +nydusd_path = "/usr/local/bin/nydusd" +nydusimage_path = "/usr/local/bin/nydus-image" +# fusedev or fscache +fs_driver = "fusedev" +# How to process when daemon dies: "none", "restart" or "failover" +recover_policy = "restart" +# Nydusd worker thread number to handle FUSE or fscache requests, [0-1024]. +# Setting to 0 will use the default configuration of nydusd. +threads_number = 4 +# Log rotation size for nydusd, in unit MB(megabytes) +log_rotation_size = 100 + +[cgroup] +# Whether to use separate cgroup for nydusd. +enable = true +# The memory limit for nydusd cgroup, which contains all nydusd processes. +# Percentage is supported as well, please ensure it is end with "%". +# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%". +memory_limit = "" + +[log] +# Print logs to stdout rather than logging files +log_to_stdout = false +# Snapshotter's log level +level = "info" +log_rotation_compress = true +log_rotation_local_time = true +# Max number of days to retain logs +log_rotation_max_age = 7 +log_rotation_max_backups = 5 +# In unit MB(megabytes) +log_rotation_max_size = 100 + +[metrics] +# Enable by assigning an address, empty indicates metrics server is disabled +address = ":9110" + +[remote] +convert_vpc_registry = false + +[remote.mirrors_config] +# Snapshotter will overwrite daemon's mirrors configuration +# if the values loaded from this driectory are not null before starting a daemon. +# Set to "" or an empty directory to disable it. +#dir = "/etc/nydus/certs.d" + +[remote.auth] +# Fetch the private registry auth by listening to K8s API server +enable_kubeconfig_keychain = false +# synchronize `kubernetes.io/dockerconfigjson` secret from kubernetes API server with specified kubeconfig (default `$KUBECONFIG` or `~/.kube/config`) +kubeconfig_path = "" +# Fetch the private registry auth as CRI image service proxy +enable_cri_keychain = false +# the target image service when using image proxy +#image_service_address = "/run/containerd/containerd.sock" + +[snapshot] +# Let containerd use nydus-overlayfs mount helper +enable_nydus_overlayfs = false +# Insert Kata Virtual Volume option to `Mount.Options` +enable_kata_volume = false +# Whether to remove resources when a snapshot is removed +sync_remove = false + +[cache_manager] +# Disable or enable recyclebin +disable = false +# How long to keep deleted files in recyclebin +gc_period = "24h" +# Directory to host cached files +cache_dir = "" + +[image] +public_key_file = "" +validate_signature = false + +# The configuraions for features that are not production ready +[experimental] +# Whether to enable stargz support +enable_stargz = false +# Whether to enable referrers support +# The option enables trying to fetch the Nydus image associated with the OCI image and run it. +# Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers +enable_referrer_detect = false +# Whether to enable authentication support +# The option enables nydus snapshot to provide backend information to nydusd. +enable_backend_source = false +[experimental.tarfs] +# Whether to enable nydus tarfs mode. Tarfs is supported by: +# - The EROFS filesystem driver since Linux 6.4 +# - Nydus Image Service release v2.3 +enable_tarfs = false +# Mount rafs on host by loopdev and EROFS +mount_tarfs_on_host = false +# Only enable nydus tarfs mode for images with `tarfs hint` label when true +tarfs_hint = false +# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default +max_concurrent_proc = 0 +# Mode to export tarfs images: +# - "none" or "": do not export tarfs +# - "layer_verity_only": only generate disk verity information for a layer blob +# - "image_verity_only": only generate disk verity information for all blobs of an image +# - "layer_block": generate a raw block disk image with tarfs for a layer +# - "image_block": generate a raw block disk image with tarfs for an image +# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info +# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info +export_mode = "" diff --git a/smoke/Makefile b/smoke/Makefile index f8b12f531c7..15af45d8e25 100644 --- a/smoke/Makefile +++ b/smoke/Makefile @@ -17,6 +17,12 @@ build: test: build sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=16 -test.run=$(TESTS) +# PERFORMANCE_TEST_MODE=nydus_v5 \ +# PERFORMANCE_TEST_IMAGE=wordpress:latest \ +# make test-performance +test-performance: build + PERFORMANCE_TEST=True;sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=1 -test.run=TestPerformance + # WORK_DIR=/tmp \ # NYDUS_STABLE_VERSION=v2.2.3 \ # NYDUS_STABLE_VERSION_EXPORT=v2_2_3 \ diff --git a/smoke/tests/performance_test.go b/smoke/tests/performance_test.go new file mode 100644 index 00000000000..966138a7be2 --- /dev/null +++ b/smoke/tests/performance_test.go @@ -0,0 +1,102 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "fmt" + "os" + "testing" + + "github.com/dragonflyoss/image-service/smoke/tests/tool" + "github.com/dragonflyoss/image-service/smoke/tests/tool/test" + "github.com/google/uuid" +) + +// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshoooter, nydusd, nydus-image and nydusify. +// Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md. + +type PerformanceTestSuite struct { + t *testing.T + testImage string + testContainerName string +} + +func (p *PerformanceTestSuite) TestPerformance(t *testing.T) { + ctx := tool.DefaultContext(p.t) + // choose test mode + mode := os.Getenv("PERFORMANCE_TEST_MODE") + if mode == "" { + mode = "fs-version-6" + } + switch mode { + case "fs-version-5": + ctx.Build.FSVersion = "5" + case "fs-version-6": + ctx.Build.FSVersion = "6" + case "zran": + ctx.Build.OCIRef = true + default: + p.t.Fatalf("PerformanceTest don't support %s mode", mode) + } + // choose test image + image := os.Getenv("PERFORMANCE_TEST_IMAGE") + + if image == "" { + image = "wordpress:6.1.1" + } else { + if !tool.SupportContainerImage(tool.ImageRepo(p.t, image)) { + p.t.Fatalf("Unsupport image " + image) + } + } + // prepare test image + p.prepareTestImage(p.t, ctx, mode, image) + + // run Contaienr + p.testContainerName = uuid.NewString() + tool.RunContainer(p.t, p.testImage, p.testContainerName, mode) + clearContainer(p.t, p.testImage, p.testContainerName) +} + +func (p *PerformanceTestSuite) prepareTestImage(t *testing.T, ctx *tool.Context, mode string, image string) { + if p.testImage != "" { + return + } + + ctx.PrepareWorkDir(t) + defer ctx.Destroy(t) + source := tool.PrepareImage(t, image) + + // Prepare options + target := fmt.Sprintf("%s-nydus-%s", source, uuid.NewString()) + fsVersion := fmt.Sprintf("--fs-version %s", ctx.Build.FSVersion) + logLevel := "--log-level warn" + if ctx.Binary.NydusifyOnlySupportV5 { + fsVersion = "" + logLevel = "" + } + enableOCIRef := "" + if ctx.Build.OCIRef { + enableOCIRef = "--oci-ref" + } + + // Convert image + convertCmd := fmt.Sprintf("%s %s convert --source %s --target %s --nydus-image %s --work-dir %s %s %s", + ctx.Binary.Nydusify, logLevel, source, target, ctx.Binary.Builder, ctx.Env.WorkDir, fsVersion, enableOCIRef) + tool.RunWithoutOutput(t, convertCmd) + p.testImage = target +} + +func clearContainer(t *testing.T, image string, containerName string) { + tool.RunWithOutput("sudo ls /var/lib/containerd-nydus/socket") + tool.RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter nydus rm -f %s", containerName)) + tool.RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter nydus image rm %s", image)) +} + +func TestPerformance(t *testing.T) { + if os.Getenv("PERFORMANCE_TEST") == "" { + t.Skip("skipping performance test") + } + test.Run(t, &PerformanceTestSuite{t: t}) +} diff --git a/smoke/tests/tool/container.go b/smoke/tests/tool/container.go new file mode 100644 index 00000000000..ee6af8a8d76 --- /dev/null +++ b/smoke/tests/tool/container.go @@ -0,0 +1,154 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tool + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type ContainerMetrics struct { + ReadCount uint64 `json:"read_count"` + ReadAmountTotal uint64 `json:"read_amount_total"` +} + +type RunArgs struct { + WaitURL string + BaselineReadCount map[string]uint64 + BaselineReadAmount map[string]uint64 +} + +var URL_WAIT = map[string]RunArgs{ + "wordpress": { + WaitURL: "http://localhost:80", + BaselineReadCount: map[string]uint64{ + "fs-version-5": 328, + "fs-version-6": 131, + "zran": 186, + }, + BaselineReadAmount: map[string]uint64{ + "fs-version-5": 54307819, + "fs-version-6": 77580818, + "zran": 79836339, + }, + }, +} + +var supportContainerImages = []string{"wordpress"} + +// SupportContainerImage help to check if we support the image or not +func SupportContainerImage(image string) bool { + return contains(supportContainerImages, image) +} + +func contains(slice []string, value string) bool { + for _, v := range slice { + if strings.Contains(v, value) { + return true + } + } + return false +} + +// runUrlWaitContainer run Contaienr util geting http response from WaitUrl +func runUrlWaitContainer(t *testing.T, image string, containerName string, runArgs RunArgs) { + cmd := fmt.Sprintf("sudo nerdctl --insecure-registry --snapshotter nydus run -d --net=host --name=%s %s", containerName, image) + RunWithoutOutput(t, cmd) + for { + resp, err := http.Get(runArgs.WaitURL) + if err == nil { + resp.Body.Close() + break + } + time.Sleep(100 * time.Millisecond) + } +} + +// RunContainer and get metrics from api socket. +// Test will fail if performance below baseline. +func RunContainer(t *testing.T, image string, containerName string, mode string) { + args, ok := URL_WAIT[ImageRepo(t, image)] + if ok { + runUrlWaitContainer(t, image, containerName, args) + } + containerMetrics, err := getContainerMetrics(t) + if err != nil { + t.Logf(err.Error()) + } + if containerMetrics.ReadAmountTotal > uint64(float64(args.BaselineReadAmount[mode])*1.05) || + containerMetrics.ReadCount > uint64(float64(args.BaselineReadCount[mode])*1.05) { + t.Fatalf(fmt.Sprintf("Performance reduction with ReadAmount %d and ReadCount %d", containerMetrics.ReadAmountTotal, containerMetrics.ReadCount)) + } + t.Logf(fmt.Sprintf("Performance Test: ReadAmount %d and ReadCount %d", containerMetrics.ReadAmountTotal, containerMetrics.ReadCount)) +} + +// getContainerMetrics get metrics by nydus api sock +func getContainerMetrics(t *testing.T) (*ContainerMetrics, error) { + transport := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + return dialer.DialContext(ctx, "unix", searchAPISockPath(t)) + }, + } + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + } + + resp, err := client.Get("http://unix/api/v1/metrics/backend") + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var info ContainerMetrics + if err = json.Unmarshal(body, &info); err != nil { + return nil, err + } + + return &info, nil +} + +// searchAPISockPath search sock filepath in nydusd work dir, default in "/var/lib/containerd-nydus/socket" +func searchAPISockPath(t *testing.T) string { + var apiSockPath string + + err := filepath.Walk("/var/lib/containerd-nydus/socket", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() && info.Name() != "socket" { + apiSockPath = path + return filepath.SkipDir + } + return nil + }) + require.NoError(t, err) + + return apiSockPath + "/api.sock" +} diff --git a/smoke/tests/tool/util.go b/smoke/tests/tool/util.go index fc3358d7dc1..427f7fa4984 100644 --- a/smoke/tests/tool/util.go +++ b/smoke/tests/tool/util.go @@ -9,6 +9,7 @@ import ( "io" "os" "os/exec" + "regexp" "strings" "testing" @@ -71,3 +72,16 @@ func GetBinary(t *testing.T, env, version string) string { } return binary } + +func ImageRepo(t *testing.T, image string) string { + if strings.Contains(image, "/") { + re := regexp.MustCompile(`^.*/`) + image = re.ReplaceAllString(image, "") + } + parts := strings.Split(image, ":") + if len(parts) > 0 { + return parts[0] + } + t.Fatalf("Can't get image repo of " + image) + return "" +}