diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 05893b78db9..2de88ad6ed7 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -47,38 +47,6 @@ jobs: name: nydusify-artifact path: contrib/nydusify/cmd - contrib-build-master: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: master - - name: Setup Golang - uses: actions/setup-go@v3 - with: - go-version: ~1.20 - - name: Golang Cache - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-golang- - - name: Build Contrib - run: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.51.2 - make -e DOCKER=false nydusify-release - make -e DOCKER=false contrib-test - - name: Upload Nydusify - uses: actions/upload-artifact@master - with: - name: nydusify-artifact-master - path: contrib/nydusify/cmd - nydus-build: runs-on: ubuntu-latest steps: @@ -101,31 +69,6 @@ jobs: target/release/nydus-image target/release/nydusd - nydus-build-master: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: master - - name: Rust Cache - uses: Swatinem/rust-cache@v2.2.0 - with: - cache-on-failure: true - shared-key: nydus-build - - name: Build Nydus - run: | - rustup component add rustfmt clippy - make - - name: Upload Nydus Binaries - uses: actions/upload-artifact@master - with: - name: nydus-artifact-master - path: | - target/release/nydus-image - target/release/nydusd - nydusd-build-macos: runs-on: macos-11 strategy: @@ -217,414 +160,6 @@ jobs: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.51.2 sudo -E make smoke-only - benchmark-oci: - runs-on: ubuntu-latest - needs: [contrib-build, nydus-build] - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus - uses: actions/download-artifact@master - with: - name: nydus-artifact - path: target/release - - name: Download Nydusify - uses: actions/download-artifact@master - with: - name: nydusify-artifact - path: contrib/nydusify/cmd - - name: Prepare OCI Environment - run: | - sudo bash misc/benchmark/prepare_env.sh oci - sudo docker pull ${{env.IMAGE}}:${{env.TAG}} && docker tag ${{env.IMAGE}}:${{env.TAG}} localhost:5000/${{env.IMAGE}}:${{env.TAG}} - sudo docker push localhost:5000/${{env.IMAGE}}:${{env.TAG}} - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode oci --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-oci - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-nydus-no-prefetch: - runs-on: ubuntu-latest - needs: [contrib-build, nydus-build] - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus - uses: actions/download-artifact@master - with: - name: nydus-artifact - path: target/release - - name: Download Nydusify - uses: actions/download-artifact@master - with: - name: nydusify-artifact - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source ${{env.IMAGE}}:${{env.TAG}} \ - --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ - --fs-version 6 - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-no-prefetch --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-nydus-no-prefetch - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-nydus-no-prefetch-master: - runs-on: ubuntu-latest - needs: [contrib-build-master, nydus-build-master] - if: github.event_name == 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus Master - uses: actions/download-artifact@master - with: - name: nydus-artifact-master - path: target/release - - name: Download Nydusify Master - uses: actions/download-artifact@master - with: - name: nydusify-artifact-master - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source ${{env.IMAGE}}:${{env.TAG}} \ - --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ - --fs-version 6 - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-no-prefetch --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-nydus-no-prefetch-master - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-zran-no-prefetch: - runs-on: ubuntu-latest - needs: [contrib-build, nydus-build] - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus - uses: actions/download-artifact@master - with: - name: nydus-artifact - path: target/release - - name: Download Nydusify - uses: actions/download-artifact@master - with: - name: nydusify-artifact - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo docker pull ${{env.IMAGE}}:${{env.TAG}} && docker tag ${{env.IMAGE}}:${{env.TAG}} localhost:5000/${{env.IMAGE}}:${{env.TAG}} - sudo docker push localhost:5000/${{env.IMAGE}}:${{env.TAG}} - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source localhost:5000/${{env.IMAGE}}:${{env.TAG}} \ - --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ - --fs-version 6 \ - --oci-ref - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-no-prefetch --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-zran-no-prefetch - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-zran-no-prefetch-master: - runs-on: ubuntu-latest - needs: [contrib-build-master, nydus-build-master] - if: github.event_name == 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus Master - uses: actions/download-artifact@master - with: - name: nydus-artifact-master - path: target/release - - name: Download Nydusify Master - uses: actions/download-artifact@master - with: - name: nydusify-artifact-master - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo docker pull ${{env.IMAGE}}:${{env.TAG}} && docker tag ${{env.IMAGE}}:${{env.TAG}} localhost:5000/${{env.IMAGE}}:${{env.TAG}} - sudo docker push localhost:5000/${{env.IMAGE}}:${{env.TAG}} - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source localhost:5000/${{env.IMAGE}}:${{env.TAG}} \ - --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ - --fs-version 6 \ - --oci-ref - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-no-prefetch --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-zran-no-prefetch-master - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-nydus-all-prefetch: - runs-on: ubuntu-latest - needs: [contrib-build, nydus-build] - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus - uses: actions/download-artifact@master - with: - name: nydus-artifact - path: target/release - - name: Download Nydusify - uses: actions/download-artifact@master - with: - name: nydusify-artifact - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source ${{env.IMAGE}}:${{env.TAG}} \ - --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ - --fs-version 6 - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-all-prefetch --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-nydus-all-prefetch - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-nydus-all-prefetch-master: - runs-on: ubuntu-latest - needs: [contrib-build-master, nydus-build-master] - if: github.event_name == 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus Master - uses: actions/download-artifact@master - with: - name: nydus-artifact-master - path: target/release - - name: Download Nydusify Master - uses: actions/download-artifact@master - with: - name: nydusify-artifact-master - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source ${{env.IMAGE}}:${{env.TAG}} \ - --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ - --fs-version 6 - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-all-prefetch --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-nydus-all-prefetch-master - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-zran-all-prefetch: - runs-on: ubuntu-latest - needs: [contrib-build, nydus-build] - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus - uses: actions/download-artifact@master - with: - name: nydus-artifact - path: target/release - - name: Download Nydusify - uses: actions/download-artifact@master - with: - name: nydusify-artifact - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo docker pull ${{env.IMAGE}}:${{env.TAG}} && docker tag ${{env.IMAGE}}:${{env.TAG}} localhost:5000/${{env.IMAGE}}:${{env.TAG}} - sudo docker push localhost:5000/${{env.IMAGE}}:${{env.TAG}} - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source localhost:5000/${{env.IMAGE}}:${{env.TAG}} \ - --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ - --fs-version 6 \ - --oci-ref - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-all-prefetch --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-zran-all-prefetch - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-zran-all-prefetch-master: - runs-on: ubuntu-latest - needs: [contrib-build-master, nydus-build-master] - if: github.event_name == 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus Master - uses: actions/download-artifact@master - with: - name: nydus-artifact-master - path: target/release - - name: Download Nydusify Master - uses: actions/download-artifact@master - with: - name: nydusify-artifact-master - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo docker pull ${{env.IMAGE}}:${{env.TAG}} && docker tag ${{env.IMAGE}}:${{env.TAG}} localhost:5000/${{env.IMAGE}}:${{env.TAG}} - sudo docker push localhost:5000/${{env.IMAGE}}:${{env.TAG}} - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source localhost:5000/${{env.IMAGE}}:${{env.TAG}} \ - --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ - --fs-version 6 \ - --oci-ref - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-all-prefetch --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-zran-all-prefetch-master - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-nydus-filelist-prefetch: - runs-on: ubuntu-latest - needs: [contrib-build, nydus-build] - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus - uses: actions/download-artifact@master - with: - name: nydus-artifact - path: target/release - - name: Download Nydusify - uses: actions/download-artifact@master - with: - name: nydusify-artifact - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source ${{env.IMAGE}}:${{env.TAG}} \ - --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ - --fs-version 6 - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-filelist-prefetch --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-nydus-filelist-prefetch - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-nydus-filelist-prefetch-master: - runs-on: ubuntu-latest - needs: [contrib-build-master, nydus-build-master] - if: github.event_name == 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus Master - uses: actions/download-artifact@master - with: - name: nydus-artifact-master - path: target/release - - name: Download Nydusify Master - uses: actions/download-artifact@master - with: - name: nydusify-artifact-master - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source ${{env.IMAGE}}:${{env.TAG}} \ - --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ - --fs-version 6 - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-filelist-prefetch --image ${{env.IMAGE}}:${{env.TAG}} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-nydus-filelist-prefetch-master - path: misc/benchmark/${{env.IMAGE}}.csv - - benchmark-result: - runs-on: ubuntu-latest - needs: [benchmark-oci, benchmark-zran-all-prefetch, benchmark-zran-no-prefetch, benchmark-nydus-no-prefetch, benchmark-nydus-all-prefetch, benchmark-nydus-filelist-prefetch] - if: github.event_name != 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - - uses: geekyeggo/delete-artifact@v2 - with: - name: '*' - - name: Save Result - run: | - sudo python3 misc/benchmark/benchmark_summary.py --mode benchmark-result > $GITHUB_STEP_SUMMARY - - benchmark-compare: - runs-on: ubuntu-latest - needs: [benchmark-oci, benchmark-zran-all-prefetch, benchmark-zran-no-prefetch, benchmark-nydus-no-prefetch, benchmark-nydus-all-prefetch, benchmark-nydus-filelist-prefetch, benchmark-zran-all-prefetch-master, benchmark-zran-no-prefetch-master, benchmark-nydus-no-prefetch-master, benchmark-nydus-all-prefetch-master, benchmark-nydus-filelist-prefetch-master] - if: github.event_name == 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - - uses: geekyeggo/delete-artifact@v2 - with: - name: '*' - - name: Save Result - run: | - sudo python3 misc/benchmark/benchmark_summary.py --mode benchmark-compare > $GITHUB_STEP_SUMMARY - nydus-unit-test: runs-on: ubuntu-latest steps: @@ -677,3 +212,33 @@ jobs: steps: - uses: actions/checkout@v3 - uses: EmbarkStudios/cargo-deny-action@v1 + + performance-test: + runs-on: ubuntu-latest + needs: [contrib-build, nydus-build] + strategy: + matrix: + include: + - mode: fs-version-5 + - mode: fs-version-6 + - mode: zran + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Download Nydus + uses: actions/download-artifact@master + with: + name: nydus-artifact + path: target/release + - name: Download Nydusify + uses: actions/download-artifact@master + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + - name: Prepare Nydus Container Environment + run: | + sudo bash misc/performance/prepare.sh + - name: Performance Test + run: | + export PERFORMANCE_TEST_MODE=${{ matrix.mode }} + sudo -E make smoke-performance diff --git a/Makefile b/Makefile index f026280d7ac..7b212981dcf 100644 --- a/Makefile +++ b/Makefile @@ -133,6 +133,9 @@ coverage-codecov: smoke-only: make -C smoke test +smoke-performance: + make -C smoke test-performance + smoke: release smoke-only docker-nydus-smoke: diff --git a/misc/performance/containerd_config.toml b/misc/performance/containerd_config.toml new file mode 100644 index 00000000000..e2bc6705352 --- /dev/null +++ b/misc/performance/containerd_config.toml @@ -0,0 +1,17 @@ +version = 2 +root = "/var/lib/containerd" +state = "/run/containerd" +oom_score = 0 + +[debug] + level = "debug" + +[plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "nydus" + disable_snapshot_annotations = false + +[proxy_plugins] + [proxy_plugins.nydus] + type = "snapshot" + address = "/run/containerd-nydus/containerd-nydus-grpc.sock" diff --git a/misc/performance/nydus-snapshotter.service b/misc/performance/nydus-snapshotter.service new file mode 100644 index 00000000000..3abdb555986 --- /dev/null +++ b/misc/performance/nydus-snapshotter.service @@ -0,0 +1,18 @@ +[Unit] +Description=nydus snapshotter +After=network.target +Before=containerd.service + +[Service] +Type=simple +Environment=HOME=/root +ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus/config.toml +Restart=always +RestartSec=1 +KillMode=process +OOMScoreAdjust=-999 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/misc/performance/nydusd_config.json b/misc/performance/nydusd_config.json new file mode 100644 index 00000000000..e3691c38400 --- /dev/null +++ b/misc/performance/nydusd_config.json @@ -0,0 +1,26 @@ +{ + "device": { + "backend": { + "type": "registry", + "config": { + "scheme": "http", + "host": "localhost:5000" + } + }, + "cache": { + "type": "blobcache", + "config": { + "work_dir": "/var/lib/containerd-nydus/cache" + } + } + }, + "mode": "direct", + "digest_validate": false, + "enable_xattr": true, + "iostats_files": false, + "access_pattern": false, + "latest_read_files": false, + "fs_prefetch": { + "enable": false + } +} diff --git a/misc/performance/prepare.sh b/misc/performance/prepare.sh new file mode 100644 index 00000000000..8ea57518af8 --- /dev/null +++ b/misc/performance/prepare.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +readonly SNAPSHOTTER_VERSION=0.13.3 +readonly NERDCTL_VERSION=1.7.0 +readonly CNI_PLUGINS_VERSION=1.3.0 + +# setup nerdctl and nydusd env +sudo install -D -m 755 contrib/nydusify/cmd/nydusify /usr/local/bin +sudo install -D -m 755 target/release/nydusd target/release/nydus-image /usr/local/bin +wget https://github.com/containerd/nydus-snapshotter/releases/download/v$SNAPSHOTTER_VERSION/nydus-snapshotter-v$SNAPSHOTTER_VERSION-x86_64.tgz +tar zxvf nydus-snapshotter-v$SNAPSHOTTER_VERSION-x86_64.tgz +sudo install -D -m 755 nydus-snapshotter/containerd-nydus-grpc /usr/local/bin +sudo wget https://github.com/containerd/nerdctl/releases/download/v$NERDCTL_VERSION/nerdctl-$NERDCTL_VERSION-linux-amd64.tar.gz +sudo tar -xzvf nerdctl-$NERDCTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin +sudo mkdir -p /opt/cni/bin +sudo wget https://github.com/containernetworking/plugins/releases/download/v$CNI_PLUGINS_VERSION/cni-plugins-linux-amd64-v$CNI_PLUGINS_VERSION.tgz +sudo tar -xzvf cni-plugins-linux-amd64-v$CNI_PLUGINS_VERSION.tgz -C /opt/cni/bin +sudo install -D misc/performance/containerd_config.toml /etc/containerd/config.toml +sudo systemctl restart containerd +sudo install -D misc/performance/nydusd_config.json /etc/nydus/nydusd-config.fusedev.json +sudo install -D misc/performance/snapshotter_config.toml /etc/nydus/config.toml +sudo install -D misc/performance/nydus-snapshotter.service /etc/systemd/system/nydus-snapshotter.service +sudo systemctl start nydus-snapshotter diff --git a/misc/performance/snapshotter_config.toml b/misc/performance/snapshotter_config.toml new file mode 100644 index 00000000000..e8d3c118d9c --- /dev/null +++ b/misc/performance/snapshotter_config.toml @@ -0,0 +1,132 @@ +version = 1 +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" +daemon_mode = "dedicated" +# Whether snapshotter should try to clean up resources when it is closed +cleanup_on_close = false + +[system] +# Snapshotter's debug and trace HTTP server interface +enable = true +# Unix domain socket path where system controller is listening on +address = "/run/containerd-nydus/system.sock" + +[system.debug] +# Snapshotter can profile the CPU utilization of each nydusd daemon when it is being started. +# This option specifies the profile duration when nydusd is downloading and uncomproessing data. +daemon_cpu_profile_duration_secs = 5 +# Enable by assigning an address, empty indicates pprof server is disabled +pprof_address = "" + +[daemon] +# Specify a configuration file for nydusd +nydusd_config = "/etc/nydus/nydusd-config.fusedev.json" +nydusd_path = "/usr/local/bin/nydusd" +nydusimage_path = "/usr/local/bin/nydus-image" +# fusedev or fscache +fs_driver = "fusedev" +# How to process when daemon dies: "none", "restart" or "failover" +recover_policy = "restart" +# Nydusd worker thread number to handle FUSE or fscache requests, [0-1024]. +# Setting to 0 will use the default configuration of nydusd. +threads_number = 4 +# Log rotation size for nydusd, in unit MB(megabytes) +log_rotation_size = 100 + +[cgroup] +# Whether to use separate cgroup for nydusd. +enable = true +# The memory limit for nydusd cgroup, which contains all nydusd processes. +# Percentage is supported as well, please ensure it is end with "%". +# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%". +memory_limit = "" + +[log] +# Print logs to stdout rather than logging files +log_to_stdout = false +# Snapshotter's log level +level = "info" +log_rotation_compress = true +log_rotation_local_time = true +# Max number of days to retain logs +log_rotation_max_age = 7 +log_rotation_max_backups = 5 +# In unit MB(megabytes) +log_rotation_max_size = 100 + +[metrics] +# Enable by assigning an address, empty indicates metrics server is disabled +address = ":9110" + +[remote] +convert_vpc_registry = false + +[remote.mirrors_config] +# Snapshotter will overwrite daemon's mirrors configuration +# if the values loaded from this driectory are not null before starting a daemon. +# Set to "" or an empty directory to disable it. +#dir = "/etc/nydus/certs.d" + +[remote.auth] +# Fetch the private registry auth by listening to K8s API server +enable_kubeconfig_keychain = false +# synchronize `kubernetes.io/dockerconfigjson` secret from kubernetes API server with specified kubeconfig (default `$KUBECONFIG` or `~/.kube/config`) +kubeconfig_path = "" +# Fetch the private registry auth as CRI image service proxy +enable_cri_keychain = false +# the target image service when using image proxy +#image_service_address = "/run/containerd/containerd.sock" + +[snapshot] +# Let containerd use nydus-overlayfs mount helper +enable_nydus_overlayfs = false +# Insert Kata Virtual Volume option to `Mount.Options` +enable_kata_volume = false +# Whether to remove resources when a snapshot is removed +sync_remove = false + +[cache_manager] +# Disable or enable recyclebin +disable = false +# How long to keep deleted files in recyclebin +gc_period = "24h" +# Directory to host cached files +cache_dir = "" + +[image] +public_key_file = "" +validate_signature = false + +# The configuraions for features that are not production ready +[experimental] +# Whether to enable stargz support +enable_stargz = false +# Whether to enable referrers support +# The option enables trying to fetch the Nydus image associated with the OCI image and run it. +# Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers +enable_referrer_detect = false +# Whether to enable authentication support +# The option enables nydus snapshot to provide backend information to nydusd. +enable_backend_source = false +[experimental.tarfs] +# Whether to enable nydus tarfs mode. Tarfs is supported by: +# - The EROFS filesystem driver since Linux 6.4 +# - Nydus Image Service release v2.3 +enable_tarfs = false +# Mount rafs on host by loopdev and EROFS +mount_tarfs_on_host = false +# Only enable nydus tarfs mode for images with `tarfs hint` label when true +tarfs_hint = false +# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default +max_concurrent_proc = 0 +# Mode to export tarfs images: +# - "none" or "": do not export tarfs +# - "layer_verity_only": only generate disk verity information for a layer blob +# - "image_verity_only": only generate disk verity information for all blobs of an image +# - "layer_block": generate a raw block disk image with tarfs for a layer +# - "image_block": generate a raw block disk image with tarfs for an image +# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info +# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info +export_mode = "" diff --git a/smoke/Makefile b/smoke/Makefile index f8b12f531c7..b38be0314b1 100644 --- a/smoke/Makefile +++ b/smoke/Makefile @@ -17,6 +17,12 @@ build: test: build sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=16 -test.run=$(TESTS) +# PERFORMANCE_TEST_MODE=fs-version-5 \ +# PERFORMANCE_TEST_IMAGE=wordpress:latest \ +# make test-performance +test-performance: build + PERFORMANCE_TEST=True;sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=1 -test.run=TestPerformance + # WORK_DIR=/tmp \ # NYDUS_STABLE_VERSION=v2.2.3 \ # NYDUS_STABLE_VERSION_EXPORT=v2_2_3 \ diff --git a/smoke/tests/performance_test.go b/smoke/tests/performance_test.go new file mode 100644 index 00000000000..de84b5e88f8 --- /dev/null +++ b/smoke/tests/performance_test.go @@ -0,0 +1,101 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "fmt" + "os" + "testing" + + "github.com/dragonflyoss/image-service/smoke/tests/tool" + "github.com/dragonflyoss/image-service/smoke/tests/tool/test" + "github.com/google/uuid" +) + +// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshoooter, nydusd, nydus-image and nydusify. +// Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md. + +type PerformanceTestSuite struct { + t *testing.T + testImage string + testContainerName string +} + +func (p *PerformanceTestSuite) TestPerformance(t *testing.T) { + ctx := tool.DefaultContext(p.t) + // choose test mode + mode := os.Getenv("PERFORMANCE_TEST_MODE") + if mode == "" { + mode = "fs-version-6" + } + switch mode { + case "fs-version-5": + ctx.Build.FSVersion = "5" + case "fs-version-6": + ctx.Build.FSVersion = "6" + case "zran": + ctx.Build.OCIRef = true + default: + p.t.Fatalf("PerformanceTest don't support %s mode", mode) + } + // choose test image + image := os.Getenv("PERFORMANCE_TEST_IMAGE") + + if image == "" { + image = "wordpress:6.1.1" + } else { + if !tool.SupportContainerImage(tool.ImageRepo(p.t, image)) { + p.t.Fatalf("Unsupport image " + image) + } + } + // prepare test image + p.prepareTestImage(p.t, ctx, mode, image) + + // run Contaienr + p.testContainerName = uuid.NewString() + tool.RunContainer(p.t, p.testImage, p.testContainerName, mode) + clearContainer(p.t, p.testImage, p.testContainerName) +} + +func (p *PerformanceTestSuite) prepareTestImage(t *testing.T, ctx *tool.Context, mode string, image string) { + if p.testImage != "" { + return + } + + ctx.PrepareWorkDir(t) + defer ctx.Destroy(t) + source := tool.PrepareImage(t, image) + + // Prepare options + target := fmt.Sprintf("%s-nydus-%s", source, uuid.NewString()) + fsVersion := fmt.Sprintf("--fs-version %s", ctx.Build.FSVersion) + logLevel := "--log-level warn" + if ctx.Binary.NydusifyOnlySupportV5 { + fsVersion = "" + logLevel = "" + } + enableOCIRef := "" + if ctx.Build.OCIRef { + enableOCIRef = "--oci-ref" + } + + // Convert image + convertCmd := fmt.Sprintf("%s %s convert --source %s --target %s --nydus-image %s --work-dir %s %s %s", + ctx.Binary.Nydusify, logLevel, source, target, ctx.Binary.Builder, ctx.Env.WorkDir, fsVersion, enableOCIRef) + tool.RunWithoutOutput(t, convertCmd) + p.testImage = target +} + +func clearContainer(t *testing.T, image string, containerName string) { + tool.RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter nydus rm -f %s", containerName)) + tool.RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter nydus image rm %s", image)) +} + +func TestPerformance(t *testing.T) { + if os.Getenv("PERFORMANCE_TEST") == "" { + t.Skip("skipping performance test") + } + test.Run(t, &PerformanceTestSuite{t: t}) +} diff --git a/smoke/tests/tool/container.go b/smoke/tests/tool/container.go new file mode 100644 index 00000000000..ee6af8a8d76 --- /dev/null +++ b/smoke/tests/tool/container.go @@ -0,0 +1,154 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tool + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type ContainerMetrics struct { + ReadCount uint64 `json:"read_count"` + ReadAmountTotal uint64 `json:"read_amount_total"` +} + +type RunArgs struct { + WaitURL string + BaselineReadCount map[string]uint64 + BaselineReadAmount map[string]uint64 +} + +var URL_WAIT = map[string]RunArgs{ + "wordpress": { + WaitURL: "http://localhost:80", + BaselineReadCount: map[string]uint64{ + "fs-version-5": 328, + "fs-version-6": 131, + "zran": 186, + }, + BaselineReadAmount: map[string]uint64{ + "fs-version-5": 54307819, + "fs-version-6": 77580818, + "zran": 79836339, + }, + }, +} + +var supportContainerImages = []string{"wordpress"} + +// SupportContainerImage help to check if we support the image or not +func SupportContainerImage(image string) bool { + return contains(supportContainerImages, image) +} + +func contains(slice []string, value string) bool { + for _, v := range slice { + if strings.Contains(v, value) { + return true + } + } + return false +} + +// runUrlWaitContainer run Contaienr util geting http response from WaitUrl +func runUrlWaitContainer(t *testing.T, image string, containerName string, runArgs RunArgs) { + cmd := fmt.Sprintf("sudo nerdctl --insecure-registry --snapshotter nydus run -d --net=host --name=%s %s", containerName, image) + RunWithoutOutput(t, cmd) + for { + resp, err := http.Get(runArgs.WaitURL) + if err == nil { + resp.Body.Close() + break + } + time.Sleep(100 * time.Millisecond) + } +} + +// RunContainer and get metrics from api socket. +// Test will fail if performance below baseline. +func RunContainer(t *testing.T, image string, containerName string, mode string) { + args, ok := URL_WAIT[ImageRepo(t, image)] + if ok { + runUrlWaitContainer(t, image, containerName, args) + } + containerMetrics, err := getContainerMetrics(t) + if err != nil { + t.Logf(err.Error()) + } + if containerMetrics.ReadAmountTotal > uint64(float64(args.BaselineReadAmount[mode])*1.05) || + containerMetrics.ReadCount > uint64(float64(args.BaselineReadCount[mode])*1.05) { + t.Fatalf(fmt.Sprintf("Performance reduction with ReadAmount %d and ReadCount %d", containerMetrics.ReadAmountTotal, containerMetrics.ReadCount)) + } + t.Logf(fmt.Sprintf("Performance Test: ReadAmount %d and ReadCount %d", containerMetrics.ReadAmountTotal, containerMetrics.ReadCount)) +} + +// getContainerMetrics get metrics by nydus api sock +func getContainerMetrics(t *testing.T) (*ContainerMetrics, error) { + transport := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + return dialer.DialContext(ctx, "unix", searchAPISockPath(t)) + }, + } + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + } + + resp, err := client.Get("http://unix/api/v1/metrics/backend") + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var info ContainerMetrics + if err = json.Unmarshal(body, &info); err != nil { + return nil, err + } + + return &info, nil +} + +// searchAPISockPath search sock filepath in nydusd work dir, default in "/var/lib/containerd-nydus/socket" +func searchAPISockPath(t *testing.T) string { + var apiSockPath string + + err := filepath.Walk("/var/lib/containerd-nydus/socket", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() && info.Name() != "socket" { + apiSockPath = path + return filepath.SkipDir + } + return nil + }) + require.NoError(t, err) + + return apiSockPath + "/api.sock" +} diff --git a/smoke/tests/tool/util.go b/smoke/tests/tool/util.go index fc3358d7dc1..427f7fa4984 100644 --- a/smoke/tests/tool/util.go +++ b/smoke/tests/tool/util.go @@ -9,6 +9,7 @@ import ( "io" "os" "os/exec" + "regexp" "strings" "testing" @@ -71,3 +72,16 @@ func GetBinary(t *testing.T, env, version string) string { } return binary } + +func ImageRepo(t *testing.T, image string) string { + if strings.Contains(image, "/") { + re := regexp.MustCompile(`^.*/`) + image = re.ReplaceAllString(image, "") + } + parts := strings.Split(image, ":") + if len(parts) > 0 { + return parts[0] + } + t.Fatalf("Can't get image repo of " + image) + return "" +}