diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cd96b50a75a..787790a55d4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -4,6 +4,7 @@ on: schedule: # Run at 03:00 clock UTC on Monday and Wednesday - cron: "0 03 * * 1,3" + push: pull_request: paths: - '.github/workflows/benchmark.yml' @@ -72,16 +73,6 @@ jobs: include: - image: wordpress tag: 6.1.1 - - image: node - tag: 19.8 - - image: python - tag: 3.10.7 - - image: golang - tag: 1.19.3 - - image: ruby - tag: 3.1.3 - - image: amazoncorretto - tag: 8-al2022-jdk steps: - name: Checkout uses: actions/checkout@v3 @@ -95,20 +86,13 @@ jobs: with: name: nydusify-artifact path: contrib/nydusify/cmd - - name: Prepare OCI Environment + - name: Prepare Environment run: | - sudo bash misc/benchmark/prepare_env.sh oci - sudo docker pull ${{ matrix.image }}:${{ matrix.tag }} && docker tag ${{ matrix.image }}:${{ matrix.tag }} localhost:5000/${{ matrix.image }}:${{ matrix.tag }} - sudo docker push localhost:5000/${{ matrix.image }}:${{ matrix.tag }} + sudo bash misc/performance/prepare.sh - name: BenchMark Test run: | - cd misc/benchmark - sudo python3 benchmark.py --mode oci --image ${{ matrix.image }}:${{ matrix.tag }} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-oci-${{ matrix.image }} - path: misc/benchmark/${{ matrix.image }}.csv + export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }} + sudo -E make smoke-benchmark benchmark-nydus-no-prefetch: runs-on: ubuntu-latest @@ -118,163 +102,6 @@ jobs: include: - image: wordpress tag: 6.1.1 - - image: node - tag: 19.8 - - image: python - tag: 3.10.7 - - image: golang - tag: 1.19.3 - - image: ruby - tag: 3.1.3 - - image: amazoncorretto - tag: 8-al2022-jdk - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus - uses: actions/download-artifact@master - with: - name: nydus-artifact - path: target/release - - name: Download Nydusify - uses: actions/download-artifact@master - with: - name: nydusify-artifact - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source ${{ matrix.image }}:${{ matrix.tag }} \ - --target localhost:5000/${{ matrix.image }}:${{ matrix.tag }}_nydus \ - --fs-version 6 - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-no-prefetch --image ${{ matrix.image }}:${{ matrix.tag }} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-nydus-no-prefetch-${{matrix.image}} - path: misc/benchmark/${{matrix.image}}.csv - - benchmark-zran-no-prefetch: - runs-on: ubuntu-latest - needs: [contrib-build, nydus-build] - strategy: - matrix: - include: - - image: wordpress - tag: 6.1.1 - - image: node - tag: 19.8 - - image: python - tag: 3.10.7 - - image: golang - tag: 1.19.3 - - image: ruby - tag: 3.1.3 - - image: amazoncorretto - tag: 8-al2022-jdk - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus - uses: actions/download-artifact@master - with: - name: nydus-artifact - path: target/release - - name: Download Nydusify - uses: actions/download-artifact@master - with: - name: nydusify-artifact - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo docker pull ${{ matrix.image }}:${{ matrix.tag }} && docker tag ${{ matrix.image }}:${{ matrix.tag }} localhost:5000/${{ matrix.image }}:${{ matrix.tag }} - sudo docker push localhost:5000/${{ matrix.image }}:${{ matrix.tag }} - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source localhost:5000/${{ matrix.image }}:${{ matrix.tag }} \ - --target localhost:5000/${{ matrix.image }}:${{ matrix.tag }}_nydus \ - --fs-version 6 \ - --oci-ref - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-no-prefetch --image ${{ matrix.image }}:${{ matrix.tag }} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-zran-no-prefetch-${{matrix.image}} - path: misc/benchmark/${{matrix.image}}.csv - - benchmark-nydus-all-prefetch: - runs-on: ubuntu-latest - needs: [contrib-build, nydus-build] - strategy: - matrix: - include: - - image: wordpress - tag: 6.1.1 - - image: node - tag: 19.8 - - image: python - tag: 3.10.7 - - image: golang - tag: 1.19.3 - - image: ruby - tag: 3.1.3 - - image: amazoncorretto - tag: 8-al2022-jdk - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus - uses: actions/download-artifact@master - with: - name: nydus-artifact - path: target/release - - name: Download Nydusify - uses: actions/download-artifact@master - with: - name: nydusify-artifact - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source ${{ matrix.image }}:${{ matrix.tag }} \ - --target localhost:5000/${{ matrix.image }}:${{ matrix.tag }}_nydus \ - --fs-version 6 - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-all-prefetch --image ${{ matrix.image }}:${{ matrix.tag }} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-nydus-all-prefetch-${{matrix.image}} - path: misc/benchmark/${{matrix.image}}.csv - - benchmark-zran-all-prefetch: - runs-on: ubuntu-latest - needs: [contrib-build, nydus-build] - strategy: - matrix: - include: - - image: wordpress - tag: 6.1.1 - - image: node - tag: 19.8 - - image: python - tag: 3.10.7 - - image: golang - tag: 1.19.3 - - image: ruby - tag: 3.1.3 - - image: amazoncorretto - tag: 8-al2022-jdk steps: - name: Checkout uses: actions/checkout@v3 @@ -288,73 +115,13 @@ jobs: with: name: nydusify-artifact path: contrib/nydusify/cmd - - name: Prepare Nydus Environment + - name: Prepare Environment run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo docker pull ${{ matrix.image }}:${{ matrix.tag }} && docker tag ${{ matrix.image }}:${{ matrix.tag }} localhost:5000/${{ matrix.image }}:${{ matrix.tag }} - sudo docker push localhost:5000/${{ matrix.image }}:${{ matrix.tag }} - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source localhost:5000/${{ matrix.image }}:${{ matrix.tag }} \ - --target localhost:5000/${{ matrix.image }}:${{ matrix.tag }}_nydus \ - --fs-version 6 \ - --oci-ref + sudo bash misc/performance/prepare.sh - name: BenchMark Test run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-all-prefetch --image ${{ matrix.image }}:${{ matrix.tag }} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-zran-all-prefetch-${{matrix.image}} - path: misc/benchmark/${{matrix.image}}.csv - - benchmark-nydus-filelist-prefetch: - runs-on: ubuntu-latest - needs: [contrib-build, nydus-build] - strategy: - matrix: - include: - - image: wordpress - tag: 6.1.1 - - image: node - tag: 19.8 - - image: python - tag: 3.10.7 - - image: golang - tag: 1.19.3 - - image: ruby - tag: 3.1.3 - - image: amazoncorretto - tag: 8-al2022-jdk - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Download Nydus - uses: actions/download-artifact@master - with: - name: nydus-artifact - path: target/release - - name: Download Nydusify - uses: actions/download-artifact@master - with: - name: nydusify-artifact - path: contrib/nydusify/cmd - - name: Prepare Nydus Environment - run: | - sudo bash misc/benchmark/prepare_env.sh nydus - sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ - --source ${{ matrix.image }}:${{ matrix.tag }} \ - --target localhost:5000/${{ matrix.image }}:${{ matrix.tag }}_nydus \ - --fs-version 6 - - name: BenchMark Test - run: | - cd misc/benchmark - sudo python3 benchmark.py --mode nydus-filelist-prefetch --image ${{ matrix.image }}:${{ matrix.tag }} - - name: Save Test Result - uses: actions/upload-artifact@v3 - with: - name: benchmark-nydus-filelist-prefetch-${{matrix.image}} - path: misc/benchmark/${{matrix.image}}.csv + export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }} + sudo -E make smoke-benchmark benchmark-description: runs-on: ubuntu-latest @@ -362,107 +129,6 @@ jobs: - name: Description run: | echo "## Benchmark Environment" > $GITHUB_STEP_SUMMARY - echo "| operating system | cpu | memory |bandwidth|" >> $GITHUB_STEP_SUMMARY - echo "|:----------------:|:---:|:------:|:--------:|" >> $GITHUB_STEP_SUMMARY - echo "| ubuntu-22.04 | 2-core CPU (x86_64) | 7GB |10MB|" >> $GITHUB_STEP_SUMMARY - - benchmark-result: - runs-on: ubuntu-latest - needs: [benchmark-oci, benchmark-zran-all-prefetch, benchmark-zran-no-prefetch, benchmark-nydus-no-prefetch, benchmark-nydus-all-prefetch, benchmark-nydus-filelist-prefetch] - strategy: - matrix: - include: - - image: wordpress - tag: 6.1.1 - - image: node - tag: 19.8 - - image: python - tag: 3.10.7 - - image: golang - tag: 1.19.3 - - image: ruby - tag: 3.1.3 - - image: amazoncorretto - tag: 8-al2022-jdk - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Get Date - id: get-date - run: | - echo "date=$(date +%s)" >> $GITHUB_OUTPUT - shell: bash - - name: Restore benchmark result - uses: actions/cache/restore@v3 - with: - path: benchmark-result - key: benchmark-${{matrix.image}}-${{ steps.get-date.outputs.date }} - restore-keys: | - benchmark-${{matrix.image}} - - name: Download benchmark-oci - uses: actions/download-artifact@v3 - with: - name: benchmark-oci-${{matrix.image}} - path: benchmark-oci - - name: Download benchmark-nydus-no-prefetch - uses: actions/download-artifact@v3 - with: - name: benchmark-nydus-no-prefetch-${{matrix.image}} - path: benchmark-nydus-no-prefetch - - name: Download benchmark-zran-no-prefetch - uses: actions/download-artifact@v3 - with: - name: benchmark-zran-no-prefetch-${{matrix.image}} - path: benchmark-zran-no-prefetch - - name: Download benchmark-nydus-all-prefetch - uses: actions/download-artifact@v3 - with: - name: benchmark-nydus-all-prefetch-${{matrix.image}} - path: benchmark-nydus-all-prefetch - - name: Download benchmark-zran-all-prefetch - uses: actions/download-artifact@v3 - with: - name: benchmark-zran-all-prefetch-${{matrix.image}} - path: benchmark-zran-all-prefetch - - name: Download benchmark-nydus-filelist-prefetch - uses: actions/download-artifact@v3 - with: - name: benchmark-nydus-filelist-prefetch-${{matrix.image}} - path: benchmark-nydus-filelist-prefetch - - uses: geekyeggo/delete-artifact@v2 - with: - name: "*-${{matrix.image}}" - - name: Benchmark Workload - run: | - case ${{matrix.image}} in - "wordpress") - echo "### workload: wait the 80 port response" > $GITHUB_STEP_SUMMARY - ;; - "node") - echo "### workload: node index.js; wait the 80 port response" > $GITHUB_STEP_SUMMARY - ;; - "python") - echo "### workload: python -c 'print("hello")'" > $GITHUB_STEP_SUMMARY - ;; - "golang") - echo "### workload: go run main.go" > $GITHUB_STEP_SUMMARY - ;; - "ruby") - echo "### workload: ruby -e "puts \"hello\""" > $GITHUB_STEP_SUMMARY - ;; - "amazoncorretto") - echo "### workload: javac Main.java; java Main" > $GITHUB_STEP_SUMMARY - ;; - esac - - - name: Benchmark - run: | - if [ ! -d "benchmark-result" ]; then - mkdir benchmark-result - fi - sudo python3 misc/benchmark/benchmark_summary.py --mode benchmark-schedule >> $GITHUB_STEP_SUMMARY - - name: Save Benchmark Result - uses: actions/cache/save@v3 - with: - path: benchmark-result - key: benchmark-${{matrix.image}}-${{ steps.get-date.outputs.date }} + echo "| operating system | cpu | memory " >> $GITHUB_STEP_SUMMARY + echo "|:----------------:|:---:|:------ " >> $GITHUB_STEP_SUMMARY + echo "| ubuntu-22.04 | 2-core CPU (x86_64) | 7GB |" >> $GITHUB_STEP_SUMMARY diff --git a/Makefile b/Makefile index 7b212981dcf..89e4d33f367 100644 --- a/Makefile +++ b/Makefile @@ -136,6 +136,9 @@ smoke-only: smoke-performance: make -C smoke test-performance +smoke-benchmark: + make -C smoke test-benchmark + smoke: release smoke-only docker-nydus-smoke: diff --git a/smoke/Makefile b/smoke/Makefile index b38be0314b1..9812c7ce31e 100644 --- a/smoke/Makefile +++ b/smoke/Makefile @@ -23,6 +23,11 @@ test: build test-performance: build PERFORMANCE_TEST=True;sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=1 -test.run=TestPerformance +# BENCHMARK_TEST_IMAGE=wordpress:6.1.1 \ +# make test-benchmark +test-benchmark: build + BENCHMARK_TEST=True;sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=1 -test.run=TestBenchmark + # WORK_DIR=/tmp \ # NYDUS_STABLE_VERSION=v2.2.3 \ # NYDUS_STABLE_VERSION_EXPORT=v2_2_3 \ diff --git a/smoke/tests/benchmark_test.go b/smoke/tests/benchmark_test.go new file mode 100644 index 00000000000..abf5a2f7088 --- /dev/null +++ b/smoke/tests/benchmark_test.go @@ -0,0 +1,43 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "fmt" + "os" + "testing" + + "github.com/dragonflyoss/image-service/smoke/tests/tool" + "github.com/dragonflyoss/image-service/smoke/tests/tool/test" + "github.com/google/uuid" +) + +// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshoooter, nydusd, nydus-image and nydusify. +// Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md. + +type BenchmarkTestSuite struct { + t *testing.T + testImage string + testContainerName string +} + +func (b *BenchmarkTestSuite) TestBenchmark(t *testing.T) { + // choose test image + image := os.Getenv("BENCHMARK_TEST_IMAGE") + if image == "" || !tool.SupportContainerImage(tool.ImageRepo(b.t, image)) { + b.t.Fatalf("Unsupport image " + image) + } + + // run Contaienr + b.testContainerName = uuid.NewString() + b.t.Logf(fmt.Sprintf("BenchmarkTestSuite: %s %s", b.testImage, b.testContainerName)) +} + +func TestBenchmark(t *testing.T) { + if os.Getenv("BENCHMARK_TEST") == "" { + t.Skip("skipping performance test") + } + test.Run(t, &PerformanceTestSuite{t: t}) +}