From 017d84bd6a681655aeae3bdc81523fef249b621d Mon Sep 17 00:00:00 2001 From: Yadong Ding Date: Wed, 6 Dec 2023 14:58:45 +0800 Subject: [PATCH 1/2] smoke: support different snapshooter in bench We can use overlayfs to test OCI V1 image. Signed-off-by: Yadong Ding --- smoke/tests/benchmark_test.go | 19 +++++++++++---- smoke/tests/performance_test.go | 2 +- smoke/tests/tool/container.go | 41 ++++++++++++++++++--------------- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/smoke/tests/benchmark_test.go b/smoke/tests/benchmark_test.go index b3e10777898..3362df87b9c 100644 --- a/smoke/tests/benchmark_test.go +++ b/smoke/tests/benchmark_test.go @@ -15,7 +15,7 @@ import ( "github.com/google/uuid" ) -// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshoooter, nydusd, nydus-image and nydusify. +// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshotter, nydusd, nydus-image and nydusify. // Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md. // TestBenchmark will dump json file(benchmark.json) which inlcudes container e2e time, image size, read-amount and read-cout. // Example: @@ -30,11 +30,16 @@ type BenchmarkTestSuite struct { t *testing.T testImage string testContainerName string + snapshotter string metric tool.ContainerMetrics } func (b *BenchmarkTestSuite) TestBenchmark(t *testing.T) { ctx := tool.DefaultContext(b.t) + b.snapshotter = os.Getenv("SNAPSHOTTER") + if b.snapshotter == "" { + b.snapshotter = "nydus" + } // choose test mode mode := os.Getenv("BENCHMARK_MODE") @@ -42,6 +47,7 @@ func (b *BenchmarkTestSuite) TestBenchmark(t *testing.T) { mode = "fs-version-6" } switch mode { + case "oci": case "fs-version-5": ctx.Build.FSVersion = "5" case "fs-version-6": @@ -65,7 +71,7 @@ func (b *BenchmarkTestSuite) TestBenchmark(t *testing.T) { // run contaienr b.testContainerName = uuid.NewString() - containerMetic := tool.RunContainer(b.t, b.testImage, b.testContainerName) + containerMetic := tool.RunContainer(b.t, b.testImage, b.snapshotter, b.testContainerName) b.metric = tool.ContainerMetrics{ E2ETime: containerMetic.E2ETime, ReadCount: containerMetic.ReadCount, @@ -106,7 +112,6 @@ func (b *BenchmarkTestSuite) prepareImage(t *testing.T, ctx *tool.Context, mode ctx.Binary.Nydusify, logLevel, source, target, ctx.Binary.Builder, ctx.Env.WorkDir, fsVersion, enableOCIRef, convertMetricFile) tool.RunWithoutOutput(t, convertCmd) defer os.Remove(convertMetricFile) - b.testImage = target metricData, err := os.ReadFile(convertMetricFile) if err != nil { @@ -119,7 +124,13 @@ func (b *BenchmarkTestSuite) prepareImage(t *testing.T, ctx *tool.Context, mode t.Fatalf("can't parsing convert metric file") return 0 } - return convertMetirc["TargetImageSize"] + if b.snapshotter == "nydus" { + b.testImage = target + return convertMetirc["TargetImageSize"] + } else { + b.testImage = source + return convertMetirc["SourceImageSize"] + } } func (b *BenchmarkTestSuite) dumpMetric() { diff --git a/smoke/tests/performance_test.go b/smoke/tests/performance_test.go index 4199e7bdd0a..e1b6b2fc9ed 100644 --- a/smoke/tests/performance_test.go +++ b/smoke/tests/performance_test.go @@ -14,7 +14,7 @@ import ( "github.com/google/uuid" ) -// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshoooter, nydusd, nydus-image and nydusify. +// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshotter, nydusd, nydus-image and nydusify. // Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md. type PerformanceTestSuite struct { diff --git a/smoke/tests/tool/container.go b/smoke/tests/tool/container.go index ea8c72114f4..dd290b90896 100644 --- a/smoke/tests/tool/container.go +++ b/smoke/tests/tool/container.go @@ -98,8 +98,8 @@ func SupportContainerImage(image string) bool { } // runUrlWaitContainer run container util geting http response from WaitUrl -func runUrlWaitContainer(t *testing.T, image string, containerName string, runArgs RunArgs) { - cmd := "sudo nerdctl --insecure-registry --snapshotter nydus run -d --net=host" +func runUrlWaitContainer(t *testing.T, image string, snapshotter string, containerName string, runArgs RunArgs) { + cmd := fmt.Sprintf("sudo nerdctl --insecure-registry --snapshotter %s run -d --net=host", snapshotter) if runArgs.Mount.source != "" { currentDir, err := os.Getwd() if err != nil { @@ -120,8 +120,8 @@ func runUrlWaitContainer(t *testing.T, image string, containerName string, runAr } // runCmdStdoutContainer run some commands in container by entrypoint.sh -func runCmdStdoutContainer(t *testing.T, image string, containerName string, runArgs RunArgs) { - cmd := "sudo nerdctl --insecure-registry --snapshotter nydus run -i --net=host" +func runCmdStdoutContainer(t *testing.T, image string, snapshotter string, containerName string, runArgs RunArgs) { + cmd := fmt.Sprintf("sudo nerdctl --insecure-registry --snapshotter %s run -i --net=host", snapshotter) if runArgs.Mount.source != "" { currentDir, err := os.Getwd() if err != nil { @@ -138,8 +138,8 @@ func runCmdStdoutContainer(t *testing.T, image string, containerName string, run func RunContainerWithBaseline(t *testing.T, image string, containerName string, mode string) { args, ok := URL_WAIT[ImageRepo(t, image)] if ok { - runUrlWaitContainer(t, image, containerName, args) - defer clearContainer(t, image, containerName) + runUrlWaitContainer(t, image, "nydus", containerName, args) + defer clearContainer(t, image, "nydus", containerName) } else { t.Fatalf(fmt.Sprintf("%s is not in URL_WAIT", image)) } @@ -155,34 +155,37 @@ func RunContainerWithBaseline(t *testing.T, image string, containerName string, } // RunContainer and return container metric -func RunContainer(t *testing.T, image string, containerName string) *ContainerMetrics { +func RunContainer(t *testing.T, image string, snapshotter string, containerName string) *ContainerMetrics { var containerMetic ContainerMetrics startTime := time.Now() // runContainer args, ok := URL_WAIT[ImageRepo(t, image)] if ok { - runUrlWaitContainer(t, image, containerName, args) - defer clearContainer(t, image, containerName) + runUrlWaitContainer(t, image, snapshotter, containerName, args) + defer clearContainer(t, image, snapshotter, containerName) } else if args, ok := CMD_STDOUT[ImageRepo(t, image)]; ok { - runCmdStdoutContainer(t, image, containerName, args) - defer clearContainer(t, image, containerName) + runCmdStdoutContainer(t, image, snapshotter, containerName, args) + defer clearContainer(t, image, snapshotter, containerName) } containerMetic.E2ETime = time.Since(startTime) - backendMetrics, err := getContainerBackendMetrics(t) - if err != nil { - t.Logf(err.Error()) + if snapshotter == "nydus" { + backendMetrics, err := getContainerBackendMetrics(t) + if err != nil { + t.Logf(err.Error()) + } + containerMetic.ReadAmountTotal = backendMetrics.ReadAmountTotal + containerMetic.ReadCount = backendMetrics.ReadCount } - containerMetic.ReadAmountTotal = backendMetrics.ReadAmountTotal - containerMetic.ReadCount = backendMetrics.ReadCount + return &containerMetic } // ClearContainer clear container by containerName -func clearContainer(t *testing.T, image string, containerName string) { - RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter nydus rm -f %s", containerName)) - RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter nydus image rm %s", image)) +func clearContainer(t *testing.T, image string, snapshotter, containerName string) { + RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter %s rm -f %s", snapshotter, containerName)) + RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter %s image rm %s", snapshotter, image)) } // getContainerBackendMetrics get backend metrics by nydus api sock From 072451ded4807476f36a3b0be8c1541ef4e2b6c6 Mon Sep 17 00:00:00 2001 From: Yadong Ding Date: Wed, 6 Dec 2023 15:11:27 +0800 Subject: [PATCH 2/2] action: add oci in benchmark Signed-off-by: Yadong Ding --- .github/workflows/benchmark.yml | 55 ++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c6aa0eb459e..6302cc7cd74 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -65,6 +65,53 @@ jobs: echo "|:----------------:|:---:|:------ " >> $GITHUB_STEP_SUMMARY echo "| ubuntu-22.04 | 2-core CPU (x86_64) | 7GB |" >> $GITHUB_STEP_SUMMARY + benchmark-oci: + runs-on: ubuntu-latest + needs: [contrib-build, nydus-build] + strategy: + matrix: + include: + - image: wordpress + tag: 6.1.1 + - image: node + tag: 19.8 + - image: python + tag: 3.10.7 + - image: golang + tag: 1.19.3 + - image: ruby + tag: 3.1.3 + - image: amazoncorretto + tag: 8-al2022-jdk + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Download Nydus + uses: actions/download-artifact@master + with: + name: nydus-artifact + path: target/release + - name: Download Nydusify + uses: actions/download-artifact@master + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + - name: Prepare Environment + run: | + sudo bash misc/performance/prepare.sh + - name: BenchMark Test + run: | + export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }} + export BENCHMARK_MODE=oci + export BENCHMARK_METRIC_FILE=${{ matrix.image }}-oci.json + export SNAPSHOTTER=overlayfs + sudo -E make smoke-benchmark + - name: Save BenchMark Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-oci-${{ matrix.image }} + path: smoke/${{ matrix.image }}-oci.json + benchmark-fsversion-v5: runs-on: ubuntu-latest needs: [contrib-build, nydus-build] @@ -205,7 +252,7 @@ jobs: benchmark-result: runs-on: ubuntu-latest - needs: [benchmark-fsversion-v5, benchmark-fsversion-v6, benchmark-zran] + needs: [benchmark-oci, benchmark-fsversion-v5, benchmark-fsversion-v6, benchmark-zran] strategy: matrix: include: @@ -224,6 +271,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + - name: Download benchmark-oci + uses: actions/download-artifact@v3 + with: + name: benchmark-oci-${{ matrix.image }} + path: benchmark-result - name: Download benchmark-fsversion-v5 uses: actions/download-artifact@v3 with: @@ -269,6 +321,7 @@ jobs: "${{ matrix.image }}-fsversion-v5.json" "${{ matrix.image }}-fsversion-v6.json" "${{ matrix.image }}-zran.json" + "${{ matrix.image }}-oci.json" ) echo "| bench-result | e2e-time(s) | read-count | read-amount(MB) | image-size(MB) |" >> $GITHUB_STEP_SUMMARY echo "|:-------------|:----------:|:----------:|:---------------:|:--------:|" >> $GITHUB_STEP_SUMMARY