Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

smoke: add oci bench #1506

Merged
merged 2 commits into from
Dec 7, 2023
Merged

smoke: add oci bench #1506

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
017d84b
smoke: support different snapshooter in bench
Desiki-high Dec 6, 2023
072451d
action: add oci in benchmark
Desiki-high Dec 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 54 additions & 1 deletion .github/workflows/benchmark.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,53 @@ jobs:
echo "|:----------------:|:---:|:------ " >> $GITHUB_STEP_SUMMARY
echo "| ubuntu-22.04 | 2-core CPU (x86_64) | 7GB |" >> $GITHUB_STEP_SUMMARY

benchmark-oci:
runs-on: ubuntu-latest
needs: [contrib-build, nydus-build]
strategy:
matrix:
include:
- image: wordpress
tag: 6.1.1
- image: node
tag: 19.8
- image: python
tag: 3.10.7
- image: golang
tag: 1.19.3
- image: ruby
tag: 3.1.3
- image: amazoncorretto
tag: 8-al2022-jdk
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Download Nydus
uses: actions/download-artifact@master
with:
name: nydus-artifact
path: target/release
- name: Download Nydusify
uses: actions/download-artifact@master
with:
name: nydusify-artifact
path: contrib/nydusify/cmd
- name: Prepare Environment
run: |
sudo bash misc/performance/prepare.sh
- name: BenchMark Test
run: |
export BENCHMARK_TEST_IMAGE=${{ matrix.image }}:${{ matrix.tag }}
export BENCHMARK_MODE=oci
export BENCHMARK_METRIC_FILE=${{ matrix.image }}-oci.json
export SNAPSHOTTER=overlayfs
sudo -E make smoke-benchmark
- name: Save BenchMark Result
uses: actions/upload-artifact@v3
with:
name: benchmark-oci-${{ matrix.image }}
path: smoke/${{ matrix.image }}-oci.json

benchmark-fsversion-v5:
runs-on: ubuntu-latest
needs: [contrib-build, nydus-build]
Expand Down Expand Up @@ -205,7 +252,7 @@ jobs:

benchmark-result:
runs-on: ubuntu-latest
needs: [benchmark-fsversion-v5, benchmark-fsversion-v6, benchmark-zran]
needs: [benchmark-oci, benchmark-fsversion-v5, benchmark-fsversion-v6, benchmark-zran]
strategy:
matrix:
include:
Expand All @@ -224,6 +271,11 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Download benchmark-oci
uses: actions/download-artifact@v3
with:
name: benchmark-oci-${{ matrix.image }}
path: benchmark-result
- name: Download benchmark-fsversion-v5
uses: actions/download-artifact@v3
with:
Expand Down Expand Up @@ -269,6 +321,7 @@ jobs:
"${{ matrix.image }}-fsversion-v5.json"
"${{ matrix.image }}-fsversion-v6.json"
"${{ matrix.image }}-zran.json"
"${{ matrix.image }}-oci.json"
)
echo "| bench-result | e2e-time(s) | read-count | read-amount(MB) | image-size(MB) |" >> $GITHUB_STEP_SUMMARY
echo "|:-------------|:----------:|:----------:|:---------------:|:--------:|" >> $GITHUB_STEP_SUMMARY
Expand Down
19 changes: 15 additions & 4 deletions smoke/tests/benchmark_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"github.com/google/uuid"
)

// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshoooter, nydusd, nydus-image and nydusify.
// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshotter, nydusd, nydus-image and nydusify.
// Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md.
// TestBenchmark will dump json file(benchmark.json) which inlcudes container e2e time, image size, read-amount and read-cout.
// Example:
Expand All @@ -30,18 +30,24 @@ type BenchmarkTestSuite struct {
t *testing.T
testImage string
testContainerName string
snapshotter string
metric tool.ContainerMetrics
}

func (b *BenchmarkTestSuite) TestBenchmark(t *testing.T) {
ctx := tool.DefaultContext(b.t)
b.snapshotter = os.Getenv("SNAPSHOTTER")
if b.snapshotter == "" {
b.snapshotter = "nydus"
}

// choose test mode
mode := os.Getenv("BENCHMARK_MODE")
if mode == "" {
mode = "fs-version-6"
}
switch mode {
case "oci":
case "fs-version-5":
ctx.Build.FSVersion = "5"
case "fs-version-6":
Expand All @@ -65,7 +71,7 @@ func (b *BenchmarkTestSuite) TestBenchmark(t *testing.T) {

// run contaienr
b.testContainerName = uuid.NewString()
containerMetic := tool.RunContainer(b.t, b.testImage, b.testContainerName)
containerMetic := tool.RunContainer(b.t, b.testImage, b.snapshotter, b.testContainerName)
b.metric = tool.ContainerMetrics{
E2ETime: containerMetic.E2ETime,
ReadCount: containerMetic.ReadCount,
Expand Down Expand Up @@ -106,7 +112,6 @@ func (b *BenchmarkTestSuite) prepareImage(t *testing.T, ctx *tool.Context, mode
ctx.Binary.Nydusify, logLevel, source, target, ctx.Binary.Builder, ctx.Env.WorkDir, fsVersion, enableOCIRef, convertMetricFile)
tool.RunWithoutOutput(t, convertCmd)
defer os.Remove(convertMetricFile)
b.testImage = target

metricData, err := os.ReadFile(convertMetricFile)
if err != nil {
Expand All @@ -119,7 +124,13 @@ func (b *BenchmarkTestSuite) prepareImage(t *testing.T, ctx *tool.Context, mode
t.Fatalf("can't parsing convert metric file")
return 0
}
return convertMetirc["TargetImageSize"]
if b.snapshotter == "nydus" {
b.testImage = target
return convertMetirc["TargetImageSize"]
} else {
b.testImage = source
return convertMetirc["SourceImageSize"]
}
}

func (b *BenchmarkTestSuite) dumpMetric() {
Expand Down
2 changes: 1 addition & 1 deletion smoke/tests/performance_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"github.com/google/uuid"
)

// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshoooter, nydusd, nydus-image and nydusify.
// Environment Requirement: Containerd, nerdctl >= 0.22, nydus-snapshotter, nydusd, nydus-image and nydusify.
// Prepare: setup nydus for containerd, reference: https://github.com/dragonflyoss/nydus/blob/master/docs/containerd-env-setup.md.

type PerformanceTestSuite struct {
Expand Down
41 changes: 22 additions & 19 deletions smoke/tests/tool/container.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ func SupportContainerImage(image string) bool {
}

// runUrlWaitContainer run container util geting http response from WaitUrl
func runUrlWaitContainer(t *testing.T, image string, containerName string, runArgs RunArgs) {
cmd := "sudo nerdctl --insecure-registry --snapshotter nydus run -d --net=host"
func runUrlWaitContainer(t *testing.T, image string, snapshotter string, containerName string, runArgs RunArgs) {
cmd := fmt.Sprintf("sudo nerdctl --insecure-registry --snapshotter %s run -d --net=host", snapshotter)
if runArgs.Mount.source != "" {
currentDir, err := os.Getwd()
if err != nil {
Expand All @@ -120,8 +120,8 @@ func runUrlWaitContainer(t *testing.T, image string, containerName string, runAr
}

// runCmdStdoutContainer run some commands in container by entrypoint.sh
func runCmdStdoutContainer(t *testing.T, image string, containerName string, runArgs RunArgs) {
cmd := "sudo nerdctl --insecure-registry --snapshotter nydus run -i --net=host"
func runCmdStdoutContainer(t *testing.T, image string, snapshotter string, containerName string, runArgs RunArgs) {
cmd := fmt.Sprintf("sudo nerdctl --insecure-registry --snapshotter %s run -i --net=host", snapshotter)
if runArgs.Mount.source != "" {
currentDir, err := os.Getwd()
if err != nil {
Expand All @@ -138,8 +138,8 @@ func runCmdStdoutContainer(t *testing.T, image string, containerName string, run
func RunContainerWithBaseline(t *testing.T, image string, containerName string, mode string) {
args, ok := URL_WAIT[ImageRepo(t, image)]
if ok {
runUrlWaitContainer(t, image, containerName, args)
defer clearContainer(t, image, containerName)
runUrlWaitContainer(t, image, "nydus", containerName, args)
defer clearContainer(t, image, "nydus", containerName)
} else {
t.Fatalf(fmt.Sprintf("%s is not in URL_WAIT", image))
}
Expand All @@ -155,34 +155,37 @@ func RunContainerWithBaseline(t *testing.T, image string, containerName string,
}

// RunContainer and return container metric
func RunContainer(t *testing.T, image string, containerName string) *ContainerMetrics {
func RunContainer(t *testing.T, image string, snapshotter string, containerName string) *ContainerMetrics {
var containerMetic ContainerMetrics
startTime := time.Now()

// runContainer
args, ok := URL_WAIT[ImageRepo(t, image)]
if ok {
runUrlWaitContainer(t, image, containerName, args)
defer clearContainer(t, image, containerName)
runUrlWaitContainer(t, image, snapshotter, containerName, args)
defer clearContainer(t, image, snapshotter, containerName)
} else if args, ok := CMD_STDOUT[ImageRepo(t, image)]; ok {
runCmdStdoutContainer(t, image, containerName, args)
defer clearContainer(t, image, containerName)
runCmdStdoutContainer(t, image, snapshotter, containerName, args)
defer clearContainer(t, image, snapshotter, containerName)
}

containerMetic.E2ETime = time.Since(startTime)
backendMetrics, err := getContainerBackendMetrics(t)
if err != nil {
t.Logf(err.Error())
if snapshotter == "nydus" {
backendMetrics, err := getContainerBackendMetrics(t)
if err != nil {
t.Logf(err.Error())
}
containerMetic.ReadAmountTotal = backendMetrics.ReadAmountTotal
containerMetic.ReadCount = backendMetrics.ReadCount
}
containerMetic.ReadAmountTotal = backendMetrics.ReadAmountTotal
containerMetic.ReadCount = backendMetrics.ReadCount

return &containerMetic
}

// ClearContainer clear container by containerName
func clearContainer(t *testing.T, image string, containerName string) {
RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter nydus rm -f %s", containerName))
RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter nydus image rm %s", image))
func clearContainer(t *testing.T, image string, snapshotter, containerName string) {
RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter %s rm -f %s", snapshotter, containerName))
RunWithoutOutput(t, fmt.Sprintf("sudo nerdctl --snapshotter %s image rm %s", snapshotter, image))
}

// getContainerBackendMetrics get backend metrics by nydus api sock
Expand Down