Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: compare benchmark between PR and master #1244

Merged
merged 3 commits into from
Apr 25, 2023
Merged

feat: compare benchmark between PR and master #1244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
082cba9
refactor: use python refactor benchmark_summary.sh
Desiki-high Apr 23, 2023
47280ab
action: add benchmark-compare for PR
Desiki-high Apr 24, 2023
b12a3d9
misc: add image-size for benchmark
Desiki-high Apr 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
253 changes: 251 additions & 2 deletions .github/workflows/smoke.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,38 @@ jobs:
name: nydusify-artifact
path: contrib/nydusify/cmd

contrib-build-master:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v3
with:
ref: master
- name: Setup Golang
uses: actions/setup-go@v3
with:
go-version: ~1.18
- name: Golang Cache
uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-golang-
- name: Build Contrib
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.51.2
make -e DOCKER=false nydusify-release
make -e DOCKER=false contrib-test
- name: Upload Nydusify
uses: actions/upload-artifact@master
with:
name: nydusify-artifact-master
path: contrib/nydusify/cmd

nydus-build:
runs-on: ubuntu-latest
steps:
Expand All @@ -68,6 +100,30 @@ jobs:
target/release/nydus-image
target/release/nydusd

nydus-build-master:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v3
with:
ref: master
- name: Rust Cache
uses: Swatinem/rust-cache@v2.2.0
with:
cache-on-failure: true
- name: Build Nydus
run: |
rustup component add rustfmt clippy
make
- name: Upload Nydus Binaries
uses: actions/upload-artifact@master
with:
name: nydus-artifact-master
path: |
target/release/nydus-image
target/release/nydusd

nydus-integration-test:
runs-on: ubuntu-latest
needs: [contrib-build, nydus-build]
Expand Down Expand Up @@ -194,6 +250,40 @@ jobs:
name: benchmark-nydus-no-prefetch
path: misc/benchmark/${{env.IMAGE}}.csv

benchmark-nydus-no-prefetch-master:
runs-on: ubuntu-latest
needs: [contrib-build-master, nydus-build-master]
if: github.event_name == 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Download Nydus Master
uses: actions/download-artifact@master
with:
name: nydus-artifact-master
path: target/release
- name: Download Nydusify Master
uses: actions/download-artifact@master
with:
name: nydusify-artifact-master
path: contrib/nydusify/cmd
- name: Prepare Nydus Environment
run: |
sudo bash misc/benchmark/prepare_env.sh nydus
sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
--source ${{env.IMAGE}}:${{env.TAG}} \
--target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \
--fs-version 6
- name: BenchMark Test
run: |
cd misc/benchmark
sudo python3 benchmark.py --mode nydus-no-prefetch
- name: Save Test Result
uses: actions/upload-artifact@v3
with:
name: benchmark-nydus-no-prefetch-master
path: misc/benchmark/${{env.IMAGE}}.csv

benchmark-zran-no-prefetch:
runs-on: ubuntu-latest
needs: [contrib-build, nydus-build]
Expand All @@ -218,7 +308,8 @@ jobs:
sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
--source localhost:5000/${{env.IMAGE}}:${{env.TAG}} \
--target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \
--fs-version 6
--fs-version 6 \
--oci-ref
- name: BenchMark Test
run: |
cd misc/benchmark
Expand All @@ -229,6 +320,43 @@ jobs:
name: benchmark-zran-no-prefetch
path: misc/benchmark/${{env.IMAGE}}.csv

benchmark-zran-no-prefetch-master:
runs-on: ubuntu-latest
needs: [contrib-build-master, nydus-build-master]
if: github.event_name == 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Download Nydus Master
uses: actions/download-artifact@master
with:
name: nydus-artifact-master
path: target/release
- name: Download Nydusify Master
uses: actions/download-artifact@master
with:
name: nydusify-artifact-master
path: contrib/nydusify/cmd
- name: Prepare Nydus Environment
run: |
sudo bash misc/benchmark/prepare_env.sh nydus
sudo docker pull ${{env.IMAGE}}:${{env.TAG}} && docker tag ${{env.IMAGE}}:${{env.TAG}} localhost:5000/${{env.IMAGE}}:${{env.TAG}}
sudo docker push localhost:5000/${{env.IMAGE}}:${{env.TAG}}
sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
--source localhost:5000/${{env.IMAGE}}:${{env.TAG}} \
--target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \
--fs-version 6 \
--oci-ref
- name: BenchMark Test
run: |
cd misc/benchmark
sudo python3 benchmark.py --mode nydus-no-prefetch
- name: Save Test Result
uses: actions/upload-artifact@v3
with:
name: benchmark-zran-no-prefetch-master
path: misc/benchmark/${{env.IMAGE}}.csv

benchmark-nydus-all-prefetch:
runs-on: ubuntu-latest
needs: [contrib-build, nydus-build]
Expand Down Expand Up @@ -262,6 +390,40 @@ jobs:
name: benchmark-nydus-all-prefetch
path: misc/benchmark/${{env.IMAGE}}.csv

benchmark-nydus-all-prefetch-master:
runs-on: ubuntu-latest
needs: [contrib-build-master, nydus-build-master]
if: github.event_name == 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Download Nydus Master
uses: actions/download-artifact@master
with:
name: nydus-artifact-master
path: target/release
- name: Download Nydusify Master
uses: actions/download-artifact@master
with:
name: nydusify-artifact-master
path: contrib/nydusify/cmd
- name: Prepare Nydus Environment
run: |
sudo bash misc/benchmark/prepare_env.sh nydus
sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
--source ${{env.IMAGE}}:${{env.TAG}} \
--target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \
--fs-version 6
- name: BenchMark Test
run: |
cd misc/benchmark
sudo python3 benchmark.py --mode nydus-all-prefetch
- name: Save Test Result
uses: actions/upload-artifact@v3
with:
name: benchmark-nydus-all-prefetch-master
path: misc/benchmark/${{env.IMAGE}}.csv

benchmark-zran-all-prefetch:
runs-on: ubuntu-latest
needs: [contrib-build, nydus-build]
Expand Down Expand Up @@ -298,6 +460,43 @@ jobs:
name: benchmark-zran-all-prefetch
path: misc/benchmark/${{env.IMAGE}}.csv

benchmark-zran-all-prefetch-master:
runs-on: ubuntu-latest
needs: [contrib-build-master, nydus-build-master]
if: github.event_name == 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Download Nydus Master
uses: actions/download-artifact@master
with:
name: nydus-artifact-master
path: target/release
- name: Download Nydusify Master
uses: actions/download-artifact@master
with:
name: nydusify-artifact-master
path: contrib/nydusify/cmd
- name: Prepare Nydus Environment
run: |
sudo bash misc/benchmark/prepare_env.sh nydus
sudo docker pull ${{env.IMAGE}}:${{env.TAG}} && docker tag ${{env.IMAGE}}:${{env.TAG}} localhost:5000/${{env.IMAGE}}:${{env.TAG}}
sudo docker push localhost:5000/${{env.IMAGE}}:${{env.TAG}}
sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
--source localhost:5000/${{env.IMAGE}}:${{env.TAG}} \
--target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \
--fs-version 6 \
--oci-ref
- name: BenchMark Test
run: |
cd misc/benchmark
sudo python3 benchmark.py --mode nydus-all-prefetch
- name: Save Test Result
uses: actions/upload-artifact@v3
with:
name: benchmark-zran-all-prefetch-master
path: misc/benchmark/${{env.IMAGE}}.csv

benchmark-nydus-filelist-prefetch:
runs-on: ubuntu-latest
needs: [contrib-build, nydus-build]
Expand Down Expand Up @@ -331,9 +530,59 @@ jobs:
name: benchmark-nydus-filelist-prefetch
path: misc/benchmark/${{env.IMAGE}}.csv

benchmark-nydus-filelist-prefetch-master:
runs-on: ubuntu-latest
needs: [contrib-build-master, nydus-build-master]
if: github.event_name == 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Download Nydus Master
uses: actions/download-artifact@master
with:
name: nydus-artifact-master
path: target/release
- name: Download Nydusify Master
uses: actions/download-artifact@master
with:
name: nydusify-artifact-master
path: contrib/nydusify/cmd
- name: Prepare Nydus Environment
run: |
sudo bash misc/benchmark/prepare_env.sh nydus
sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
--source ${{env.IMAGE}}:${{env.TAG}} \
--target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \
--fs-version 6
- name: BenchMark Test
run: |
cd misc/benchmark
sudo python3 benchmark.py --mode nydus-filelist-prefetch
- name: Save Test Result
uses: actions/upload-artifact@v3
with:
name: benchmark-nydus-filelist-prefetch-master
path: misc/benchmark/${{env.IMAGE}}.csv

benchmark-result:
runs-on: ubuntu-latest
needs: [benchmark-oci, benchmark-zran-all-prefetch, benchmark-zran-no-prefetch, benchmark-nydus-no-prefetch, benchmark-nydus-all-prefetch, benchmark-nydus-filelist-prefetch]
if: github.event_name != 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: actions/download-artifact@v3
- uses: geekyeggo/delete-artifact@v2
with:
name: '*'
- name: Save Result
run: |
sudo python3 misc/benchmark/benchmark_summary.py --mode benchmark-result > $GITHUB_STEP_SUMMARY

benchmark-compare:
runs-on: ubuntu-latest
needs: [benchmark-oci, benchmark-zran-all-prefetch, benchmark-zran-no-prefetch, benchmark-nydus-no-prefetch, benchmark-nydus-all-prefetch, benchmark-nydus-filelist-prefetch, benchmark-zran-all-prefetch-master, benchmark-zran-no-prefetch-master, benchmark-nydus-no-prefetch-master, benchmark-nydus-all-prefetch-master, benchmark-nydus-filelist-prefetch-master]
if: github.event_name == 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v3
Expand All @@ -343,7 +592,7 @@ jobs:
name: '*'
- name: Save Result
run: |
sudo bash misc/benchmark/benchmark_summary.sh > $GITHUB_STEP_SUMMARY
sudo python3 misc/benchmark/benchmark_summary.py --mode benchmark-compare > $GITHUB_STEP_SUMMARY

nydus-unit-test:
runs-on: ubuntu-latest
Expand Down
9 changes: 6 additions & 3 deletions misc/benchmark/bench.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,13 +184,16 @@ def run_cmd_url_wait(self, repo, runargs):
run_elapsed = datetime.timestamp(end_run) - datetime.timestamp(start_run)

print("Run time: %f s" % run_elapsed)

read_amount, read_count = "-", "-"
if self.snapshotter == "nydus":
read_amount, read_count = metrics.collect_backend()
image_size = metrics.collect_size(image_repo(repo), image_tag(repo))

if self.cleanup:
self.clean_up(image_ref, container_id)

return pull_elapsed, create_elapsed, run_elapsed, read_amount, read_count
return pull_elapsed, create_elapsed, run_elapsed, image_size, read_amount, read_count

def pull_cmd(self, image_ref):
insecure_flag = "--insecure-registry" if self.insecure_registry else ""
Expand Down Expand Up @@ -253,11 +256,11 @@ def bench_image(local_registry, insecure_local_registry, image, f: TextIOWrapper
cleanup=True,
insecure_registry=insecure_local_registry,
)
pull_elapsed, create_elapsed, run_elapsed, read_amount, read_count = runner.run(bench)
pull_elapsed, create_elapsed, run_elapsed, image_size, read_amount, read_count = runner.run(bench)
total_elapsed = f"{pull_elapsed + create_elapsed + run_elapsed: .6f}"
pull_elapsed = f"{pull_elapsed: .6f}"
create_elapsed = f"{create_elapsed: .6f}"
run_elapsed = f"{run_elapsed: .6f}"
line = f"{bench.name},{pull_elapsed},{create_elapsed},{run_elapsed},{total_elapsed},{read_amount},{read_count}"
line = f"{pull_elapsed},{create_elapsed},{run_elapsed},{total_elapsed},{image_size},{read_amount},{read_count}"
f.writelines(line + "\n")
f.flush()
4 changes: 1 addition & 3 deletions misc/benchmark/benchmark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ def main():
# bench
start_bench(cfg, cfg["image"], mode)


def collect_metrics(cfg: dict, image: str) -> str:
"""
collect container access metrics
Expand All @@ -53,9 +54,6 @@ def start_bench(cfg: dict, image: str, mode: str):
bench oci, nydus without prefetch, nydus with all prefetch, nydus witch prefetch file list
"""
f = open(util.image_repo(image) + ".csv", "w")
csv_headers = "repo,pull_elapsed(s),create_elapsed(s),run_elapsed(s),total_elapsed(s),read_amount(MB),read_count"
f.writelines(csv_headers + "\n")
f.flush()
if mode == "oci":
util.enable_wondersphaper(cfg["bandwith"])
bench.bench_image(cfg["local_registry"], cfg["insecure_local_registry"], image, f)
Expand Down
Loading