diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 69589a1ff88..aa6e674fa17 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -47,6 +47,38 @@ jobs: name: nydusify-artifact path: contrib/nydusify/cmd + contrib-build-master: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: master + - name: Setup Golang + uses: actions/setup-go@v3 + with: + go-version: ~1.18 + - name: Golang Cache + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-golang-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-golang- + - name: Build Contrib + run: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.51.2 + make -e DOCKER=false nydusify-release + make -e DOCKER=false contrib-test + - name: Upload Nydusify + uses: actions/upload-artifact@master + with: + name: nydusify-artifact-master + path: contrib/nydusify/cmd + nydus-build: runs-on: ubuntu-latest steps: @@ -68,6 +100,30 @@ jobs: target/release/nydus-image target/release/nydusd + nydus-build-master: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: master + - name: Rust Cache + uses: Swatinem/rust-cache@v2.2.0 + with: + cache-on-failure: true + - name: Build Nydus + run: | + rustup component add rustfmt clippy + make + - name: Upload Nydus Binaries + uses: actions/upload-artifact@master + with: + name: nydus-artifact-master + path: | + target/release/nydus-image + target/release/nydusd + nydus-integration-test: runs-on: ubuntu-latest needs: [contrib-build, nydus-build] @@ -194,6 +250,42 @@ jobs: name: benchmark-nydus-no-prefetch path: misc/benchmark/${{env.IMAGE}}.csv + benchmark-nydus-no-prefetch-master: + runs-on: ubuntu-latest + needs: [contrib-build-master, nydus-build-master] + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: master + - name: Download Nydus Master + uses: actions/download-artifact@master + with: + name: nydus-artifact-master + path: target/release + - name: Download Nydusify Master + uses: actions/download-artifact@master + with: + name: nydusify-artifact-master + path: contrib/nydusify/cmd + - name: Prepare Nydus Environment + run: | + sudo bash misc/benchmark/prepare_env.sh nydus + sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ + --source ${{env.IMAGE}}:${{env.TAG}} \ + --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ + --fs-version 6 + - name: BenchMark Test + run: | + cd misc/benchmark + sudo python3 benchmark.py --mode nydus-no-prefetch + - name: Save Test Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-nydus-no-prefetch-master + path: misc/benchmark/${{env.IMAGE}}.csv + benchmark-zran-no-prefetch: runs-on: ubuntu-latest needs: [contrib-build, nydus-build] @@ -229,6 +321,44 @@ jobs: name: benchmark-zran-no-prefetch path: misc/benchmark/${{env.IMAGE}}.csv + benchmark-zran-no-prefetch-master: + runs-on: ubuntu-latest + needs: [contrib-build-master, nydus-build-master] + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: master + - name: Download Nydus Master + uses: actions/download-artifact@master + with: + name: nydus-artifact-master + path: target/release + - name: Download Nydusify Master + uses: actions/download-artifact@master + with: + name: nydusify-artifact-master + path: contrib/nydusify/cmd + - name: Prepare Nydus Environment + run: | + sudo bash misc/benchmark/prepare_env.sh nydus + sudo docker pull ${{env.IMAGE}}:${{env.TAG}} && docker tag ${{env.IMAGE}}:${{env.TAG}} localhost:5000/${{env.IMAGE}}:${{env.TAG}} + sudo docker push localhost:5000/${{env.IMAGE}}:${{env.TAG}} + sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ + --source localhost:5000/${{env.IMAGE}}:${{env.TAG}} \ + --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ + --fs-version 6 + - name: BenchMark Test + run: | + cd misc/benchmark + sudo python3 benchmark.py --mode nydus-no-prefetch + - name: Save Test Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-zran-no-prefetch-master + path: misc/benchmark/${{env.IMAGE}}.csv + benchmark-nydus-all-prefetch: runs-on: ubuntu-latest needs: [contrib-build, nydus-build] @@ -262,6 +392,42 @@ jobs: name: benchmark-nydus-all-prefetch path: misc/benchmark/${{env.IMAGE}}.csv + benchmark-nydus-all-prefetch-master: + runs-on: ubuntu-latest + needs: [contrib-build-master, nydus-build-master] + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: master + - name: Download Nydus Master + uses: actions/download-artifact@master + with: + name: nydus-artifact-master + path: target/release + - name: Download Nydusify Master + uses: actions/download-artifact@master + with: + name: nydusify-artifact-master + path: contrib/nydusify/cmd + - name: Prepare Nydus Environment + run: | + sudo bash misc/benchmark/prepare_env.sh nydus + sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ + --source ${{env.IMAGE}}:${{env.TAG}} \ + --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ + --fs-version 6 + - name: BenchMark Test + run: | + cd misc/benchmark + sudo python3 benchmark.py --mode nydus-all-prefetch + - name: Save Test Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-nydus-all-prefetch-master + path: misc/benchmark/${{env.IMAGE}}.csv + benchmark-zran-all-prefetch: runs-on: ubuntu-latest needs: [contrib-build, nydus-build] @@ -298,6 +464,45 @@ jobs: name: benchmark-zran-all-prefetch path: misc/benchmark/${{env.IMAGE}}.csv + benchmark-zran-all-prefetch-master: + runs-on: ubuntu-latest + needs: [contrib-build-master, nydus-build-master] + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: master + - name: Download Nydus Master + uses: actions/download-artifact@master + with: + name: nydus-artifact-master + path: target/release + - name: Download Nydusify Master + uses: actions/download-artifact@master + with: + name: nydusify-artifact-master + path: contrib/nydusify/cmd + - name: Prepare Nydus Environment + run: | + sudo bash misc/benchmark/prepare_env.sh nydus + sudo docker pull ${{env.IMAGE}}:${{env.TAG}} && docker tag ${{env.IMAGE}}:${{env.TAG}} localhost:5000/${{env.IMAGE}}:${{env.TAG}} + sudo docker push localhost:5000/${{env.IMAGE}}:${{env.TAG}} + sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ + --source localhost:5000/${{env.IMAGE}}:${{env.TAG}} \ + --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ + --fs-version 6 \ + --oci-ref + - name: BenchMark Test + run: | + cd misc/benchmark + sudo python3 benchmark.py --mode nydus-all-prefetch + - name: Save Test Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-zran-all-prefetch-master + path: misc/benchmark/${{env.IMAGE}}.csv + benchmark-nydus-filelist-prefetch: runs-on: ubuntu-latest needs: [contrib-build, nydus-build] @@ -331,9 +536,61 @@ jobs: name: benchmark-nydus-filelist-prefetch path: misc/benchmark/${{env.IMAGE}}.csv + benchmark-nydus-filelist-prefetch-master: + runs-on: ubuntu-latest + needs: [contrib-build-master, nydus-build-master] + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: master + - name: Download Nydus Master + uses: actions/download-artifact@master + with: + name: nydus-artifact-master + path: target/release + - name: Download Nydusify Master + uses: actions/download-artifact@master + with: + name: nydusify-artifact-master + path: contrib/nydusify/cmd + - name: Prepare Nydus Environment + run: | + sudo bash misc/benchmark/prepare_env.sh nydus + sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ + --source ${{env.IMAGE}}:${{env.TAG}} \ + --target localhost:5000/${{env.IMAGE}}:${{env.TAG}}_nydus \ + --fs-version 6 + - name: BenchMark Test + run: | + cd misc/benchmark + sudo python3 benchmark.py --mode nydus-filelist-prefetch + - name: Save Test Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-nydus-filelist-prefetch-master + path: misc/benchmark/${{env.IMAGE}}.csv + benchmark-result: runs-on: ubuntu-latest needs: [benchmark-oci, benchmark-zran-all-prefetch, benchmark-zran-no-prefetch, benchmark-nydus-no-prefetch, benchmark-nydus-all-prefetch, benchmark-nydus-filelist-prefetch] + if: github.event_name != 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + - uses: geekyeggo/delete-artifact@v2 + with: + name: '*' + - name: Save Result + run: | + sudo python3 misc/benchmark/benchmark_summary.py --mode benchmark-result > $GITHUB_STEP_SUMMARY + + benchmark-compare: + runs-on: ubuntu-latest + needs: [benchmark-oci, benchmark-zran-all-prefetch, benchmark-zran-no-prefetch, benchmark-nydus-no-prefetch, benchmark-nydus-all-prefetch, benchmark-nydus-filelist-prefetch, benchmark-zran-all-prefetch-master, benchmark-zran-no-prefetch-master, benchmark-nydus-no-prefetch-master, benchmark-nydus-all-prefetch-master, benchmark-nydus-filelist-prefetch-master] + if: github.event_name == 'pull_request' steps: - name: Checkout uses: actions/checkout@v3 @@ -343,7 +600,7 @@ jobs: name: '*' - name: Save Result run: | - sudo bash misc/benchmark/benchmark_summary.sh > $GITHUB_STEP_SUMMARY + sudo python3 misc/benchmark/benchmark_summary.py --mode benchmark-compare > $GITHUB_STEP_SUMMARY nydus-unit-test: runs-on: ubuntu-latest