bench.yml

name: Benchmark

# We use environments to require approval to run benchmarks on PRs, but not on pushes to `main`
# (which have been approved already since PRs are required for `main`).
on:
  workflow_call:
    inputs:
      environment:
        type: string
      publish:
        type: boolean
        default: false
      ref:
        required: true
        type: string
      s3_bench_results_prefix:
        type: string

env:
  RUST_BACKTRACE: 1
  CARGO_TERM_COLOR: always
  S3_BUCKET_NAME: ${{ vars.S3_BENCH_BUCKET_NAME }}
  S3_BUCKET_TEST_PREFIX: ${{ vars.S3_BUCKET_BENCH_PREFIX || 'mountpoint-benchmark/' }}
  S3_BUCKET_BENCH_FILE: ${{ vars.BENCH_FILE_NAME || 'bench100GB.bin' }}
  S3_BUCKET_SMALL_BENCH_FILE: ${{ vars.SMALL_BENCH_FILE_NAME || 'bench5MB.bin' }}
  # Optional endpoint url, can be empty
  S3_ENDPOINT_URL: ${{ vars.S3_BENCH_ENDPOINT_URL }}
  S3_REGION: ${{ vars.S3_BENCH_REGION }}

jobs:
  bench:
    name: Benchmark (Throughput)
    runs-on: [self-hosted, linux, x64, nvme-high-performance]

    environment: ${{ inputs.environment }}

    steps:
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ vars.ACTIONS_BENCH_IAM_ROLE }}
        aws-region: ${{ vars.S3_BENCH_REGION }}
        role-duration-seconds: 21600
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        ref: ${{ inputs.ref }}
        submodules: true
        persist-credentials: false
    - name: Install operating system dependencies
      uses: ./.github/actions/install-dependencies
      with:
        fuseVersion: 2
        libunwind: true
        fio: true
    - name: Set up Rust toolchain
      uses: actions-rust-lang/setup-rust-toolchain@v1
      with:
        # setup-rust-toolchain sets "-D warnings" by default, and Rust treats any warning as compile error.
        # We need to this currently because `vendor/fuser` contains some warnings and it breaks the build.
        rustflags: ""
    - name: Build
      run: cargo build --release
    - name: Run Benchmark
      run: mountpoint-s3/scripts/fs_bench.sh
    - name: Save benchmark results in S3
      if: inputs.s3_bench_results_prefix
      run: .github/actions/scripts/save-benchmark-results.sh
      env:
        S3_BENCH_REGION: ${{ vars.S3_BENCH_REGION }}
        S3_BENCH_BUCKET_NAME: ${{ vars.S3_BENCH_RESULTS_BUCKET_NAME }}
        S3_BENCH_RESULTS_PREFIX: ${{ inputs.s3_bench_results_prefix }}/${{ github.job }}
        COMMIT_ID: ${{ inputs.ref }}
    - name: Check benchmark results
      uses: benchmark-action/github-action-benchmark@v1
      with:
        name: Throughput Benchmark (S3 Standard)
        tool: 'customBiggerIsBetter'
        output-file-path: results/output.json
        # This is default value, just make it more explicit
        benchmark-data-dir-path: dev/bench
        alert-threshold: "200%"
        fail-on-alert: true
        # GitHub API token to make a commit comment
        github-token: ${{ secrets.GITHUB_TOKEN }}
        # Store the results and deploy GitHub pages automatically if the results are from main branch
        auto-push: ${{ inputs.publish }}
        comment-on-alert: true
        max-items-in-chart: 20
        summary-always: true
    - name: Check resource utilization
      uses: benchmark-action/github-action-benchmark@v1
      with:
        name: Throughput Benchmark - Peak Memory Usage (S3 Standard)
        tool: 'customSmallerIsBetter'
        output-file-path: results/peak_mem_usage.json
        benchmark-data-dir-path: dev/bench/peak_mem_usage
        alert-threshold: "200%"
        fail-on-alert: false
        # GitHub API token to make a commit comment
        github-token: ${{ secrets.GITHUB_TOKEN }}
        # Store the results and deploy GitHub pages automatically if the results are from main branch
        auto-push: ${{ inputs.publish }}
        comment-on-alert: false
        max-items-in-chart: 20
        skip-fetch-gh-pages: true
        summary-always: true

  latency-bench:
    name: Benchmark (Latency)
    runs-on: [self-hosted, linux, x64, nvme-high-performance]

    environment: ${{ inputs.environment }}

    steps:
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ vars.ACTIONS_BENCH_IAM_ROLE }}
        aws-region: ${{ vars.S3_BENCH_REGION }}
        role-duration-seconds: 21600
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        ref: ${{ inputs.ref }}
        submodules: true
        persist-credentials: false
    - name: Install operating system dependencies
      uses: ./.github/actions/install-dependencies
      with:
        fuseVersion: 2
        libunwind: true
        fio: true
    - name: Set up Rust toolchain
      uses: actions-rust-lang/setup-rust-toolchain@v1
      with:
        # setup-rust-toolchain sets "-D warnings" by default, and Rust treats any warning as compile error.
        # We need to this currently because `vendor/fuser` contains some warnings and it breaks the build.
        rustflags: ""
    - name: Build
      run: cargo build --release
    - name: Run Benchmark
      run: mountpoint-s3/scripts/fs_latency_bench.sh
    - name: Save benchmark results in S3
      if: inputs.s3_bench_results_prefix
      run: .github/actions/scripts/save-benchmark-results.sh
      env:
        S3_BENCH_REGION: ${{ vars.S3_BENCH_REGION }}
        S3_BENCH_BUCKET_NAME: ${{ vars.S3_BENCH_RESULTS_BUCKET_NAME }}
        S3_BENCH_RESULTS_PREFIX: ${{ inputs.s3_bench_results_prefix }}/${{ github.job }}
        COMMIT_ID: ${{ inputs.ref }}
    - name: Check benchmark results
      uses: benchmark-action/github-action-benchmark@v1
      with:
        name: Latency Benchmark (S3 Standard)
        tool: 'customSmallerIsBetter'
        output-file-path: results/output.json
        benchmark-data-dir-path: dev/latency_bench
        alert-threshold: "200%"
        fail-on-alert: true
        # GitHub API token to make a commit comment
        github-token: ${{ secrets.GITHUB_TOKEN }}
        # Store the results and deploy GitHub pages automatically if the results are from main branch
        auto-push: ${{ inputs.publish }}
        comment-on-alert: false
        max-items-in-chart: 20
        summary-always: true

  cache-bench:
    name: Benchmark (Cache)
    runs-on: [self-hosted, linux, x64, nvme-high-performance]

    environment: ${{ inputs.environment }}

    steps:
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ vars.ACTIONS_BENCH_IAM_ROLE }}
        aws-region: ${{ vars.S3_BENCH_REGION }}
        role-duration-seconds: 21600
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        ref: ${{ inputs.ref }}
        submodules: true
        persist-credentials: false
    - name: Install operating system dependencies
      uses: ./.github/actions/install-dependencies
      with:
        fuseVersion: 2
        libunwind: true
        fio: true
    - name: Set up Rust toolchain
      uses: actions-rust-lang/setup-rust-toolchain@v1
      with:
        # setup-rust-toolchain sets "-D warnings" by default, and Rust treats any warning as compile error.
        # We need to this currently because `vendor/fuser` contains some warnings and it breaks the build.
        rustflags: ""
    - name: Build
      run: cargo build --release
    - name: Run Benchmark
      env:
        S3_MOUNT_LOCAL_STORAGE: yes
      run: mountpoint-s3/scripts/fs_cache_bench.sh
    - name: Save benchmark results in S3
      if: inputs.s3_bench_results_prefix
      run: .github/actions/scripts/save-benchmark-results.sh
      env:
        S3_BENCH_REGION: ${{ vars.S3_BENCH_REGION }}
        S3_BENCH_BUCKET_NAME: ${{ vars.S3_BENCH_RESULTS_BUCKET_NAME }}
        S3_BENCH_RESULTS_PREFIX: ${{ inputs.s3_bench_results_prefix }}/${{ github.job }}
        COMMIT_ID: ${{ inputs.ref }}
    - name: Check benchmark results
      uses: benchmark-action/github-action-benchmark@v1
      with:
        name: Cache Throughput Benchmark (S3 Standard)
        tool: 'customBiggerIsBetter'
        output-file-path: results/output.json
        benchmark-data-dir-path: dev/cache_bench
        alert-threshold: "200%"
        fail-on-alert: true
        # GitHub API token to make a commit comment
        github-token: ${{ secrets.GITHUB_TOKEN }}
        # Store the results and deploy GitHub pages automatically if the results are from main branch
        auto-push: ${{ inputs.publish }}
        comment-on-alert: false
        max-items-in-chart: 20
        summary-always: true
    - name: Check resource utilization
      uses: benchmark-action/github-action-benchmark@v1
      with:
        name: Cache Throughput Benchmark - Peak Memory Usage (S3 Standard)
        tool: 'customSmallerIsBetter'
        output-file-path: results/peak_mem_usage.json
        benchmark-data-dir-path: dev/cache_bench/peak_mem_usage
        alert-threshold: "200%"
        fail-on-alert: false
        # GitHub API token to make a commit comment
        github-token: ${{ secrets.GITHUB_TOKEN }}
        # Store the results and deploy GitHub pages automatically if the results are from main branch
        auto-push: ${{ inputs.publish }}
        comment-on-alert: false
        max-items-in-chart: 20
        skip-fetch-gh-pages: true
        summary-always: true