benchmark.yml

name: Benchmark

on:
    workflow_dispatch:
    pull_request:
        types: [opened, synchronize]
        paths-ignore:
            - '**/*.md'

concurrency:
    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
    cancel-in-progress: true

jobs:
    benchmark:
        strategy:
            matrix:
                os: [ubuntu-latest, windows-latest] # `macos-latest` is too unstable to be useful for benchmark, the variance is always huge.
        name: Run benchmark on ${{ matrix.os }}
        runs-on: ${{ matrix.os }}
        steps:
            - name: Checkout
              uses: actions/checkout@v3
              with:
                  ref: master

            - name: Install Rust
              uses: actions-rs/toolchain@v1
              with:
                  profile: minimal
                  toolchain: stable
                  override: true

            - name: Run Bench on Master Branch
              run: |
                  cargo bench --bench codec_benchmarks --features bench -- --save-baseline master
                  cargo bench --bench array_ops_benchmarks --features bench -- --save-baseline master
                  # temp disable due to wired memory allocation issue in linux
                  # i cannot reproduce this issue in mac, but can reproduce in gce and ci
                  # cargo bench --bench map_ops_benchmarks --features bench -- --save-baseline master
                  cargo bench --bench text_ops_benchmarks --features bench -- --save-baseline master
                  cargo bench --bench update_benchmarks --features bench -- --save-baseline master

            - name: Checkout master branch
              uses: actions/checkout@v3
              with:
                  clean: false
                  ref: ${{ github.event.pull_request.head.sha }}

            - name: Run Bench on PR Branch
              run: |
                  cargo bench --bench codec_benchmarks --features bench -- --save-baseline pr
                  cargo bench --bench array_ops_benchmarks --features bench -- --save-baseline pr
                  # cargo bench --bench map_ops_benchmarks --features bench -- --save-baseline pr
                  cargo bench --bench text_ops_benchmarks --features bench -- --save-baseline pr
                  cargo bench --bench update_benchmarks --features bench -- --save-baseline pr

            - name: Upload benchmark results
              uses: actions/upload-artifact@v3
              with:
                  name: benchmark-results-${{ matrix.os }}
                  path: ./target/criterion

    benchmark-compare:
        runs-on: ubuntu-latest
        name: Compare Benchmarks
        needs:
            - benchmark

        steps:
            - name: Install critcmp
              uses: taiki-e/install-action@v2
              with:
                  tool: critcmp

            - name: Linux | Download PR benchmark results
              uses: actions/download-artifact@v3
              with:
                  name: benchmark-results-ubuntu-latest
                  path: ./target/criterion

            - name: Linux | Compare benchmark results
              shell: bash
              run: |
                  echo "## Benchmark Results" >> summary.md
                  echo "### Linux" >> summary.md
                  echo "\`\`\`" >> summary.md
                  critcmp master pr >> summary.md
                  echo "\`\`\`" >> summary.md
                  echo "" >> summary.md

            - name: Linux | Cleanup benchmark results
              run: rm -rf ./target/criterion

            - name: Windows | Download PR benchmark results
              uses: actions/download-artifact@v3
              with:
                  name: benchmark-results-windows-latest
                  path: ./target/criterion

            - name: Windows | Compare benchmark results
              shell: bash
              run: |
                  echo "### Windows" >> summary.md
                  echo "\`\`\`" >> summary.md
                  critcmp master pr >> summary.md
                  echo "\`\`\`" >> summary.md
                  echo "" >> summary.md
                  cat summary.md > $GITHUB_STEP_SUMMARY

            - name: Find Comment
              # Check if the event is not triggered by a fork
              if: github.event.pull_request.head.repo.full_name == github.repository
              uses: peter-evans/find-comment@v2
              id: fc
              with:
                  issue-number: ${{ github.event.pull_request.number }}
                  comment-author: 'github-actions[bot]'
                  body-includes: Benchmark Results

            - name: Create or update comment
              # Check if the event is not triggered by a fork
              if: github.event.pull_request.head.repo.full_name == github.repository
              uses: peter-evans/create-or-update-comment@v2
              with:
                  issue-number: ${{ github.event.pull_request.number }}
                  edit-mode: replace
                  comment-id: ${{ steps.fc.outputs.comment-id }}
                  body-file: summary.md