Benchmarks #339
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| push: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: 'Pull Request Number' | |
| required: false | |
| default: '' | |
| name: Benchmarks | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| benchmark: | |
| name: run benchmarks | |
| runs-on: self-hosted | |
| steps: | |
| - name: Checkout sources for a PR | |
| if: ${{ github.event.inputs.ref }} | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.inputs.ref }} | |
| fetch-depth: 0 | |
| - name: Checkout sources | |
| if: github.event.inputs.ref == '' | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 10 | |
| - name: Set up for PR context | |
| if: github.event.inputs.pr_number | |
| run: | | |
| echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV | |
| PR_DATA=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr_number }} --jq '{ baseRefName: .base.ref, headRefName: .head.ref }') | |
| echo "PR_BASE_REF=$(echo $PR_DATA | jq -r '.baseRefName')" >> $GITHUB_ENV | |
| echo "PR_HEAD_REF=$(echo $PR_DATA | jq -r '.headRefName')" >> $GITHUB_ENV | |
| # https://stackoverflow.com/questions/58066966/commenting-a-pull-request-in-a-github-action | |
| # https://github.com/boa-dev/criterion-compare-action/blob/main/main.js | |
| - name: test comment | |
| if: ${{ env.PR_BASE_REF }} | |
| uses: actions/github-script@v6 | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| let stuff = require('fs').readFileSync('crates/bench/clippy.toml', 'utf8'); | |
| let body = `Test comment: clippy.toml contents: '${stuff}'`; | |
| let number = parseInt(process.env.PR_NUMBER); | |
| core.info("context: issue number: "+number) | |
| try { | |
| const { data: comment } = await github.rest.issues.createComment({ | |
| owner: "clockworklabs", | |
| repo: "SpacetimeDB", | |
| issue_number: number, | |
| body: body, | |
| }); | |
| core.info( | |
| `Created comment id '${comment.id}' on issue '${contextObj.number}' in '${contextObj.repo}'.` | |
| ); | |
| core.setOutput("comment-id", comment.id); | |
| } catch (err) { | |
| core.warning(`Failed to comment: ${err}`); | |
| core.info("Commenting is not possible from forks."); | |
| core.info("Logging here instead."); | |
| console.log(body); | |
| } | |
| - name: Install stable toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: stable | |
| target: wasm32-unknown-unknown | |
| override: true | |
| - name: ⚡ Cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Build | |
| working-directory: crates/bench/ | |
| run: | | |
| cargo build --release | |
| - name: Install clippy for module build | |
| run: | | |
| rustup component add clippy | |
| - name: Master; run bench | |
| if: github.ref == 'ref/head/master' | |
| working-directory: crates/bench/ | |
| run: | | |
| echo "Running benchmarks with sqlite" | |
| cargo bench --bench generic --bench special -- --save-baseline master | |
| cargo run --bin summarize pack master | |
| # https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts | |
| - name: Master; upload packed bench results | |
| if: github.ref == 'ref/head/master' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: master_packed_bench_json-${{ github.sha }} | |
| path: target/criterion/master.json | |
| # TODO: can we optionally download if it only might fail? | |
| #- name: PR; download bench results for compare | |
| # if: ${{ env.PR_BASE_REF }} | |
| # uses: actions/github-script@v6 | |
| # with: | |
| # github-token: ${{secrets.GITHUB_TOKEN}} | |
| # script: | | |
| # try { | |
| # let artifact = github.rest.actions.getArtifact({ | |
| # owner: "clockwork", | |
| # repo: context. | |
| # }) | |
| # } | |
| - name: PR; run bench | |
| if: ${{ env.PR_BASE_REF }} | |
| working-directory: crates/bench/ | |
| run: | | |
| echo "Running benchmarks without sqlite" | |
| # have to pass explicit names, otherwise it will try to run the tests and fail for some reason... | |
| cargo bench --bench generic --bench special -- --save-baseline branch '(special|stdb_module|stdb_raw)' | |
| cargo bench -- --save-baseline branch '(special|stdb_module|stdb_raw)' | |
| cargo run --bin summarize pack branch | |
| - name: PR; compare benchmarks | |
| if: ${{ env.PR_BASE_REF }} | |
| working-directory: crates/bench/ | |
| run: | | |
| if [ -e target/criterion/master.json ]; then | |
| cargo run --bin summarize markdown-report branch.json master.json --report-name report | |
| else | |
| cargo run --bin summarize markdown-report branch.json --report-name report | |
| fi | |
| # https://stackoverflow.com/questions/58066966/commenting-a-pull-request-in-a-github-action | |
| # https://github.com/boa-dev/criterion-compare-action/blob/main/main.js | |
| - name: test comment | |
| if: ${{ env.PR_BASE_REF }} | |
| uses: actions/github-script@v6 | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| let stuff = require('fs').readFileSync('target/criterion/report.md', 'utf8'); | |
| let body = `<details><summary>Benchmark results</summary>\n\n{stuff}\n\n</details>`; | |
| let number = parseInt(process.env.PR_NUMBER); | |
| core.info("context: issue number: "+number) | |
| try { | |
| const { data: comment } = await github.rest.issues.createComment({ | |
| owner: "clockworklabs", | |
| repo: "SpacetimeDB", | |
| issue_number: number, | |
| body: body, | |
| }); | |
| core.info( | |
| `Created comment id '${comment.id}' on issue '${contextObj.number}' in '${contextObj.repo}'.` | |
| ); | |
| core.setOutput("comment-id", comment.id); | |
| } catch (err) { | |
| core.warning(`Failed to comment: ${err}`); | |
| core.info("Commenting is not possible from forks."); | |
| core.info("Logging here instead."); | |
| console.log(body); | |
| } | |
| - name: Clean up | |
| if: always() | |
| run: | | |
| rm -fr /stdb/* | |