Benchmarks #454
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| push: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: 'Pull Request Number' | |
| required: false | |
| default: '' | |
| # note: the "benchmarks please" comments aren't dispatched here, | |
| # there's a script running on one of our internal servers that reads those and then | |
| # dispatches to this workflow using the workflow_dispatch there. | |
| name: Benchmarks | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| benchmark: | |
| name: run benchmarks | |
| runs-on: benchmarks-runner | |
| steps: | |
| - name: Enable CPU boost | |
| run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost | |
| - name: Checkout sources for a PR | |
| if: ${{ github.event.inputs.ref }} | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.inputs.ref }} | |
| fetch-depth: 0 | |
| - name: Checkout sources | |
| if: github.event.inputs.ref == '' | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 10 | |
| - name: Set up for PR context | |
| if: github.event.inputs.pr_number | |
| run: | | |
| echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV | |
| - name: Install stable toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: stable | |
| target: wasm32-unknown-unknown | |
| override: true | |
| - name: Build | |
| working-directory: crates/bench/ | |
| run: | | |
| cargo build --release | |
| - name: Install clippy for module build | |
| run: | | |
| rustup component add clippy | |
| - name: Install latest wasm-opt for module optimisations | |
| run: | | |
| curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1 | |
| - name: Disable CPU boost | |
| run: echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost | |
| - name: Extract branch name | |
| if: "! github.event.inputs.pr_number" | |
| shell: bash | |
| run: | | |
| BRANCH_NAME=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | |
| echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV | |
| echo "NORMALIZED_BRANCH_NAME=${BRANCH_NAME//\//-}" >> $GITHUB_ENV | |
| - name: Branch; run bench | |
| if: "! github.event.inputs.pr_number" | |
| run: | | |
| echo "Running benchmarks with sqlite" | |
| pushd crates/bench | |
| cargo bench --bench generic --bench special -- --save-baseline $NORMALIZED_BRANCH_NAME | |
| cargo run --bin summarize pack $NORMALIZED_BRANCH_NAME | |
| popd | |
| mkdir criterion-results | |
| cp target/criterion/$NORMALIZED_BRANCH_NAME.json criterion-results/ | |
| cp target/criterion/$NORMALIZED_BRANCH_NAME.json criterion-results/$GITHUB_SHA.json | |
| # TODO: can we optionally download if it only might fail? | |
| #- name: PR; download bench results for compare | |
| # if: github.event.inputs.pr_number | |
| # uses: actions/github-script@v6 | |
| # with: | |
| # github-token: ${{secrets.GITHUB_TOKEN}} | |
| # script: | | |
| # try { | |
| # let artifact = github.rest.actions.getArtifact({ | |
| # owner: "clockwork", | |
| # repo: "SpacetimeDB", | |
| # | |
| # }) | |
| # } | |
| - name: PR; run bench | |
| if: github.event.inputs.pr_number | |
| run: | | |
| echo "Running benchmarks without sqlite" | |
| # have to pass explicit names, otherwise it will try to run the tests and fail for some reason... | |
| pushd crates/bench | |
| cargo bench --bench generic --bench special -- --save-baseline branch '(special|stdb_module|stdb_raw)' | |
| cargo run --bin summarize pack branch | |
| popd | |
| mkdir criterion-results | |
| cp target/criterion/branch.json criterion-results/pr-$PR_NUMBER.json | |
| - name: PR; compare benchmarks | |
| if: github.event.inputs.pr_number | |
| working-directory: crates/bench/ | |
| run: | | |
| if [ -e target/criterion/$NORMALIZED_BRANCH_NAME.json ]; then | |
| cargo run --bin summarize markdown-report branch.json $NORMALIZED_BRANCH_NAME.json --report-name report | |
| else | |
| cargo run --bin summarize markdown-report branch.json --report-name report | |
| fi | |
| # this will work for both PR and master | |
| - name: Upload criterion results to DO spaces | |
| uses: shallwefootball/s3-upload-action@master | |
| with: | |
| aws_key_id: ${{ secrets.AWS_KEY_ID }} | |
| aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}} | |
| aws_bucket: "spacetimedb-ci-benchmarks" | |
| source_dir: criterion-results | |
| endpoint: https://nyc3.digitaloceanspaces.com | |
| destination_dir: benchmarks | |
| - name: Fetch markdown summary PR | |
| if: github.event.inputs.pr_number | |
| run: | | |
| curl -sS https://benchmarks.spacetimedb.com/compare/master/pr-$PR_NUMBER > report.md | |
| - name: Fetch markdown summary PR | |
| if: "! github.event.inputs.pr_number" | |
| run: | | |
| git fetch | |
| old=$(git rev-parse HEAD~1) | |
| curl -sS https://benchmarks.spacetimedb.com/compare/$old/$GITHUB_SHA > report.md | |
| # https://stackoverflow.com/questions/58066966/commenting-a-pull-request-in-a-github-action | |
| # https://github.com/boa-dev/criterion-compare-action/blob/main/main.js | |
| - name: test comment | |
| uses: actions/github-script@v6 | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| let stuff = require('fs').readFileSync('report.md', 'utf8'); | |
| let body = `<details><summary>Benchmark results</summary>\n\n${stuff}\n\n</details>`; | |
| try { | |
| if (process.env.PR_NUMBER) { | |
| let number = parseInt(process.env.PR_NUMBER); | |
| core.info("context: issue number: "+number) | |
| const { data: comment } = await github.rest.issues.createComment({ | |
| owner: "clockworklabs", | |
| repo: "SpacetimeDB", | |
| issue_number: number, | |
| body: body, | |
| }); | |
| core.info( | |
| `Created comment id '${comment.id}' on issue '${number}' in 'clockworklabs/SpacetimeDB'.` | |
| ); | |
| core.setOutput("comment-id", comment.id); | |
| } else { | |
| const { data: comment } = github.rest.repos.createCommitComment({ | |
| commit_sha: context.sha, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: body | |
| }) | |
| core.info( | |
| `Created comment id '${comment.id}' on commit '${context.sha}' in 'clockworklabs/SpacetimeDB'.` | |
| ); | |
| core.setOutput("comment-id", comment.id); | |
| } | |
| } catch (err) { | |
| core.warning(`Failed to comment: ${err}`); | |
| core.info("Commenting is not possible from forks."); | |
| core.info("Logging here instead."); | |
| console.log(body); | |
| } | |
| - name: Clean up | |
| if: always() | |
| run: | | |
| rm -fr /stdb/* |