Fix out of order subscription updates (#556) #612
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| push: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: 'Pull Request Number' | |
| required: false | |
| default: '' | |
| issue_comment: | |
| types: [created] | |
| name: Benchmarks | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| benchmark: | |
| name: run benchmarks | |
| runs-on: benchmarks-runner | |
| # filter for a comment containing 'benchmarks please' | |
| if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }} | |
| env: | |
| PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }} | |
| steps: | |
| - name: Enable CPU boost | |
| run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost | |
| - name: Check membership | |
| if: ${{ github.event_name == 'issue_comment' }} | |
| env: | |
| CONTRIB_ORG: clockworklabs | |
| COMMENT_AUTHOR: ${{ github.event.comment.user.login }} | |
| ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }} | |
| run: | | |
| curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb | |
| if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then | |
| gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks." | |
| exit 1 | |
| fi | |
| - name: Post initial comment | |
| run: | | |
| if [[ $PR_NUMBER ]]; then | |
| comment_parent=issues/$PR_NUMBER | |
| comment_update=issues/comments | |
| else | |
| comment_parent=commits/$GITHUB_SHA | |
| comment_update=comments | |
| fi | |
| comment_body="Benchmark in progress..." | |
| comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id) | |
| echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV | |
| - name: find PR branch | |
| if: ${{ env.PR_NUMBER }} | |
| run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV" | |
| - name: Checkout sources | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ env.PR_REF || github.ref }} | |
| # if we're on master we want to know what the sha of HEAD~1 is so | |
| # that we can compare results from it to HEAD (in the "Fetch markdown | |
| # summary PR" step). otherwise, we can use a fully shallow checkout | |
| fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }} | |
| - name: Install stable toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| components: clippy | |
| toolchain: stable | |
| target: wasm32-unknown-unknown | |
| override: true | |
| - name: Build | |
| working-directory: crates/bench/ | |
| run: | | |
| cargo build --release | |
| - name: Install latest wasm-opt for module optimisations | |
| run: | | |
| curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1 | |
| - name: Disable CPU boost | |
| run: echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost | |
| - name: Branch; run bench | |
| run: | | |
| if [[ $PR_NUMBER ]]; then | |
| BASELINE_NAME=branch | |
| RESULTS_NAME=pr-$PR_NUMBER | |
| BENCH_FILTER='(special|stdb_module|stdb_raw)' | |
| echo "Running benchmarks without sqlite" | |
| else | |
| BASELINE_NAME=master | |
| RESULTS_NAME=$GITHUB_SHA | |
| BENCH_FILTER='.*' | |
| echo "Running benchmarks with sqlite" | |
| fi | |
| pushd crates/bench | |
| cargo bench --bench generic --bench special -- --save-baseline "$BASELINE_NAME" "$BENCH_FILTER" | |
| cargo run --bin summarize pack "$BASELINE_NAME" | |
| popd | |
| mkdir criterion-results | |
| [[ ! $PR_NUMBER ]] && cp target/criterion/$BASELINE_NAME.json criterion-results/ | |
| cp target/criterion/$BASELINE_NAME.json criterion-results/$RESULTS_NAME.json | |
| # TODO: can we optionally download if it only might fail? | |
| #- name: PR; download bench results for compare | |
| # if: env.PR_NUMBER | |
| # uses: actions/github-script@v6 | |
| # with: | |
| # github-token: ${{secrets.GITHUB_TOKEN}} | |
| # script: | | |
| # try { | |
| # let artifact = github.rest.actions.getArtifact({ | |
| # owner: "clockwork", | |
| # repo: "SpacetimeDB", | |
| # | |
| # }) | |
| # } | |
| - name: PR; compare benchmarks | |
| if: ${{ env.PR_NUMBER }} | |
| working-directory: crates/bench/ | |
| run: | | |
| cargo run --bin summarize markdown-report branch.json --report-name report | |
| # this will work for both PR and master | |
| - name: Upload criterion results to DO spaces | |
| uses: shallwefootball/s3-upload-action@master | |
| with: | |
| aws_key_id: ${{ secrets.AWS_KEY_ID }} | |
| aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}} | |
| aws_bucket: "spacetimedb-ci-benchmarks" | |
| source_dir: criterion-results | |
| endpoint: https://nyc3.digitaloceanspaces.com | |
| destination_dir: benchmarks | |
| - name: Fetch markdown summary PR | |
| run: | | |
| if [[ $PR_NUMBER ]]; then | |
| OLD=master | |
| NEW=pr-$PR_NUMBER | |
| else | |
| OLD=$(git rev-parse HEAD~1) | |
| NEW=$GITHUB_SHA | |
| fi | |
| curl -sS https://benchmarks.spacetimedb.com/compare/$OLD/$NEW > report.md | |
| - name: Post comment | |
| run: | | |
| BODY="<details><summary>Benchmark results</summary> | |
| $(cat report.md) | |
| </details>" | |
| gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY" | |
| - name: Post failure comment | |
| if: ${{ failure() && env.COMMENT_UPDATE_URL }} | |
| run: | | |
| BODY="Benchmarking failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details." | |
| gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY" | |
| - name: Clean up | |
| if: always() | |
| run: | | |
| rm -fr /stdb/* |