benchmarks.yml

on:
  push:
    branches:
      - master
  
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'Pull Request Number'
        required: false
        default: ''

  # note: the "benchmarks please" comments aren't dispatched here,
  # there's a script running on one of our internal servers that reads those and then
  # dispatches to this workflow using the workflow_dispatch there.

name: Benchmarks

env:
  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
  benchmark:
    name: run benchmarks
    runs-on: benchmarks-runner
    steps:
      - name: Enable CPU boost
        run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost

      - name: Checkout sources for a PR
        if: ${{ github.event.inputs.ref }}
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.inputs.ref }}
          fetch-depth: 0

      - name: Checkout sources
        if: github.event.inputs.ref == ''
        uses: actions/checkout@v3
        with:
          fetch-depth: 10

      - name: Set up for PR context
        if: github.event.inputs.pr_number
        run: |
          echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV

      - name: Install stable toolchain
        uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: stable
          target: wasm32-unknown-unknown
          override: true

      - name: Build
        working-directory: crates/bench/
        run: |
          cargo build --release

      - name: Install clippy for module build
        run: |
          rustup component add clippy

      - name: Install latest wasm-opt for module optimisations
        run: |
          curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1

      - name: Disable CPU boost
        run: echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost

      - name: Extract branch name
        if: "! github.event.inputs.pr_number"
        shell: bash
        run: |
          BRANCH_NAME=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
          echo "NORMALIZED_BRANCH_NAME=${BRANCH_NAME//\//-}" >> $GITHUB_ENV

      - name: Branch; run bench
        if: "! github.event.inputs.pr_number"
        run: |
          echo "Running benchmarks with sqlite"
          pushd crates/bench
          cargo bench --bench generic --bench special -- --save-baseline $NORMALIZED_BRANCH_NAME
          cargo run --bin summarize pack $NORMALIZED_BRANCH_NAME
          popd
          mkdir criterion-results
          cp target/criterion/$NORMALIZED_BRANCH_NAME.json criterion-results/
          cp target/criterion/$NORMALIZED_BRANCH_NAME.json criterion-results/$GITHUB_SHA.json

      # TODO: can we optionally download if it only might fail?
      #- name: PR; download bench results for compare
      #  if: github.event.inputs.pr_number
      #  uses: actions/github-script@v6  
      #  with:
      #    github-token: ${{secrets.GITHUB_TOKEN}}
      #    script: |
      #      try {
      #        let artifact = github.rest.actions.getArtifact({
      #          owner: "clockwork",
      #          repo: "SpacetimeDB",
      #          
      #        })
      #      }

      - name: PR; run bench
        if: github.event.inputs.pr_number
        run: |
          echo "Running benchmarks without sqlite"
          # have to pass explicit names, otherwise it will try to run the tests and fail for some reason...
          pushd crates/bench
          cargo bench --bench generic --bench special -- --save-baseline branch '(special|stdb_module|stdb_raw)'
          cargo run --bin summarize pack branch
          popd
          mkdir criterion-results
          cp target/criterion/branch.json criterion-results/pr-$PR_NUMBER.json

      - name: PR; compare benchmarks
        if: github.event.inputs.pr_number
        working-directory: crates/bench/
        run: |
          if [ -e target/criterion/$NORMALIZED_BRANCH_NAME.json ]; then
            cargo run --bin summarize report branch.json $NORMALIZED_BRANCH_NAME.json --report-name report --markdown
          else
            cargo run --bin summarize report branch.json --report-name report --markdown
          fi

      # this will work for both PR and master
      - name: Upload criterion results to DO spaces
        uses: shallwefootball/s3-upload-action@master
        with:
          aws_key_id: ${{ secrets.AWS_KEY_ID }}
          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
          aws_bucket: "spacetimedb-ci-benchmarks"
          source_dir: criterion-results
          endpoint: https://nyc3.digitaloceanspaces.com
          destination_dir: benchmarks

      - name: Fetch markdown summary PR
        if: github.event.inputs.pr_number
        run: |
          curl -sS https://benchmarks.spacetimedb.com/compare/master/pr-$PR_NUMBER > report.md

      - name: Fetch markdown summary PR
        if: "! github.event.inputs.pr_number"
        run: |
          git fetch
          old=$(git rev-parse HEAD~1)
          curl -sS https://benchmarks.spacetimedb.com/compare/$old/$GITHUB_SHA > report.md

      # https://stackoverflow.com/questions/58066966/commenting-a-pull-request-in-a-github-action
      # https://github.com/boa-dev/criterion-compare-action/blob/main/main.js
      - name: test comment
        uses: actions/github-script@v6
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            let stuff = require('fs').readFileSync('report.md', 'utf8');
            let body = `<details><summary>Benchmark results</summary>\n\n${stuff}\n\n</details>`;

            try {
              if (process.env.PR_NUMBER) {
                let number = parseInt(process.env.PR_NUMBER);
                core.info("context: issue number: "+number)
                const { data: comment } = await github.rest.issues.createComment({
                  owner: "clockworklabs",
                  repo: "SpacetimeDB",
                  issue_number: number,
                  body: body,
                });
                core.info(
                  `Created comment id '${comment.id}' on issue '${number}' in 'clockworklabs/SpacetimeDB'.`
                );
                core.setOutput("comment-id", comment.id);
              } else {
                const { data: comment } = github.rest.repos.createCommitComment({
                  commit_sha: context.sha,
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  body: body
                })
                core.info(
                  `Created comment id '${comment.id}' on commit '${context.sha}' in 'clockworklabs/SpacetimeDB'.`
                );
                core.setOutput("comment-id", comment.id);
              }
            } catch (err) {
              core.warning(`Failed to comment: ${err}`);
              core.info("Commenting is not possible from forks.");
              core.info("Logging here instead.");
              console.log(body);
            }


      - name: Clean up
        if: always()
        run: |
          rm -fr /stdb/*