benchmark-runtime-weights.yml

name: Benchmark runtime weights

on:
  workflow_dispatch:
    inputs:
      rebuild-docker:
        type: boolean
        # if the runtime-benchmarks image should be rebuilt or pulled from hub
        description: rebuild-docker
        required: true
        default: true
      litentry:
        type: boolean
        description: litentry
        required: true
        default: true
      litmus:
        type: boolean
        description: litmus
        required: true
        default: true
      rococo:
        type: boolean
        description: rococo
        required: true
        default: true
      pallets:
        description: pallets to benchmark, * for all, or comma listed (e.g. frame-system,pallet-proxy)
        default: "*"
        required: true

env:
  INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }}  # remote AWS host to run benchmarking
  BENCHMARK_SSH_USER: ${{ secrets.BENCHMARK_SSH_USER }}
  BENCHMARK_SSH_KEYPATH: ${{ secrets.BENCHMARK_SSH_KEYPATH }}
  DOCKER_BUILDKIT: 1

jobs:
  ## build docker image with runtime-benchmarks feature and push it to the hub
  build-docker:
    if: ${{ github.event.inputs.rebuild-docker == 'true' }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout codes on ${{ github.ref }}
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      # try to increase usable memory
      # https://github.com/actions/runner/issues/1051
      - name: Set Swap Space
        uses: pierotofy/set-swap-space@master
        with:
          swap-size-gb: 10

      - name: Build docker image
        run: |
          ./scripts/build-docker.sh production runtime-benchmarks --features=runtime-benchmarks

      - name: Dockerhub login
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}

      - name: Push docker image
        run: |
          docker push litentry/litentry-parachain:runtime-benchmarks

  ## run the benchmarking remotely
  benchmark:
    runs-on: jumphost
    needs: build-docker
    # see https://github.com/actions/runner/issues/491
    if: |
      always() &&
      (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped') 
    steps:
      - name: Set env
        run: |
          chain=""
          if [ "${{ github.event.inputs.litmus }}" = "true" ]; then
            chain="$chain litmus"
          fi
          if [ "${{ github.event.inputs.litentry }}" = "true" ]; then
            chain="$chain litentry"
          fi
          if [ "${{ github.event.inputs.rococo }}" = "true" ]; then
            chain="$chain rococo"
          fi
          if [ "$chain" = "" ]; then
            echo "::error::Please select at least one chain."
            exit 1
          fi
          echo "CHAIN=$chain" >> $GITHUB_ENV

      - name: Checkout codes on ${{ github.ref }}
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Pull docker image
        run: |
          docker pull litentry/litentry-parachain:runtime-benchmarks

      # TODO: maybe use GHA to start/stop remote instance
      - name: Start remote instance
        timeout-minutes: 10
        id: start_instance
        run: |
          aws ec2 start-instances --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }}
          sleep 5
          instance_status="aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].InstanceStatus.Status' --output text"
          system_status="aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].SystemStatus.Status' --output text"
          SECONDS=0
          while : ; do
            if [ "$(eval $instance_status)" = "ok" ] && [ "$(eval $system_status)" = "ok" ]; then
              break
            else
              sleep 20
            fi
          done
          echo "Remote instance reachable now after $SECONDS seconds"
          remote_ip=`aws ec2 describe-instances --region ap-southeast-1 --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text`
          echo "Running instances ip address: $remote_ip"
          echo "remote_ip=$remote_ip" >> $GITHUB_OUTPUT

      # exit status should propagate through ssh
      - name: Remotely benchmark pallets ${{ github.event.inputs.pallets }} for ${{ env.CHAIN }}
        timeout-minutes: 240
        run: |
          # prepend the asterisk with \ to go through ssh
          echo "Running instances ip address: ${{ steps.start_instance.outputs.remote_ip }}"
          arg="${{ github.event.inputs.pallets }}"
          chain="${{ env.CHAIN }}"
          if [ "$arg" = "*" ]; then
            arg="\\$arg";
          fi
          for c in $chain; do
            ssh -x -o StrictHostKeychecking=no ${{ secrets.BENCHMARK_INSTANCE_IP }} -l ${{ env.BENCHMARK_SSH_USER }} 'bash -s' < scripts/benchmark-weight-remote.sh "$c" "${GITHUB_REF#refs/heads/}" "$arg"
            echo "copy generated weights files back ..."
            scp -o StrictHostKeychecking=no "${{ env.BENCHMARK_SSH_USER }}"@"${{ secrets.BENCHMARK_INSTANCE_IP }}":/tmp/litentry-parachain/runtime/$c/src/weights/*.rs runtime/$c/src/weights/
          done
          echo "======================"
          git status

      - name: Stop remote instance
        if: always()
        run: |
          aws ec2 stop-instances --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }}
          sleep 5
          ret=`aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} | jq '.InstanceStatuses[0].InstanceState.Name'`
          echo "Remote instance running state: $ret"

      - name: Create auto PR
        uses: peter-evans/create-pull-request@v3
        with:
          commit-message: "[benchmarking bot] Auto commit generated weights files"
          committer: benchmarking bot <noreply@github.com>
          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
          signoff: false
          branch: benchmarking-bot-${{ github.run_id }}
          delete-branch: true
          title: "[benchmarking bot] Update generated weights files"
          body: |
            This is an automatically created PR.
            It updates the weights files under `runtime/*/src/weights/*.rs` after running benchmarks on the remote machine: ${{ env.INSTANCE_ID }}

            Pallets: "${{ github.event.inputs.pallets }}"
            Chain: "${{ env.CHAIN }}"
            Github action run: https://github.com/litentry/litentry-parachain/actions/runs/${{ github.run_id }}
          labels: |
            automated-pr
          assignees: ${{ github.actor }}
          reviewers: ${{ github.actor }}
          draft: false