lint-then-benchmark.yml

# Copyright 2022 Democratized Data Foundation
#
# Use of this software is governed by the Business Source License
# included in the file licenses/BSL.txt.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.

name: Lint Then Benchmark Workflow

on:
  pull_request:

  push:
    tags:
      - 'v[0-9]+.[0-9]+.[0-9]+'
    branches:
      - master
      - develop

## These are the permissions for the lint check job.
permissions:
  # Must have a secret (`secrets.ONLY_DEFRADB_REPO_CI_PAT`) setup with the following permissions:
  # - Actions: Read-Write
  # - Administration: Read-Write
  # - Pull Request: Read-Write
  # - Contents: Read-Only
  # - Metadata: Read-Only
  # - Secrets: Read-Only

  # Allow read access to pull request (Required for the `only-new-issues` option.)
  pull-requests: read
  contents: read

env:
  # This is the default benchmark type which if no labels are specified will be used.
  DEFAULT_BENCHMARK_TYPE: SHORT

jobs:
  # ========================================================= Step-1: Run the lint check.
  golangci:
    name: Lint check job

    strategy:
      matrix:
        os: [ubuntu-latest]

    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout code into the directory
        uses: actions/checkout@v4

      # Setting up Go explicitly is required for v3.0.0+ of golangci/golangci-lint-action.
      - name: Setup Go environment explicitly
        uses: actions/setup-go@v5
        with:
          go-version-file: 'go.mod'
          check-latest: true
          cache: false

      - name: Run the golangci-lint
        uses: golangci/golangci-lint-action@v6

        with:
          # Required: the version of golangci-lint is required.
          # Note: The version should not pick the patch version as the latest patch
          #  version is what will always be used.
          version: v1.51

          # Optional: working directory, useful for monorepos or if we wanted to run this
          #  on a non-root directory.
          # working-directory: ./

          # Optional: golangci-lint command line arguments.
          # Note: we can set `--issues-exit-code=0` if we want a successcode always,
          #  indicating that the linter ran successfully (weather or not linter errors
          #  exist or not doesn't matter). But the good think is that the annotations
          #  will still show up. I think this can be useful if we don't want the pipeline
          #  to stop just because we had some linter errors.
          args: --issues-exit-code=1 --config tools/configs/golangci.yaml

          # Optional: we can set the below to `true` if we only want to see newly
          #  introduced linter errors, however I found that in practive that option is a
          #  bit gimmicky, as it passes the linter check despite having new linter errors
          #  in some cases. So we opt in for all annotations of linter errors to show up,
          #  this is actually nicer because we suppress our linter errors manually
          #  anyways so there shouldn't be any linter errors anyways. The enforces us to
          #  always have a clean lint state.
          only-new-issues: false

  # =================== Step-2: Decide what type of benchmarks to run based on label(s).
  # This job acts like a switch to simplify our ci control flow later on.
  decide-benchmark-type:
    name: Decide which benchmarks to run based on flags

    strategy:
      matrix:
        os: [ubuntu-latest]

    runs-on: ${{ matrix.os }}

    outputs:
      # Is either 'NONE', 'FULL', or 'SHORT'.
      benchmark-type: ${{ steps.set-benchmark-type.outputs.type }}

    needs:
      - golangci # only run if the linter check passed.

    steps:
      - name: Check for full benchmark label
        if: contains(github.event.pull_request.labels.*.name, 'action/full-benchmark')
        run: echo "DEFAULT_BENCHMARK_TYPE=FULL" >> ${GITHUB_ENV}

      - name: Check for label that skips the benchmark or non-develop branch
        if: |
          github.event_name == 'pull_request' &&
          github.base_ref != 'develop' ||
          contains(github.event.pull_request.labels.*.name, 'action/no-benchmark') ||
          github.actor == 'dependabot[bot]'
        run: echo "DEFAULT_BENCHMARK_TYPE=NONE" >> ${GITHUB_ENV}

      - name: Run full benchmarks if merged PR (push event) on develop
        if: |
          github.event_name == 'push' &&
          github.ref_name == 'develop'
        run: echo "DEFAULT_BENCHMARK_TYPE=FULL" >> ${GITHUB_ENV}

      - name: Set the output to be the benchmark type
        id: set-benchmark-type
        run: echo "::set-output name=type::${DEFAULT_BENCHMARK_TYPE}"

  # ================== Step-3: Start the runner and get it registered as a github runner.
  start-runner:
    name: Start self-hosted EC2 runner job

    needs:
      - golangci # only run if the linter check passed.
      - decide-benchmark-type # type of benchmark to run.

    if: needs.decide-benchmark-type.outputs.benchmark-type != 'NONE'

    runs-on: ubuntu-latest

    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}

    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      - name: Start EC2 runner
        id: start-ec2-runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: start
          github-token: ${{ secrets.ONLY_DEFRADB_REPO_CI_PAT }}
          ec2-image-id: ${{ secrets.EC2_IMAGE_ID }}
          ec2-instance-type: t3.xlarge
          subnet-id: ${{ secrets.SUBNET_ID }}
          security-group-id: ${{ secrets.SECURITY_GROUP_ID }}
          ## iam-role-name: my-role-name # optional, requires additional permissions
          ## aws-resource-tags: > # optional, requires additional permissions
          ##   [
          ##     {"Key": "Name", "Value": "ec2-github-runner"},
          ##     {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
          ##   ]

  # =========================== Step-4: Run the benchmarks on the runner we just started.
  benchmark-ec2-runner:
    name: Run the benchmarks on the started EC2 runner

    needs:
      - golangci # only run if the linter check passed.
      - decide-benchmark-type # type of benchmark to run.
      - start-runner # required to start the main job when the runner is ready.

    if: needs.decide-benchmark-type.outputs.benchmark-type != 'NONE'

    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner

    env:
      # This is also the same directory as `$GITHUB_WORKSPACE/..`
      HOME: /actions-runner/_work
      GOPATH: /actions-runner/_work/go

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Run the full bechmarking suite
        if: needs.decide-benchmark-type.outputs.benchmark-type == 'FULL'
        run: make test:bench -s | tee current.txt

      - name: Run only the shorter benchmarks
        if: needs.decide-benchmark-type.outputs.benchmark-type == 'SHORT'
        run: make test:bench-short -s | tee current.txt

      # ------------- Upload Report - If pushed on develop branch

      - name: Prepare artifact for uploading for a push on develop
        if: |
          github.event_name == 'push' &&
          github.ref_name == 'develop'
        run: cp current.txt bench-artifact-${{ github.sha }}.txt

      - name: Upload artifact for push event on develop
        if: |
          github.event_name == 'push' &&
          github.ref_name == 'develop'
        uses: actions/upload-artifact@v4
        with:
          name: bench-artifact-${{ github.sha }}
          path: bench-artifact-${{ github.sha }}.txt
          retention-days: 90

      # ------------- Download & Compare - If PR going into develop

      # To find the latest commit on remote develop branch we essentially
      #  just need to do the following:
      # >> git fetch origin develop > /dev/null 2>&1 \
      #    && git rev-parse origin/develop
      # However I wanted to ensure that we get the latest commit on develop
      #  that has a passing `lint-then-benchmark.yml` workflow as well. To
      #  ensure that the artifact was successfully uploaded of the latest
      #  commit that is available on develop. Hence the use of this action.
      - name: Find the last successfull commit's sha on remote of develop
        if: |
          github.event_name == 'pull_request' &&
          github.base_ref == 'develop'
        uses: nrwl/last-successful-commit-action@v1
        id: last_successful_upload_on_develop
        with:
          branch: develop
          workflow_id: lint-then-benchmark.yml
          github_token: ${{ secrets.ONLY_DEFRADB_REPO_CI_PAT }}

      - name: Download the latest benchmark artifact on develop
        if: |
          github.event_name == 'pull_request' &&
          github.base_ref == 'develop'
        uses: dawidd6/action-download-artifact@v6
        with:
          github_token: ${{ secrets.ONLY_DEFRADB_REPO_CI_PAT }}
          workflow: lint-then-benchmark.yml
          branch: develop
          name: bench-artifact-${{ steps.last_successful_upload_on_develop.outputs.commit_hash }}
          repo: ${{ github.repository }}
          check_artifacts: false
          search_artifacts: false

      - name: Prepare benchmark reports for comparisons
        if: |
          github.event_name == 'pull_request' &&
          github.base_ref == 'develop'
        run: >
          make deps:bench &&
          cp bench-artifact-${{ steps.last_successful_upload_on_develop.outputs.commit_hash }}.txt develop.txt &&
          sed -i '/^pkg: /s/^pkg:\ .*\/bench\//pkg:\ /g' develop.txt &&
          sed -i '/^pkg: /s/^pkg:\ .*\/bench\//pkg:\ /g' current.txt

      - name: Run the benchmark comparisons
        if: |
          github.event_name == 'pull_request' &&
          github.base_ref == 'develop'
        run: >
          ${GOPATH}/bin/benchstat -html -alpha 1.1 develop.txt current.txt |
          sed -n "/<body>/,/<\/body>/p" > comparison.html &&
          ./tools/scripts/pretty-benchstat-html.sh comparison.html > pretty-comparison.md

      - name: Comment Benchmark Results on PR
        if: |
          github.event_name == 'pull_request' &&
          github.base_ref == 'develop'
        uses: machine-learning-apps/pr-comment@master
        env:
          GITHUB_TOKEN: ${{ secrets.ONLY_DEFRADB_REPO_CI_PAT }}
        with:
          path: pretty-comparison.md

  # =============================== Step-5: Stop the runner once the benchmarks have ran.
  stop-runner:
    name: Stop self-hosted EC2 runner

    needs:
      - golangci # only run if the linter check passed.
      - decide-benchmark-type # type of benchmark to run.
      - start-runner # required to get output from the start-runner job.
      - benchmark-ec2-runner # required to wait when the main job is done.

    # Stop the runner even if an error happened in the previous jobs. Also ensure that
    #  if the EC2 runner was actually started, only then we stop it.
    if: |
      always() &&
      needs.start-runner.result == 'success' &&
      needs.decide-benchmark-type.outputs.benchmark-type != 'NONE'

    runs-on: ubuntu-latest

    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: stop
          github-token: ${{ secrets.ONLY_DEFRADB_REPO_CI_PAT }}
          label: ${{ needs.start-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}