ci: Add ability to control benchmarks by PR labels. (#267)

DESCRIPTION:

This PR enables picking the type of benchmarks we want to run on our first commit (the one that PR is opened with) or controlling the benchmarks we want to run on our next commit on that PR.

USAGE:

The two labels we can use at the moment are: `action/no-benchmark` and `action/full-benchmark`.

If PR has following labels, here is what it will do:

* Label(s) = [`action/no-benchmark`]
> No benchmarks will be ran.

* Label(s) = [`action/full-benchmark`]
> All the benchmarks will be ran.

* Label(s) = [`action/no-benchmark` and `action/full-benchmark`]
> `action/no-benchmark` takes precedence therefore, no benchmarks will be ran.

* Label(s) = [] (no label provided)
> Partial / short benchmarks will be ran.

.github/workflows/lint-then-benchmark.yml

@@ -19,6 +19,11 @@ permissions:
   contents: read
 
 
+env:
+  # This is the default benchmark type which if no labels are specified will be used.
+  DEFAULT_BENCHMARK_TYPE: SHORT
+
+
 jobs:
 
 
@@ -27,7 +32,6 @@ jobs:
     name: Lint Check
     strategy:
       matrix:
-        go-version: [1.17.5]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}
@@ -43,7 +47,7 @@ jobs:
           # Required: the version of golangci-lint is required.
           # Note: The version should not pick the patch version as the latest patch
           #  version is what will always be used.
-          version: v1.43
+          version: v1.44
 
           # Optional: working directory, useful for monorepos or if we wanted to run this
           #  on a non-root directory.
@@ -67,17 +71,54 @@ jobs:
           only-new-issues: false
 
 
-  # ================== Step-2: Start the runner and get it registered as a github runner.
+  # =================== Step-2: Decide what type of benchmarks to run based on label(s).
+  # This job acts like a switch to simplify our ci control flow later on.
+  decide-benchmark-type:
+    name: Deciding which benchmarks to run based on flags.
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    outputs:
+      # Is either 'NONE', 'FULL', or 'SHORT'.
+      benchmark-type: ${{ steps.set-benchmark-type.outputs.type }}
+
+    needs:
+      - golangci # only run if the linter check passed.
+
+    steps:
+      - name: Check for full benchmark label.
+        if: contains(github.event.pull_request.labels.*.name, 'action/full-benchmark')
+        run: echo "DEFAULT_BENCHMARK_TYPE=FULL" >> ${GITHUB_ENV}
+
+      - name: Check for label that skips the benchmark.
+        if: contains(github.event.pull_request.labels.*.name, 'action/no-benchmark')
+        run: echo "DEFAULT_BENCHMARK_TYPE=NONE" >> ${GITHUB_ENV}
+
+      - name: Set the output to be the benchmark type.
+        id: set-benchmark-type
+        run: echo "::set-output name=type::${DEFAULT_BENCHMARK_TYPE}"
+
+
+  # ================== Step-3: Start the runner and get it registered as a github runner.
   start-runner:
     name: Start self-hosted EC2 runner
-    needs: golangci # only run if the linter check passed
+
+    needs:
+      - golangci # only run if the linter check passed.
+      - decide-benchmark-type # type of benchmark to run.
+
+    if: needs.decide-benchmark-type.outputs.benchmark-type != 'NONE'
+
     runs-on: ubuntu-latest
+
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
 
     steps:
-
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -103,12 +144,17 @@ jobs:
           ##   ]
 
 
-  # ============================Step-3: Run the benchmarks on the runner we just started.
+  # =========================== Step-4: Run the benchmarks on the runner we just started.
   benchmark-ec2-runner:
     name: Run the benchmarks on the started EC2 runner
+
     needs:
       - golangci # only run if the linter check passed.
-      - start-runner # required to start the main job when the runner is ready
+      - decide-benchmark-type # type of benchmark to run.
+      - start-runner # required to start the main job when the runner is ready.
+
+    if: needs.decide-benchmark-type.outputs.benchmark-type != 'NONE'
+
     runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
 
     env:
@@ -120,25 +166,35 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Start Running the bechmarks
+      - name: Run the full bechmarking suite
+        if: needs.decide-benchmark-type.outputs.benchmark-type == 'FULL'
         run: make test:bench
 
+      - name: Run only the shorter benchmarks
+        if: needs.decide-benchmark-type.outputs.benchmark-type == 'SHORT'
+        run: make test:bench-short
 
-  # =============================== Step-4: Stop the runner once the benchmarks have ran.
+
+  # =============================== Step-5: Stop the runner once the benchmarks have ran.
   stop-runner:
     name: Stop self-hosted EC2 runner
+
     needs:
       - golangci # only run if the linter check passed.
-      - start-runner # required to get output from the start-runner job
-      - benchmark-ec2-runner # required to wait when the main job is done
-    runs-on: ubuntu-latest
+      - decide-benchmark-type # type of benchmark to run.
+      - start-runner # required to get output from the start-runner job.
+      - benchmark-ec2-runner # required to wait when the main job is done.
 
     # Stop the runner even if an error happened in the previous jobs. Also ensure that
     #  if the EC2 runner was actually started, only then we stop it.
-    if: ${{ always() && (needs.start-runner.result == 'success') }} 
+    if: |
+      always() &&
+      needs.start-runner.result == 'success' &&
+      needs.decide-benchmark-type.outputs.benchmark-type != 'NONE'
 
-    steps:
+    runs-on: ubuntu-latest
 
+    steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:

Makefile

@@ -24,16 +24,16 @@ start: build
 dump: build
 	./build/defradb client dump
 
-.PHONY: deps\:golangci-lint
-deps\:golangci-lint:
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.43.0
+.PHONY: deps\:lint
+deps\:lint:
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.44.0
 
 .PHONY: deps\:go-acc
 deps\:go-acc:
 	go install github.com/ory/go-acc@latest
 
 .PHONY: deps
-deps: deps\:golangci-lint deps\:go-acc
+deps: deps\:lint deps\:go-acc
 	go mod download
 
 .PHONY: tidy