ci: Add ability to control types of benchmarks we want to run using

labels.

.github/workflows/lint-then-benchmark.yml

@@ -19,6 +19,11 @@ permissions:
   contents: read
 
 
+env:
+  # This is the default benchmark type which if no labels are specified will be used.
+  DEFAULT_BENCHMARK_TYPE: SHORT
+
+
 jobs:
 
 
@@ -27,7 +32,6 @@ jobs:
     name: Lint Check
     strategy:
       matrix:
-        go-version: [1.17.5]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}
@@ -43,7 +47,7 @@ jobs:
           # Required: the version of golangci-lint is required.
           # Note: The version should not pick the patch version as the latest patch
           #  version is what will always be used.
-          version: v1.43
+          version: v1.44
 
           # Optional: working directory, useful for monorepos or if we wanted to run this
           #  on a non-root directory.
@@ -67,17 +71,54 @@ jobs:
           only-new-issues: false
 
 
-  # ================== Step-2: Start the runner and get it registered as a github runner.
+  # =================== Step-2: Decide what type of benchmarks to runaa based on label(s).
+  # This job acts like a switch to simplify our ci control flow later on.
+  decide-benchmark-type:
+    name: Deciding which benchmarks to run based on flags.
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    outputs:
+      # Is either 'NONE', 'FULL', or 'SHORT'.
+      benchmark-type: ${{ steps.set-benchmark-type.outputs.type }}
+
+    needs:
+      - golangci # only run if the linter check passed.
+
+    steps:
+      - name: Check for full benchmark label.
+        if: contains(github.event.pull_request.labels.*.name, 'action/full-benchmark')
+        run: echo "DEFAULT_BENCHMARK_TYPE=FULL" >> ${GITHUB_ENV}
+
+      - name: Check for label that skips the benchmark.
+        if: contains(github.event.pull_request.labels.*.name, 'action/no-benchmark')
+        run: echo "DEFAULT_BENCHMARK_TYPE=NONE" >> ${GITHUB_ENV}
+
+      - name: Set the output to be the benchmark type.
+        id: set-benchmark-type
+        run: echo "::set-output name=type::${DEFAULT_BENCHMARK_TYPE}"
+
+
+  # ================== Step-3: Start the runner and get it registered as a github runner.
   start-runner:
     name: Start self-hosted EC2 runner
-    needs: golangci # only run if the linter check passed
+
+    needs:
+      - golangci # only run if the linter check passed.
+      - decide-benchmark-type # type of benchmark to run.
+
+    if: needs.decide-benchmark-type.outputs.benchmark-type != 'NONE'
+
     runs-on: ubuntu-latest
+
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
 
     steps:
-
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -103,12 +144,17 @@ jobs:
           ##   ]
 
 
-  # ============================Step-3: Run the benchmarks on the runner we just started.
+  # =========================== Step-4: Run the benchmarks on the runner we just started.
   benchmark-ec2-runner:
     name: Run the benchmarks on the started EC2 runner
+
     needs:
       - golangci # only run if the linter check passed.
-      - start-runner # required to start the main job when the runner is ready
+      - decide-benchmark-type # type of benchmark to run.
+      - start-runner # required to start the main job when the runner is ready.
+
+    if: needs.decide-benchmark-type.outputs.benchmark-type != 'NONE'
+
     runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
 
     env:
@@ -120,25 +166,35 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Start Running the bechmarks
+      - name: Run the full bechmarking suite
+        if: needs.decide-benchmark-type.outputs.benchmark-type == 'FULL'
         run: make test:bench
 
+      - name: Run only the shorter benchmarks
+        if: needs.decide-benchmark-type.outputs.benchmark-type == 'SHORT'
+        run: make test:bench-short
 
-  # =============================== Step-4: Stop the runner once the benchmarks have ran.
+
+  # =============================== Step-5: Stop the runner once the benchmarks have ran.
   stop-runner:
     name: Stop self-hosted EC2 runner
+
     needs:
       - golangci # only run if the linter check passed.
-      - start-runner # required to get output from the start-runner job
-      - benchmark-ec2-runner # required to wait when the main job is done
-    runs-on: ubuntu-latest
+      - decide-benchmark-type # type of benchmark to run.
+      - start-runner # required to get output from the start-runner job.
+      - benchmark-ec2-runner # required to wait when the main job is done.
 
     # Stop the runner even if an error happened in the previous jobs. Also ensure that
     #  if the EC2 runner was actually started, only then we stop it.
-    if: ${{ always() && (needs.start-runner.result == 'success') }} 
+    if: |
+      always() &&
+      needs.start-runner.result == 'success' &&
+      needs.decide-benchmark-type.outputs.benchmark-type != 'NONE'
 
-    steps:
+    runs-on: ubuntu-latest
 
+    steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:

Makefile

@@ -24,16 +24,16 @@ start: build
 dump: build
 	./build/defradb client dump
 
-.PHONY: deps\:golangci-lint
-deps\:golangci-lint:
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.43.0
+.PHONY: deps\:lint
+deps\:lint:
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.44.0
 
 .PHONY: deps\:go-acc
 deps\:go-acc:
 	go install github.com/ory/go-acc@latest
 
 .PHONY: deps
-deps: deps\:golangci-lint deps\:go-acc
+deps: deps\:lint deps\:go-acc
 	go mod download
 
 .PHONY: tidy