ci(integration): Execute backend integration tests in parallel

.gitlab-ci.yml

@@ -80,18 +80,16 @@ default:
     - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
       alias: docker
   variables:
-    DOCKER_BUILDARGS: "--push"
+    DOCKER_BUILDARGS: "--push --cache-to=$CI_REGISTRY_IMAGE:cache --cache-from=$CI_REGISTRY_IMAGE:cache"
   before_script:
     - apk add make bash git
     - *dind-login
     # NOTE: If we're running on a PR, do not build multiplatform
-    - test "$CI_COMMIT_REF_PROTECTED" != "true" && unset DOCKER_PLATFORM
-    - if test -n "${DOCKER_PLATFORM}"; then
-      docker context create ci;
-      docker builder create --name ci-builder ci;
-      export DOCKER_BUILDARGS="${DOCKER_BUILDARGS} --builder=ci-builder";
-      unset DOCKER_HOST;
-      fi
+    - test "$CI_COMMIT_REF_PROTECTED" != "true" && export DOCKER_PLATFORM=linux/amd64
+    - docker context create ci;
+    - docker builder create --name ci-builder ci;
+    - export DOCKER_BUILDARGS="${DOCKER_BUILDARGS} --builder=ci-builder";
+    - unset DOCKER_HOST;
 
 build:backend:docker:
   extends: .template:build:docker
@@ -114,14 +112,25 @@ build:backend:docker:
 
 build:backend:docker-acceptance:
   extends: build:backend:docker
-  before_script:
-    - apk add make bash git
-    - *dind-login
-    # We're only building acceptance test images for CI runner platform.
-    - unset DOCKER_PLATFORM
   script:
+    # We're only building acceptance test images for CI runner platform.
+    - export DOCKER_PLATFORM=linux/amd64
     # NOTE: Only build for test platform (default) for the acceptance test images
     - make -C backend docker-acceptance
+    - docker build $DOCKER_BUILDARGS -f backend/tests/Dockerfile.acceptance -t $CI_REGISTRY_IMAGE/acceptance:$CI_COMMIT_REF_NAME backend/tests
+
+build:backend:docker-integration:
+  extends: .template:build:docker
+  rules:
+    - changes:
+        paths: ["backend/**/*"]
+        compare_to: "${RULES_CHANGES_COMPARE_TO_REF}"
+      when: always
+    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
+      when: always
+  script:
+    - unset DOCKER_PLATFORM
+    - docker build $DOCKER_BUILDARGS -f backend/tests/Dockerfile.integration -t $CI_REGISTRY_IMAGE/integration:$CI_COMMIT_REF_NAME backend/tests
 
 test:backend:static:
   stage: test
@@ -242,6 +251,7 @@ test:backend:integration:
       when: on_success
   tags:
     - hetzner-amd-beefy
+  parallel: 2
   services:
     - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
       alias: docker
@@ -250,6 +260,8 @@ test:backend:integration:
       artifacts: false
     - job: build:backend:docker-acceptance
       artifacts: false
+    - job: build:backend:docker-integration
+      artifacts: false
   before_script:
     - apk add make bash git curl
     - *dind-login
@@ -375,7 +387,6 @@ coveralls:done:
   tags:
     - hetzner-amd-beefy
 
-
 lint:commit:
   stage: lint
   needs: []
@@ -394,10 +405,11 @@ changelog:
   image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/node:20
   stage: changelog
   variables:
-    GIT_DEPTH: 0  # Always get the full history
-    GIT_STRATEGY: clone  # Always get the full history
-    GIT_CLIFF__BUMP__INITIAL_TAG: "4.0.0"  # TODO: after the new tag is created,
-                                           # remove this variable
+    GIT_DEPTH: 0 # Always get the full history
+    GIT_STRATEGY: clone # Always get the full history
+    GIT_CLIFF__BUMP__INITIAL_TAG:
+      "4.0.0" # TODO: after the new tag is created,
+      # remove this variable
   tags:
     - hetzner-amd-beefy
   rules:
@@ -422,12 +434,12 @@ changelog:
     # getting the centralized git cliff config
   script:
     - release-please release-pr
-        --token=${GITHUB_BOT_TOKEN_REPO_FULL}
-        --repo-url=${GITHUB_REPO_URL}
-        --target-branch=${CI_COMMIT_REF_NAME} || echo "INFO - release already exists" # workaround because we shifted to prerelease versioning strategy and there's already a PR open
+      --token=${GITHUB_BOT_TOKEN_REPO_FULL}
+      --repo-url=${GITHUB_REPO_URL}
+      --target-branch=${CI_COMMIT_REF_NAME} || echo "INFO - release already exists" # workaround because we shifted to prerelease versioning strategy and there's already a PR open
     # git cliff: override the changelog
     - test $GIT_CLIFF == "false" && echo "INFO - Skipping git-cliff" && exit 0
-    - git remote add github-${CI_JOB_ID} https://${GITHUB_USER_NAME}:${GITHUB_BOT_TOKEN_REPO_FULL}@github.com/${GITHUB_REPO_URL} || true  # Ignore already existing remote
+    - git remote add github-${CI_JOB_ID} https://${GITHUB_USER_NAME}:${GITHUB_BOT_TOKEN_REPO_FULL}@github.com/${GITHUB_REPO_URL} || true # Ignore already existing remote
     - gh repo set-default https://${GITHUB_USER_NAME}:${GITHUB_BOT_TOKEN_REPO_FULL}@github.com/${GITHUB_REPO_URL}
     - RELEASE_PLEASE_PR=$(gh pr list --author "${GITHUB_USER_NAME}" --head "release-please--branches--${CI_COMMIT_REF_NAME}" --json number | jq -r '.[0].number // empty')
     - test -z "$RELEASE_PLEASE_PR" && echo "No release-please PR found" && exit 0
@@ -459,6 +471,6 @@ release:github:
   script:
     - npm install -g release-please
     - release-please github-release
-        --token=${GITHUB_BOT_TOKEN_REPO_FULL}
-        --repo-url=${GITHUB_REPO_URL}
-        --target-branch=${CI_COMMIT_REF_NAME}
+      --token=${GITHUB_BOT_TOKEN_REPO_FULL}
+      --repo-url=${GITHUB_REPO_URL}
+      --target-branch=${CI_COMMIT_REF_NAME}

backend/services/Makefile.common

@@ -37,8 +37,14 @@ BUILDFLAGS += -tags=$(subst $(_space),$(_comma),$(strip $(BUILDTAGS) $(BUILDTAGS
 
 # DOCKER_PLATFORM default is defined with respect to the docker server.
 # This fixes the default case on darwin where docker is running inside a VM (linux).
+# NOTE: The platform always falls back on linux/amd64.
 define default_docker_platform_template
-{{- .OSType}}/{{- if eq .Architecture "aarch64" -}}
+{{- if .OSType -}}
+{{- .OSType -}}
+{{- else -}}
+{{ print "linux" }}
+{{- end -}}/
+{{- if eq .Architecture "aarch64" -}}
   arm64
 {{- else -}}
   amd64
@@ -84,7 +90,7 @@ $(binfile): $(GOFILES)
 		GOARCH=$(GOARCH) \
 		go build -o $(binfile) \
 			-ldflags '$(LDFLAGS)' \
-			-ldflags '-X github.com/mendersoftware/mender-server/pkg/version.version=$(VERSION)' \
+			-ldflags '-X "github.com/mendersoftware/mender-server/pkg/version.version=$(VERSION)"' \
 			$(BUILDFLAGS)
 
 .PHONY: build

backend/tests/Dockerfile.integration

@@ -16,4 +16,5 @@ COPY requirements-integration.txt .
 
 RUN pip3 install -r requirements-integration.txt
 
-ENTRYPOINT ["bash", "/tests/run.sh"]
+WORKDIR /tests
+CMD ["pytest"]

backend/tests/docker-compose.yml

@@ -4,7 +4,7 @@ include:
 services:
   acceptance-tester:
     scale: 0
-    image: "${CI_REGISTRY_IMAGE:-localhost/backend-tester}:${CI_ACCEPTANCE_IMAGE_TAG:-acceptance}"
+    image: "${CI_REGISTRY_IMAGE:-localhost/backend-tester}/acceptance:${CI_COMMIT_REF_NAME:-acceptance}"
     build:
       dockerfile: Dockerfile.acceptance
     volumes:
@@ -22,15 +22,14 @@ services:
 
   integration-tester:
     scale: 0
-    image: "${CI_REGISTRY_IMAGE:-localhost/backend-tester}:${CI_INTEGRATION_IMAGE_TAG:-integration}"
+    image: "${CI_REGISTRY_IMAGE:-localhost/backend-tester}/integration:${CI_COMMIT_REF_NAME:-integration}"
     build:
       dockerfile: Dockerfile.integration
     volumes:
-        - ${MENDER_SERVER_PATH:-../../}/backend/tests/integration/docs:/docs
         - ${MENDER_SERVER_PATH:-../../}/backend/tests/integration/downloaded-tools/mender-artifact:/usr/local/bin/mender-artifact
-        - ${MENDER_SERVER_PATH:-../../}/backend/tests/integration/tests:/tests
+        - ${MENDER_SERVER_PATH:-../../}/backend/tests/integration:/tests
         - /var/run/docker.sock:/var/run/docker.sock
-    command: ${PYTEST_ARGS}
+    command: pytest ${PYTEST_ARGS}
     environment:
         PYTHONPATH: "/tests"
         PYTHONDONTWRITEBYTECODE: 1

backend/tests/integration/ci-gitlab-parallel-pytest.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# Copyright 2024 Northern.tech AS
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# A simple script for praparing command line arguments for splitting
+# pytest classes into a Gitlab Parallel matrix.
+
+import os
+import re
+import subprocess
+import sys
+
+INDEX = int(os.environ.get("CI_NODE_INDEX", "1"))
+TOTAL = int(os.environ.get("CI_NODE_TOTAL", "1"))
+
+output = subprocess.check_output(["pytest", "--co", "tests"] + sys.argv[1:])
+
+classes = []
+expr = re.compile("<Class ([^>]+)>")
+for line in output.decode("UTF-8").splitlines():
+    match = expr.search(line)
+    if match:
+        classes.append(match.group(1))
+classes.sort()
+
+
+n_batch = len(classes) // TOTAL
+n_rest = len(classes) % TOTAL
+
+offset = n_batch * (INDEX - 1)
+if INDEX <= n_rest:
+    n_batch += 1
+    offset += INDEX
+else:
+    offset += n_rest
+
+print(" or ".join(classes[offset : offset + n_batch]), end="")

backend/tests/integration/run

@@ -183,17 +183,27 @@ prepare_pytest_args() {
         if [ "$val" == "-k" ]; then
             filter="next"
         elif [ "$filter" == "next" ]; then
-            PYTEST_FILTER="$PYTEST_FILTER and $val"
+            PYTEST_FILTER="$PYTEST_FILTER and ( $val )"
             filter="done"
         else
             PYTEST_ARGS="$PYTEST_ARGS $val"
         fi
     done
 
+    if test $CI_NODE_TOTAL -gt 1 && test $CI_NODE_INDEX -gt 0; then
+      # Rewrite the pytest filter into the current node's batch.
+      PYTEST_FILTER=$(compose_cmd run --rm -it \
+        $RUN_ARGS \
+        -e CI_NODE_TOTAL=$CI_NODE_TOTAL \
+        -e CI_NODE_INDEX=$CI_NODE_INDEX \
+        -v "$MENDER_SERVER_PATH/backend/tests/integration/ci-gitlab-parallel-pytest.py:/ci-gitlab-parallel-pytest.py" \
+        --entrypoint=/ci-gitlab-parallel-pytest.py \
+        integration-tester -k "$PYTEST_FILTER")
+    fi
     echo "-- using PYTEST_FILTER=$PYTEST_FILTER"
-    PYTEST_ARGS="$PYTEST_ARGS -k '$PYTEST_FILTER' $PYTEST_REPORT"
-
+    PYTEST_ARGS="$PYTEST_ARGS -k '${PYTEST_FILTER# }' $PYTEST_REPORT"
     export PYTEST_ARGS
+    echo "-- using PYTEST_ARGS=$PYTEST_ARGS"
 }
 
 cleanup() {