Introduce multi-version support for e2e tests

.github/workflows/e2e-tests.yaml

@@ -17,8 +17,8 @@ env:
   HTTPS_ENABLE: true
   NETBOX_RESTORATION_HASH_FIELD_NAME: netboxOperatorRestorationHash
 jobs:
-  e2e-tests:
-    name: E2E tests for netbox operator
+  e2e-tests-3-7-8:
+    name: Against netbox version 3.7.8
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -28,7 +28,6 @@ jobs:
     - name: Import environment variables from file
       run: |
         cat ".github/env" >> "$GITHUB_ENV"
-        echo "E2E_DIAGNOSTIC_DIRECTORY=$(mktemp -d)" >> "$GITHUB_ENV"
     - name: Start kind cluster
       uses: helm/kind-action@ae94020eaf628e9b9b9f341a10cc0cdcf5c018fb # v1.11.0
       with:
@@ -47,27 +46,88 @@ jobs:
         kubectl get pods -A
         echo "Cluster information"
         kubectl cluster-info
-    - name: Setup kind cluster with required software such as NetBox
+    - name: Run e2e tests
       run: |
-        make create-kind
-    - name: Deploy NetBox operator to the kind cluster
+        # # Very straight forward way of implementing a test and checking the result
+        # kubectl apply -f config/samples/netbox_v1_prefixclaim.yaml
+        # kubectl get prefixclaim,prefix,ipaddressclaim,ipaddress,iprange,iprangeclaim
+        # kubectl wait --for=condition=ready --timeout=30s prefixclaim.netbox.dev/prefixclaim-sample
+
+        # Use Chainsaw
+        make test-e2e-3.7.8
+  e2e-tests-4-0-11:
+    name: Against netbox version 4.0.11
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      with: 
+        go-version: 1.23.4
+    - name: Import environment variables from file
       run: |
-        make deploy-kind
-    - name: Run tests
-      env:
-        E2E_DIAGNOSTIC_DIRECTORY: ${{ env.E2E_DIAGNOSTIC_DIRECTORY }}
+        cat ".github/env" >> "$GITHUB_ENV"
+    - name: Start kind cluster
+      uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+      with:
+        version: ${{ env.kind-version }}
+        node_image: ${{ env.kind-image }}
+        wait: 300s
+        config: ./tests/e2e/kind-config.yaml
+        cluster_name: e2e
+    - name: Wait for cluster to finish bootstraping
+      run: |
+        echo "Waiting for all nodes to be ready..."
+        kubectl wait --for=condition=Ready nodes --all --timeout=120s
+        kubectl get nodes
+        echo "Waiting for all pods to be ready..."
+        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s
+        kubectl get pods -A
+        echo "Cluster information"
+        kubectl cluster-info
+    - name: Run e2e tests
       run: |
         # # Very straight forward way of implementing a test and checking the result
         # kubectl apply -f config/samples/netbox_v1_prefixclaim.yaml
         # kubectl get prefixclaim,prefix,ipaddressclaim,ipaddress,iprange,iprangeclaim
         # kubectl wait --for=condition=ready --timeout=30s prefixclaim.netbox.dev/prefixclaim-sample
 
         # Use Chainsaw
-        make test-e2e
-    - name: Upload diagnostics artifact
-      if: ${{ failure() }}
-      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        make test-e2e-4.0.11
+  e2e-tests-4-1-7:
+    name: Against netbox version 4.1.7
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      with: 
+        go-version: 1.23.4
+    - name: Import environment variables from file
+      run: |
+        cat ".github/env" >> "$GITHUB_ENV"
+    - name: Start kind cluster
+      uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
       with:
-        name: cluster-state
-        path: ${{ env.E2E_DIAGNOSTIC_DIRECTORY }}
-        retention-days: 15
+        version: ${{ env.kind-version }}
+        node_image: ${{ env.kind-image }}
+        wait: 300s
+        config: ./tests/e2e/kind-config.yaml
+        cluster_name: e2e
+    - name: Wait for cluster to finish bootstraping
+      run: |
+        echo "Waiting for all nodes to be ready..."
+        kubectl wait --for=condition=Ready nodes --all --timeout=120s
+        kubectl get nodes
+        echo "Waiting for all pods to be ready..."
+        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s
+        kubectl get pods -A
+        echo "Cluster information"
+        kubectl cluster-info
+    - name: Run e2e tests
+      run: |
+        # # Very straight forward way of implementing a test and checking the result
+        # kubectl apply -f config/samples/netbox_v1_prefixclaim.yaml
+        # kubectl get prefixclaim,prefix,ipaddressclaim,ipaddress,iprange,iprangeclaim
+        # kubectl wait --for=condition=ready --timeout=30s prefixclaim.netbox.dev/prefixclaim-sample
+
+        # Use Chainsaw
+        make test-e2e-4.1.7

.gitignore

@@ -9,7 +9,11 @@ Dockerfile.cross
 
 # Test binary, built with `go test -c`
 *.test
+
+# Test/Intermediate files
 database.sql
+load-data.sh
+local-demo-data.sql
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out

Makefile

@@ -237,6 +237,25 @@ generate_mocks: ## TODO: auto install go install go.uber.org/mock/mockgen@latest
 	mkdir -p ${GEN_DIR}
 	mockgen -destination ${GEN_DIR}/${NETBOX_MOCKS_OUTPUT_FILE} -source=${INTERFACE_DEFITIONS_DIR}
 
-.PHONY: test-e2e
-test-e2e: install-$(GO_PACKAGE_NAME_CHAINSAW)
-	chainsaw test --namespace e2e
+# e2e tests
+E2E_PARAM := --namespace e2e --parallel 3 --apply-timeout 3m --assert-timeout 3m --delete-timeout 3m --error-timeout 3m --exec-timeout 3m # --skip-delete (add this argument for local debugging)
+.PHONY: create-kind-3.7.8
+create-kind-3.7.8:
+	./kind/local-env.sh --version 3.7.8
+.PHONY: test-e2e-3.7.8
+test-e2e-3.7.8: create-kind-3.7.8 deploy-kind install-$(GO_PACKAGE_NAME_CHAINSAW) 
+	chainsaw test $(E2E_PARAM)
+
+.PHONY: create-kind-4.0.11
+create-kind-4.0.11:
+	./kind/local-env.sh --version 4.0.11
+.PHONY: test-e2e-4.0.11
+test-e2e-4.0.11: create-kind-4.0.11 deploy-kind install-$(GO_PACKAGE_NAME_CHAINSAW) 
+	chainsaw test $(E2E_PARAM)
+
+.PHONY: create-kind-4.1.7
+create-kind-4.1.7:
+	./kind/local-env.sh --version 4.1.7
+.PHONY: test-e2e-4.1.7
+test-e2e-4.1.7: create-kind-4.1.7 deploy-kind install-$(GO_PACKAGE_NAME_CHAINSAW) 
+	chainsaw test $(E2E_PARAM)

kind/load-data-job/README.md

@@ -0,0 +1,7 @@
+# load-data-job
+
+Due to database schema changes cross major/minor NetBox versions, we have to `patch` the SQL files and demo data link on-the-fly.
+
+The default values stems from the NetBox 4.1.x version. So the patching will only happen for 3.7.x and 4.0.x versions. 
+
+Please see `../local-env.sh`, that's where all the patching happen.

kind/load-data-job/load-data.sh → kind/load-data-job/load-data.orig.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -o errexit
 
 TMP_SQL_FILE=$(mktemp /tmp/netbox-data-dump.XXXXXXX.sql) || exit 1
 curl -k https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql > "${TMP_SQL_FILE}"

kind/load-data-job/local-demo-data.sql → kind/load-data-job/local-demo-data.orig.sql

@@ -17,6 +17,15 @@ VALUES (6, 'boolean', 'cfDataTypeBool', 'cf Data Type Bool', 'Custom field 3 for
 INSERT INTO public.extras_customfield (id, type, name, label, description, required, filter_logic, "default", weight, validation_minimum, validation_maximum, validation_regex, created, last_updated, related_object_type_id, group_name, search_weight, is_cloneable, choice_set_id, ui_editable, ui_visible, comments, "unique", related_object_filter)
 VALUES (7, 'integer', 'cfDataTypeInteger', 'cf Data Type Integer', 'Custom field 4 for ParentPrefixSelector', false, 'exact', NULL, 100, NULL, NULL, '', '2024-06-13 15:17:08.65334+00', '2024-06-13 15:17:08.653359+00', NULL, 'netbox-operator', 100, false, NULL, 'hidden', 'always', '', false, NULL);
 
+--
+-- Data for Name: extras_customfield_content_types; Type: TABLE DATA; Schema: public; Owner: netbox
+--
+
+COPY public.extras_customfield_content_types (id, customfield_id, contenttype_id) FROM stdin;
+1	1	110
+2	2	70
+\.
+
 -- associate custom fields with IP
 INSERT INTO public.extras_customfield_object_types (id, customfield_id, objecttype_id)
 VALUES (2, 2, 69);

kind/local-env.sh

@@ -1,15 +1,108 @@
 #!/bin/bash
-set -o errexit
+set -e -u -o pipefail
 
-kind create cluster || echo "cluster already exists, continuing..."
+NAMESPACE=""
+VERSION="4.1.7" # default value
+NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.163/netbox-5.0.0-beta.163.tgz" # default value
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -n|--namespace)
+      NAMESPACE="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -v|--version)
+      VERSION="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -*|--*)
+      echo "Unknown option $1"
+      exit 1
+      ;;
+  esac
+done
 
-if [ -z "$1" ]; then
-    echo "Using default namespace."
+echo "=======Parsed arguments======="
+echo "Namespace   = ${NAMESPACE}"
+echo "Version     = ${VERSION}"
+echo "=============================="
+
+# aurgment check / init
+if [ -z "$NAMESPACE" ]; then
+    echo "Using default namespace"
     NAMESPACE="default"
 else
-    echo "Using namespace: $1"
-    NAMESPACE="$1"
+    echo "Using namespace: $NAMESPACE"
+fi
+
+if [[ "${VERSION}" == "3.7.8" ]] ;then
+  echo "Using version ${VERSION}"
+  # need to align with netbox-chart otherwise the creation of the cluster will hang
+  declare -a Images=( \
+  "gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1" \
+  "busybox:1.36.1" \
+  "docker.io/bitnami/redis:7.2.4-debian-12-r9" \
+  "docker.io/netboxcommunity/netbox:v3.7.8" \
+  "ghcr.io/zalando/postgres-operator:v1.12.2" \
+  "ghcr.io/zalando/spilo-16:3.2-p3" \
+  )
+  NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta5/netbox-5.0.0-beta5.tgz"
+
+  # perform patching, as we need different demo data and adapt to the database schema 
+  # to avoid accidental check-in of the files, the base file is renamed to xx.orig.yy, and the xx.yy is added to .gitignore
+  # patch load-data.sh
+  sed 's/netbox-demo-v4.1.sql/netbox-demo-v3.7.sql/g' $(dirname "$0")/load-data-job/load-data.orig.sh > $(dirname "$0")/load-data-job/load-data.sh && chmod +x $(dirname "$0")/load-data-job/load-data.sh
+
+  # patch local-demo-data.sql
+  sed \
+    -e "s/public.extras_customfield_object_types (id, customfield_id, objecttype_id)/public.extras_customfield_content_types (id, customfield_id, contenttype_id)/g" \
+    -e 's/related_object_type_id/object_type_id/g' \
+    -e 's/, comments, \"unique\", related_object_filter//g' \
+    -e "s/, '', false, NULL//g" $(dirname "$0")/load-data-job/local-demo-data.orig.sql > $(dirname "$0")/load-data-job/local-demo-data.sql
+elif [[ "${VERSION}" == "4.0.11" ]] ;then
+  echo "Using version ${VERSION}"
+  # need to align with netbox-chart otherwise the creation of the cluster will hang
+  declare -a Images=( \
+  "gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1" \
+  "busybox:1.36.1" \
+  "docker.io/bitnami/redis:7.4.0-debian-12-r2" \
+  "ghcr.io/netbox-community/netbox:v4.0.11" \
+  "ghcr.io/zalando/postgres-operator:v1.12.2" \
+  "ghcr.io/zalando/spilo-16:3.2-p3" \
+  )
+  NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.84/netbox-5.0.0-beta.84.tgz"
+
+  # patch load-data.sh
+  sed 's/netbox-demo-v4.1.sql/netbox-demo-v4.0.sql/g' $(dirname "$0")/load-data-job/load-data.orig.sh > $(dirname "$0")/load-data-job/load-data.sh && chmod +x $(dirname "$0")/load-data-job/load-data.sh
+
+  # patch local-demo-data.sql
+  sed \
+    -e "s/comments, \"unique\", related_object_filter)/comments)/g" \
+    -e "s/'', false, NULL);/'');/g" $(dirname "$0")/load-data-job/local-demo-data.orig.sql > $(dirname "$0")/load-data-job/local-demo-data.sql
+elif [[ "${VERSION}" == "4.1.7" ]] ;then
+  echo "Using version ${VERSION}"
+  # need to align with netbox-chart otherwise the creation of the cluster will hang
+  declare -a Images=( \
+  "gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1" \
+  "busybox:1.37.0" \
+  "docker.io/bitnami/redis:7.4.1-debian-12-r2" \
+  "ghcr.io/netbox-community/netbox:v4.1.7" \
+  "ghcr.io/zalando/postgres-operator:v1.12.2" \
+  "ghcr.io/zalando/spilo-16:3.2-p3" \
+  )
+
+  cp $(dirname "$0")/load-data-job/load-data.orig.sh $(dirname "$0")/load-data-job/load-data.sh
+  cp $(dirname "$0")/load-data-job/local-demo-data.orig.sql $(dirname "$0")/load-data-job/local-demo-data.sql
+else
+  echo "Unknown version ${VERSION}"
+  exit 1
 fi
+
+# create a kind cluster
+kind create cluster || echo "cluster already exists, continuing..."
+
+kubectl wait --for=jsonpath='{.status.phase}'=Active --timeout=1s namespace/${NAMESPACE}
 if ! kubectl get namespaces | grep -q "^${NAMESPACE} "; then
     echo "Namespace ${NAMESPACE} does not exist."
     exit 1
@@ -47,6 +140,10 @@ helm upgrade --install --namespace="${NAMESPACE}" netbox \
   --set externalDatabase.existingSecretName="netbox.netbox-db.credentials.postgresql.acid.zalan.do" \
   --set externalDatabase.existingSecretKey="password" \
   --set redis.auth.password="password" \
-  https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.163/netbox-5.0.0-beta.163.tgz
+  ${NETBOX_HELM_CHART}
 
 kubectl rollout status --namespace="${NAMESPACE}" deployment netbox
+
+# clean up
+rm $(dirname "$0")/load-data-job/load-data.sh
+rm $(dirname "$0")/load-data-job/local-demo-data.sql