From 039c1c3e7ba63e0527b91bc29043bbdc2f55ef61 Mon Sep 17 00:00:00 2001 From: Chun-Hung Tseng Date: Thu, 12 Dec 2024 21:35:58 +0100 Subject: [PATCH] Introduce multi-version support for e2e tests --- .github/workflows/e2e-tests.yaml | 94 ++++++++++++--- .gitignore | 4 + Makefile | 25 +++- kind/load-data-job/README.md | 7 ++ .../{load-data.sh => load-data.orig.sh} | 1 + ...demo-data.sql => local-demo-data.orig.sql} | 9 ++ kind/local-env.sh | 111 ++++++++++++++++-- 7 files changed, 224 insertions(+), 27 deletions(-) create mode 100644 kind/load-data-job/README.md rename kind/load-data-job/{load-data.sh => load-data.orig.sh} (96%) rename kind/load-data-job/{local-demo-data.sql => local-demo-data.orig.sql} (98%) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 43f1c68..e92fb27 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -17,8 +17,8 @@ env: HTTPS_ENABLE: true NETBOX_RESTORATION_HASH_FIELD_NAME: netboxOperatorRestorationHash jobs: - e2e-tests: - name: E2E tests for netbox operator + e2e-tests-3-7-8: + name: Against netbox version 3.7.8 runs-on: ubuntu-latest steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -28,7 +28,6 @@ jobs: - name: Import environment variables from file run: | cat ".github/env" >> "$GITHUB_ENV" - echo "E2E_DIAGNOSTIC_DIRECTORY=$(mktemp -d)" >> "$GITHUB_ENV" - name: Start kind cluster uses: helm/kind-action@ae94020eaf628e9b9b9f341a10cc0cdcf5c018fb # v1.11.0 with: @@ -47,15 +46,45 @@ jobs: kubectl get pods -A echo "Cluster information" kubectl cluster-info - - name: Setup kind cluster with required software such as NetBox + - name: Run e2e tests run: | - make create-kind - - name: Deploy NetBox operator to the kind cluster + # # Very straight forward way of implementing a test and checking the result + # kubectl apply -f config/samples/netbox_v1_prefixclaim.yaml + # kubectl get prefixclaim,prefix,ipaddressclaim,ipaddress,iprange,iprangeclaim + # kubectl wait --for=condition=ready --timeout=30s prefixclaim.netbox.dev/prefixclaim-sample + + # Use Chainsaw + make test-e2e-3.7.8 + e2e-tests-4-0-11: + name: Against netbox version 4.0.11 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + with: + go-version: 1.23.4 + - name: Import environment variables from file run: | - make deploy-kind - - name: Run tests - env: - E2E_DIAGNOSTIC_DIRECTORY: ${{ env.E2E_DIAGNOSTIC_DIRECTORY }} + cat ".github/env" >> "$GITHUB_ENV" + - name: Start kind cluster + uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0 + with: + version: ${{ env.kind-version }} + node_image: ${{ env.kind-image }} + wait: 300s + config: ./tests/e2e/kind-config.yaml + cluster_name: e2e + - name: Wait for cluster to finish bootstraping + run: | + echo "Waiting for all nodes to be ready..." + kubectl wait --for=condition=Ready nodes --all --timeout=120s + kubectl get nodes + echo "Waiting for all pods to be ready..." + kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s + kubectl get pods -A + echo "Cluster information" + kubectl cluster-info + - name: Run e2e tests run: | # # Very straight forward way of implementing a test and checking the result # kubectl apply -f config/samples/netbox_v1_prefixclaim.yaml @@ -63,11 +92,42 @@ jobs: # kubectl wait --for=condition=ready --timeout=30s prefixclaim.netbox.dev/prefixclaim-sample # Use Chainsaw - make test-e2e - - name: Upload diagnostics artifact - if: ${{ failure() }} - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + make test-e2e-4.0.11 + e2e-tests-4-1-7: + name: Against netbox version 4.1.7 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + with: + go-version: 1.23.4 + - name: Import environment variables from file + run: | + cat ".github/env" >> "$GITHUB_ENV" + - name: Start kind cluster + uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0 with: - name: cluster-state - path: ${{ env.E2E_DIAGNOSTIC_DIRECTORY }} - retention-days: 15 + version: ${{ env.kind-version }} + node_image: ${{ env.kind-image }} + wait: 300s + config: ./tests/e2e/kind-config.yaml + cluster_name: e2e + - name: Wait for cluster to finish bootstraping + run: | + echo "Waiting for all nodes to be ready..." + kubectl wait --for=condition=Ready nodes --all --timeout=120s + kubectl get nodes + echo "Waiting for all pods to be ready..." + kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s + kubectl get pods -A + echo "Cluster information" + kubectl cluster-info + - name: Run e2e tests + run: | + # # Very straight forward way of implementing a test and checking the result + # kubectl apply -f config/samples/netbox_v1_prefixclaim.yaml + # kubectl get prefixclaim,prefix,ipaddressclaim,ipaddress,iprange,iprangeclaim + # kubectl wait --for=condition=ready --timeout=30s prefixclaim.netbox.dev/prefixclaim-sample + + # Use Chainsaw + make test-e2e-4.1.7 diff --git a/.gitignore b/.gitignore index ea12f27..805728d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,11 @@ Dockerfile.cross # Test binary, built with `go test -c` *.test + +# Test/Intermediate files database.sql +load-data.sh +local-demo-data.sql # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/Makefile b/Makefile index abc4640..c5b586c 100644 --- a/Makefile +++ b/Makefile @@ -237,6 +237,25 @@ generate_mocks: ## TODO: auto install go install go.uber.org/mock/mockgen@latest mkdir -p ${GEN_DIR} mockgen -destination ${GEN_DIR}/${NETBOX_MOCKS_OUTPUT_FILE} -source=${INTERFACE_DEFITIONS_DIR} -.PHONY: test-e2e -test-e2e: install-$(GO_PACKAGE_NAME_CHAINSAW) - chainsaw test --namespace e2e +# e2e tests +E2E_PARAM := --namespace e2e --parallel 3 --apply-timeout 3m --assert-timeout 3m --delete-timeout 3m --error-timeout 3m --exec-timeout 3m # --skip-delete (add this argument for local debugging) +.PHONY: create-kind-3.7.8 +create-kind-3.7.8: + ./kind/local-env.sh --version 3.7.8 +.PHONY: test-e2e-3.7.8 +test-e2e-3.7.8: create-kind-3.7.8 deploy-kind install-$(GO_PACKAGE_NAME_CHAINSAW) + chainsaw test $(E2E_PARAM) + +.PHONY: create-kind-4.0.11 +create-kind-4.0.11: + ./kind/local-env.sh --version 4.0.11 +.PHONY: test-e2e-4.0.11 +test-e2e-4.0.11: create-kind-4.0.11 deploy-kind install-$(GO_PACKAGE_NAME_CHAINSAW) + chainsaw test $(E2E_PARAM) + +.PHONY: create-kind-4.1.7 +create-kind-4.1.7: + ./kind/local-env.sh --version 4.1.7 +.PHONY: test-e2e-4.1.7 +test-e2e-4.1.7: create-kind-4.1.7 deploy-kind install-$(GO_PACKAGE_NAME_CHAINSAW) + chainsaw test $(E2E_PARAM) diff --git a/kind/load-data-job/README.md b/kind/load-data-job/README.md new file mode 100644 index 0000000..40582c8 --- /dev/null +++ b/kind/load-data-job/README.md @@ -0,0 +1,7 @@ +# load-data-job + +Due to database schema changes cross major/minor NetBox versions, we have to `patch` the SQL files and demo data link on-the-fly. + +The default values stems from the NetBox 4.1.x version. So the patching will only happen for 3.7.x and 4.0.x versions. + +Please see `../local-env.sh`, that's where all the patching happen. diff --git a/kind/load-data-job/load-data.sh b/kind/load-data-job/load-data.orig.sh similarity index 96% rename from kind/load-data-job/load-data.sh rename to kind/load-data-job/load-data.orig.sh index a98ca9f..4158cdd 100755 --- a/kind/load-data-job/load-data.sh +++ b/kind/load-data-job/load-data.orig.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -o errexit TMP_SQL_FILE=$(mktemp /tmp/netbox-data-dump.XXXXXXX.sql) || exit 1 curl -k https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql > "${TMP_SQL_FILE}" diff --git a/kind/load-data-job/local-demo-data.sql b/kind/load-data-job/local-demo-data.orig.sql similarity index 98% rename from kind/load-data-job/local-demo-data.sql rename to kind/load-data-job/local-demo-data.orig.sql index 21bbeca..f811f63 100644 --- a/kind/load-data-job/local-demo-data.sql +++ b/kind/load-data-job/local-demo-data.orig.sql @@ -17,6 +17,15 @@ VALUES (6, 'boolean', 'cfDataTypeBool', 'cf Data Type Bool', 'Custom field 3 for INSERT INTO public.extras_customfield (id, type, name, label, description, required, filter_logic, "default", weight, validation_minimum, validation_maximum, validation_regex, created, last_updated, related_object_type_id, group_name, search_weight, is_cloneable, choice_set_id, ui_editable, ui_visible, comments, "unique", related_object_filter) VALUES (7, 'integer', 'cfDataTypeInteger', 'cf Data Type Integer', 'Custom field 4 for ParentPrefixSelector', false, 'exact', NULL, 100, NULL, NULL, '', '2024-06-13 15:17:08.65334+00', '2024-06-13 15:17:08.653359+00', NULL, 'netbox-operator', 100, false, NULL, 'hidden', 'always', '', false, NULL); +-- +-- Data for Name: extras_customfield_content_types; Type: TABLE DATA; Schema: public; Owner: netbox +-- + +COPY public.extras_customfield_content_types (id, customfield_id, contenttype_id) FROM stdin; +1 1 110 +2 2 70 +\. + -- associate custom fields with IP INSERT INTO public.extras_customfield_object_types (id, customfield_id, objecttype_id) VALUES (2, 2, 69); diff --git a/kind/local-env.sh b/kind/local-env.sh index cf60de6..05405d9 100755 --- a/kind/local-env.sh +++ b/kind/local-env.sh @@ -1,15 +1,108 @@ #!/bin/bash -set -o errexit +set -e -u -o pipefail -kind create cluster || echo "cluster already exists, continuing..." +NAMESPACE="" +VERSION="4.1.7" # default value +NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.163/netbox-5.0.0-beta.163.tgz" # default value +while [[ $# -gt 0 ]]; do + case $1 in + -n|--namespace) + NAMESPACE="$2" + shift # past argument + shift # past value + ;; + -v|--version) + VERSION="$2" + shift # past argument + shift # past value + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + esac +done -if [ -z "$1" ]; then - echo "Using default namespace." +echo "=======Parsed arguments=======" +echo "Namespace = ${NAMESPACE}" +echo "Version = ${VERSION}" +echo "==============================" + +# aurgment check / init +if [ -z "$NAMESPACE" ]; then + echo "Using default namespace" NAMESPACE="default" else - echo "Using namespace: $1" - NAMESPACE="$1" + echo "Using namespace: $NAMESPACE" +fi + +if [[ "${VERSION}" == "3.7.8" ]] ;then + echo "Using version ${VERSION}" + # need to align with netbox-chart otherwise the creation of the cluster will hang + declare -a Images=( \ + "gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1" \ + "busybox:1.36.1" \ + "docker.io/bitnami/redis:7.2.4-debian-12-r9" \ + "docker.io/netboxcommunity/netbox:v3.7.8" \ + "ghcr.io/zalando/postgres-operator:v1.12.2" \ + "ghcr.io/zalando/spilo-16:3.2-p3" \ + ) + NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta5/netbox-5.0.0-beta5.tgz" + + # perform patching, as we need different demo data and adapt to the database schema + # to avoid accidental check-in of the files, the base file is renamed to xx.orig.yy, and the xx.yy is added to .gitignore + # patch load-data.sh + sed 's/netbox-demo-v4.1.sql/netbox-demo-v3.7.sql/g' $(dirname "$0")/load-data-job/load-data.orig.sh > $(dirname "$0")/load-data-job/load-data.sh && chmod +x $(dirname "$0")/load-data-job/load-data.sh + + # patch local-demo-data.sql + sed \ + -e "s/public.extras_customfield_object_types (id, customfield_id, objecttype_id)/public.extras_customfield_content_types (id, customfield_id, contenttype_id)/g" \ + -e 's/related_object_type_id/object_type_id/g' \ + -e 's/, comments, \"unique\", related_object_filter//g' \ + -e "s/, '', false, NULL//g" $(dirname "$0")/load-data-job/local-demo-data.orig.sql > $(dirname "$0")/load-data-job/local-demo-data.sql +elif [[ "${VERSION}" == "4.0.11" ]] ;then + echo "Using version ${VERSION}" + # need to align with netbox-chart otherwise the creation of the cluster will hang + declare -a Images=( \ + "gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1" \ + "busybox:1.36.1" \ + "docker.io/bitnami/redis:7.4.0-debian-12-r2" \ + "ghcr.io/netbox-community/netbox:v4.0.11" \ + "ghcr.io/zalando/postgres-operator:v1.12.2" \ + "ghcr.io/zalando/spilo-16:3.2-p3" \ + ) + NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.84/netbox-5.0.0-beta.84.tgz" + + # patch load-data.sh + sed 's/netbox-demo-v4.1.sql/netbox-demo-v4.0.sql/g' $(dirname "$0")/load-data-job/load-data.orig.sh > $(dirname "$0")/load-data-job/load-data.sh && chmod +x $(dirname "$0")/load-data-job/load-data.sh + + # patch local-demo-data.sql + sed \ + -e "s/comments, \"unique\", related_object_filter)/comments)/g" \ + -e "s/'', false, NULL);/'');/g" $(dirname "$0")/load-data-job/local-demo-data.orig.sql > $(dirname "$0")/load-data-job/local-demo-data.sql +elif [[ "${VERSION}" == "4.1.7" ]] ;then + echo "Using version ${VERSION}" + # need to align with netbox-chart otherwise the creation of the cluster will hang + declare -a Images=( \ + "gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1" \ + "busybox:1.37.0" \ + "docker.io/bitnami/redis:7.4.1-debian-12-r2" \ + "ghcr.io/netbox-community/netbox:v4.1.7" \ + "ghcr.io/zalando/postgres-operator:v1.12.2" \ + "ghcr.io/zalando/spilo-16:3.2-p3" \ + ) + + cp $(dirname "$0")/load-data-job/load-data.orig.sh $(dirname "$0")/load-data-job/load-data.sh + cp $(dirname "$0")/load-data-job/local-demo-data.orig.sql $(dirname "$0")/load-data-job/local-demo-data.sql +else + echo "Unknown version ${VERSION}" + exit 1 fi + +# create a kind cluster +kind create cluster || echo "cluster already exists, continuing..." + +kubectl wait --for=jsonpath='{.status.phase}'=Active --timeout=1s namespace/${NAMESPACE} if ! kubectl get namespaces | grep -q "^${NAMESPACE} "; then echo "Namespace ${NAMESPACE} does not exist." exit 1 @@ -47,6 +140,10 @@ helm upgrade --install --namespace="${NAMESPACE}" netbox \ --set externalDatabase.existingSecretName="netbox.netbox-db.credentials.postgresql.acid.zalan.do" \ --set externalDatabase.existingSecretKey="password" \ --set redis.auth.password="password" \ - https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.163/netbox-5.0.0-beta.163.tgz + ${NETBOX_HELM_CHART} kubectl rollout status --namespace="${NAMESPACE}" deployment netbox + +# clean up +rm $(dirname "$0")/load-data-job/load-data.sh +rm $(dirname "$0")/load-data-job/local-demo-data.sql