Add crossplane framework for testing

Problem: We want a way to verify nginx configuration reliably in our tests. This is especially useful when introducing new policies, without the desire for testing nginx functionality directly.

Solution: Added a framework for getting the nginx config and passing through crossplane into a structured JSON format for easier parsing.

Because we now use a local container for crossplane in our functional tests, we'll only support running these tests in a kind cluster.

Author: sjberman

.github/workflows/functional.yml

@@ -118,7 +118,7 @@ jobs:
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
-          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CI=true
+          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
         working-directory: ./tests
 
       - name: Run functional graceful-recovery tests
@@ -132,5 +132,5 @@ jobs:
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
-          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CI=true
+          make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
         working-directory: ./tests

tests/Dockerfile.crossplane

@@ -0,0 +1,11 @@
+FROM python:3.12-alpine
+
+ARG NGINX_CONF_DIR
+
+RUN pip install crossplane
+
+COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
+
+USER 101:1001
+
+ENTRYPOINT ["sh"]

tests/Makefile

@@ -12,6 +12,7 @@ GW_SERVICE_TYPE = NodePort## Service type to use for the gateway
 GW_SVC_GKE_INTERNAL = false
 NGF_VERSION ?= edge## NGF version to be tested
 PULL_POLICY = Never## Pull policy for the images
+NGINX_CONF_DIR = internal/mode/static/nginx/conf
 PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml
 SUPPORTED_EXTENDED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification
 STANDARD_CONFORMANCE_PROFILES = GATEWAY-HTTP,GATEWAY-GRPC
@@ -38,6 +39,10 @@ update-go-modules: ## Update the gateway-api go modules to latest main version
 build-test-runner-image: ## Build conformance test runner image
 	docker build -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f conformance/Dockerfile .
 
+.PHONY: build-crossplane-image
+build-crossplane-image: ## Build the crossplane image
+	docker build --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f Dockerfile.crossplane ..
+
 .PHONY: run-conformance-tests
 run-conformance-tests: ## Run conformance tests
 	kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --name $(CLUSTER_NAME)
@@ -80,9 +85,6 @@ ifeq ($(PLUS_ENABLED),true)
 	NGINX_PREFIX := $(NGINX_PLUS_PREFIX)
 endif
 
-.PHONY: setup-gcp-and-run-tests
-setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the functional tests
-
 .PHONY: setup-gcp-and-run-nfr-tests
 setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests
 
@@ -102,13 +104,9 @@ create-gke-router: ## Create a GKE router to allow egress traffic from private n
 sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM
 	./scripts/sync-files-to-vm.sh
 
-.PHONY: run-tests-on-vm
-run-tests-on-vm: ## Run the functional tests on a GCP VM
-	./scripts/run-tests-gcp-vm.sh
-
 .PHONY: nfr-test
 nfr-test: ## Run the NFR tests on a GCP VM
-	NFR=true CI=$(CI) ./scripts/run-tests-gcp-vm.sh
+	CI=$(CI) ./scripts/run-tests-gcp-vm.sh
 
 .PHONY: start-longevity-test
 start-longevity-test: export START_LONGEVITY=true
@@ -130,7 +128,8 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 		--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
 
 .PHONY: test
-test: ## Runs the functional tests on your default k8s cluster
+test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster
+	kind load docker-image nginx-crossplane:latest --name $(CLUSTER_NAME)
 	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \
 		--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
 		--label-filter "functional" $(GINKGO_FLAGS) ./suite -- \

tests/README.md

@@ -28,10 +28,8 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide
 - [System Testing](#system-testing)
   - [Logging in tests](#logging-in-tests)
   - [Step 1 - Run the tests](#step-1---run-the-tests)
-    - [1a - Run the functional tests locally](#1a---run-the-functional-tests-locally)
-    - [1b - Run the tests on a GKE cluster from a GCP VM](#1b---run-the-tests-on-a-gke-cluster-from-a-gcp-vm)
-    - [Functional Tests](#functional-tests)
-    - [NFR tests](#nfr-tests)
+    - [Run the functional tests locally](#run-the-functional-tests-locally)
+    - [Run the NFR tests on a GKE cluster from a GCP VM](#run-the-nfr-tests-on-a-gke-cluster-from-a-gcp-vm)
       - [Longevity testing](#longevity-testing)
   - [Common test amendments](#common-test-amendments)
   - [Step 2 - Cleanup](#step-2---cleanup)
@@ -47,7 +45,7 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide
 - [yq](https://github.com/mikefarah/yq/#install)
 - Make.
 
-If running NFR tests, or running functional tests in GKE:
+If running NFR tests:
 
 - The [gcloud CLI](https://cloud.google.com/sdk/docs/install)
 - A GKE cluster (if `master-authorized-networks` is enabled, please set `ADD_VM_IP_AUTH_NETWORKS=true` in your vars.env file)
@@ -59,9 +57,7 @@ All the commands below are executed from the `tests` directory. You can see all
 
 ### Step 1 - Create a Kubernetes cluster
 
-This can be done in a cloud provider of choice, or locally using `kind`.
-
-**Important**: NFR tests can only be run on a GKE cluster.
+**Important**: Functional/conformance tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster.
 
 To create a local `kind` cluster:
 
@@ -237,7 +233,7 @@ When running locally, the tests create a port-forward from your NGF Pod to local
 test framework. Traffic is sent over this port. If running on a GCP VM targeting a GKE cluster, the tests will create an
 internal LoadBalancer service which will receive the test traffic.
 
-**Important**: NFR tests can only be run on a GKE cluster.
+**Important**: Functional tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster.
 
 Directory structure is as follows:
 
@@ -252,7 +248,7 @@ To log in the tests, use the `GinkgoWriter` interface described here: https://on
 
 ### Step 1 - Run the tests
 
-#### 1a - Run the functional tests locally
+#### Run the functional tests locally
 
 ```makefile
 make test TAG=$(whoami)
@@ -273,9 +269,7 @@ To run the telemetry test:
 make test TAG=$(whoami) GINKGO_LABEL=telemetry
 ```
 
-#### 1b - Run the tests on a GKE cluster from a GCP VM
-
-This step only applies if you are running the NFR tests, or would like to run the functional tests on a GKE cluster from a GCP based VM.
+#### Run the NFR tests on a GKE cluster from a GCP VM
 
 Before running the below `make` commands, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the
 required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions.
@@ -292,7 +286,7 @@ To just set up the VM with no router (this will not run the tests):
 make create-and-setup-vm
 ```
 
-Otherwise, you can set up the VM, router, and run the tests with a single command. See the options in the sections below.
+Otherwise, you can set up the VM, router, and run the tests with a single command. See the options below.
 
 By default, the tests run using the version of NGF that was `git cloned` during the setup. If you want to make
 incremental changes and copy your local changes to the VM to test, you can run
@@ -301,22 +295,6 @@ incremental changes and copy your local changes to the VM to test, you can run
 make sync-files-to-vm
 ```
 
-#### Functional Tests
-
-To set up the GCP environment with the router and VM and then run the tests, run the following command:
-
-```makefile
-make setup-gcp-and-run-tests
-```
-
-To use an existing VM to run the tests, run the following
-
-```makefile
-make run-tests-on-vm
-```
-
-#### NFR tests
-
 To set up the GCP environment with the router and VM and then run the tests, run the following command:
 
 ```makefile
@@ -374,7 +352,7 @@ or to pass a specific flag, e.g. run a specific test, use the GINKGO_FLAGS varia
 make test TAG=$(whoami) GINKGO_FLAGS='-ginkgo.focus "writes the system info to a results file"'
 ```
 
-> Note: if filtering on NFR tests (or functional tests on GKE), set the filter in the appropriate field in your `vars.env` file.
+> Note: if filtering on NFR tests, set the filter in the appropriate field in your `vars.env` file.
 
 If you are running the tests in GCP, add your required label/ flags to `scripts/var.env`.