Skip to content

Commit e8f966a

Browse files
sjbermansalonichf5
authored andcommitted
Add crossplane framework for testing (#2522)
Problem: We want a way to verify nginx configuration reliably in our tests. This is especially useful when introducing new policies, without the desire for testing nginx functionality directly. Solution: Added a framework for getting the nginx config and passing through crossplane into a structured JSON format for easier parsing. Because we now use a local container for crossplane in our functional tests, we'll only support running these tests in a kind cluster.
1 parent 4b0ba5a commit e8f966a

File tree

13 files changed

+523
-66
lines changed

13 files changed

+523
-66
lines changed

.github/workflows/functional.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ jobs:
118118
run: |
119119
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
120120
ngf_tag=${{ steps.ngf-meta.outputs.version }}
121-
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CI=true
121+
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
122122
working-directory: ./tests
123123

124124
- name: Run functional graceful-recovery tests
@@ -132,5 +132,5 @@ jobs:
132132
run: |
133133
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
134134
ngf_tag=${{ steps.ngf-meta.outputs.version }}
135-
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CI=true
135+
make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true
136136
working-directory: ./tests

tests/Makefile

+8-9
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ GW_SERVICE_TYPE = NodePort## Service type to use for the gateway
1212
GW_SVC_GKE_INTERNAL = false
1313
NGF_VERSION ?= edge## NGF version to be tested
1414
PULL_POLICY = Never## Pull policy for the images
15+
NGINX_CONF_DIR = internal/mode/static/nginx/conf
1516
PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml
1617
SUPPORTED_EXTENDED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification
1718
STANDARD_CONFORMANCE_PROFILES = GATEWAY-HTTP,GATEWAY-GRPC
@@ -38,6 +39,10 @@ update-go-modules: ## Update the gateway-api go modules to latest main version
3839
build-test-runner-image: ## Build conformance test runner image
3940
docker build -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f conformance/Dockerfile .
4041

42+
.PHONY: build-crossplane-image
43+
build-crossplane-image: ## Build the crossplane image
44+
docker build --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f framework/crossplane/Dockerfile ..
45+
4146
.PHONY: run-conformance-tests
4247
run-conformance-tests: ## Run conformance tests
4348
kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --name $(CLUSTER_NAME)
@@ -80,9 +85,6 @@ ifeq ($(PLUS_ENABLED),true)
8085
NGINX_PREFIX := $(NGINX_PLUS_PREFIX)
8186
endif
8287

83-
.PHONY: setup-gcp-and-run-tests
84-
setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the functional tests
85-
8688
.PHONY: setup-gcp-and-run-nfr-tests
8789
setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests
8890

@@ -102,13 +104,9 @@ create-gke-router: ## Create a GKE router to allow egress traffic from private n
102104
sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM
103105
./scripts/sync-files-to-vm.sh
104106

105-
.PHONY: run-tests-on-vm
106-
run-tests-on-vm: ## Run the functional tests on a GCP VM
107-
./scripts/run-tests-gcp-vm.sh
108-
109107
.PHONY: nfr-test
110108
nfr-test: ## Run the NFR tests on a GCP VM
111-
NFR=true CI=$(CI) ./scripts/run-tests-gcp-vm.sh
109+
CI=$(CI) ./scripts/run-tests-gcp-vm.sh
112110

113111
.PHONY: start-longevity-test
114112
start-longevity-test: export START_LONGEVITY=true
@@ -130,7 +128,8 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
130128
--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
131129

132130
.PHONY: test
133-
test: ## Runs the functional tests on your default k8s cluster
131+
test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster
132+
kind load docker-image nginx-crossplane:latest --name $(CLUSTER_NAME)
134133
go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \
135134
--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
136135
--label-filter "functional" $(GINKGO_FLAGS) ./suite -- \

tests/README.md

+9-31
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,8 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide
2828
- [System Testing](#system-testing)
2929
- [Logging in tests](#logging-in-tests)
3030
- [Step 1 - Run the tests](#step-1---run-the-tests)
31-
- [1a - Run the functional tests locally](#1a---run-the-functional-tests-locally)
32-
- [1b - Run the tests on a GKE cluster from a GCP VM](#1b---run-the-tests-on-a-gke-cluster-from-a-gcp-vm)
33-
- [Functional Tests](#functional-tests)
34-
- [NFR tests](#nfr-tests)
31+
- [Run the functional tests locally](#run-the-functional-tests-locally)
32+
- [Run the NFR tests on a GKE cluster from a GCP VM](#run-the-nfr-tests-on-a-gke-cluster-from-a-gcp-vm)
3533
- [Longevity testing](#longevity-testing)
3634
- [Common test amendments](#common-test-amendments)
3735
- [Step 2 - Cleanup](#step-2---cleanup)
@@ -47,7 +45,7 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide
4745
- [yq](https://github.com/mikefarah/yq/#install)
4846
- Make.
4947

50-
If running NFR tests, or running functional tests in GKE:
48+
If running NFR tests:
5149

5250
- The [gcloud CLI](https://cloud.google.com/sdk/docs/install)
5351
- A GKE cluster (if `master-authorized-networks` is enabled, please set `ADD_VM_IP_AUTH_NETWORKS=true` in your vars.env file)
@@ -59,9 +57,7 @@ All the commands below are executed from the `tests` directory. You can see all
5957

6058
### Step 1 - Create a Kubernetes cluster
6159

62-
This can be done in a cloud provider of choice, or locally using `kind`.
63-
64-
**Important**: NFR tests can only be run on a GKE cluster.
60+
**Important**: Functional/conformance tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster.
6561

6662
To create a local `kind` cluster:
6763

@@ -237,7 +233,7 @@ When running locally, the tests create a port-forward from your NGF Pod to local
237233
test framework. Traffic is sent over this port. If running on a GCP VM targeting a GKE cluster, the tests will create an
238234
internal LoadBalancer service which will receive the test traffic.
239235

240-
**Important**: NFR tests can only be run on a GKE cluster.
236+
**Important**: Functional tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster.
241237

242238
Directory structure is as follows:
243239

@@ -252,7 +248,7 @@ To log in the tests, use the `GinkgoWriter` interface described here: https://on
252248

253249
### Step 1 - Run the tests
254250

255-
#### 1a - Run the functional tests locally
251+
#### Run the functional tests locally
256252

257253
```makefile
258254
make test TAG=$(whoami)
@@ -273,9 +269,7 @@ To run the telemetry test:
273269
make test TAG=$(whoami) GINKGO_LABEL=telemetry
274270
```
275271

276-
#### 1b - Run the tests on a GKE cluster from a GCP VM
277-
278-
This step only applies if you are running the NFR tests, or would like to run the functional tests on a GKE cluster from a GCP based VM.
272+
#### Run the NFR tests on a GKE cluster from a GCP VM
279273

280274
Before running the below `make` commands, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the
281275
required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions.
@@ -292,7 +286,7 @@ To just set up the VM with no router (this will not run the tests):
292286
make create-and-setup-vm
293287
```
294288

295-
Otherwise, you can set up the VM, router, and run the tests with a single command. See the options in the sections below.
289+
Otherwise, you can set up the VM, router, and run the tests with a single command. See the options below.
296290

297291
By default, the tests run using the version of NGF that was `git cloned` during the setup. If you want to make
298292
incremental changes and copy your local changes to the VM to test, you can run
@@ -301,22 +295,6 @@ incremental changes and copy your local changes to the VM to test, you can run
301295
make sync-files-to-vm
302296
```
303297

304-
#### Functional Tests
305-
306-
To set up the GCP environment with the router and VM and then run the tests, run the following command:
307-
308-
```makefile
309-
make setup-gcp-and-run-tests
310-
```
311-
312-
To use an existing VM to run the tests, run the following
313-
314-
```makefile
315-
make run-tests-on-vm
316-
```
317-
318-
#### NFR tests
319-
320298
To set up the GCP environment with the router and VM and then run the tests, run the following command:
321299

322300
```makefile
@@ -374,7 +352,7 @@ or to pass a specific flag, e.g. run a specific test, use the GINKGO_FLAGS varia
374352
make test TAG=$(whoami) GINKGO_FLAGS='-ginkgo.focus "writes the system info to a results file"'
375353
```
376354

377-
> Note: if filtering on NFR tests (or functional tests on GKE), set the filter in the appropriate field in your `vars.env` file.
355+
> Note: if filtering on NFR tests, set the filter in the appropriate field in your `vars.env` file.
378356
379357
If you are running the tests in GCP, add your required label/ flags to `scripts/var.env`.
380358

tests/framework/crossplane.go

+225
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
package framework
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"net/http"
8+
"strings"
9+
"time"
10+
11+
core "k8s.io/api/core/v1"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/client-go/kubernetes"
14+
"k8s.io/client-go/kubernetes/scheme"
15+
"k8s.io/client-go/rest"
16+
"k8s.io/client-go/tools/remotecommand"
17+
)
18+
19+
// ExpectedNginxField contains an nginx directive key and value,
20+
// and the expected file, server, and location block that it should exist in.
21+
type ExpectedNginxField struct {
22+
// Directive is the directive name.
23+
Directive string
24+
// Value is the value for the directive. Can be the full value or a substring. If it's a substring,
25+
// then ValueSubstringAllowed should be true.
26+
Value string
27+
// File is the file name that should contain the directive. Can be a full filename or a substring.
28+
File string
29+
// Location is the location name that the directive should exist in.
30+
Location string
31+
// Servers are the server names that the directive should exist in.
32+
Servers []string
33+
// ValueSubstringAllowed allows the expected value to be a substring of the real value.
34+
// This makes it easier for cases when real values are complex file names or contain things we
35+
// don't care about, and we just want to check if a substring exists.
36+
ValueSubstringAllowed bool
37+
}
38+
39+
// ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field,
40+
// and returns whether or not that field exists where it should.
41+
func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) error {
42+
for _, config := range conf.Config {
43+
if !strings.Contains(config.File, expFieldCfg.File) {
44+
continue
45+
}
46+
47+
for _, directive := range config.Parsed {
48+
if len(expFieldCfg.Servers) == 0 {
49+
if expFieldCfg.fieldFound(directive) {
50+
return nil
51+
}
52+
continue
53+
}
54+
55+
for _, serverName := range expFieldCfg.Servers {
56+
if directive.Directive == "server" && getServerName(directive.Block) == serverName {
57+
for _, serverDirective := range directive.Block {
58+
if expFieldCfg.Location == "" && expFieldCfg.fieldFound(serverDirective) {
59+
return nil
60+
} else if serverDirective.Directive == "location" &&
61+
fieldExistsInLocation(serverDirective, expFieldCfg) {
62+
return nil
63+
}
64+
}
65+
}
66+
}
67+
}
68+
}
69+
70+
b, err := json.Marshal(conf)
71+
if err != nil {
72+
return fmt.Errorf("error marshaling nginx config: %w", err)
73+
}
74+
75+
return fmt.Errorf("field not found; expected: %+v\nNGINX conf: %s", expFieldCfg, string(b))
76+
}
77+
78+
func getServerName(serverBlock Directives) string {
79+
for _, directive := range serverBlock {
80+
if directive.Directive == "server_name" {
81+
return directive.Args[0]
82+
}
83+
}
84+
85+
return ""
86+
}
87+
88+
func (e ExpectedNginxField) fieldFound(directive *Directive) bool {
89+
arg := strings.Join(directive.Args, " ")
90+
91+
valueMatch := arg == e.Value
92+
if e.ValueSubstringAllowed {
93+
valueMatch = strings.Contains(arg, e.Value)
94+
}
95+
96+
return directive.Directive == e.Directive && valueMatch
97+
}
98+
99+
func fieldExistsInLocation(locationDirective *Directive, expFieldCfg ExpectedNginxField) bool {
100+
// location could start with '=', so get the last element which is the path
101+
loc := locationDirective.Args[len(locationDirective.Args)-1]
102+
if loc == expFieldCfg.Location {
103+
for _, directive := range locationDirective.Block {
104+
if expFieldCfg.fieldFound(directive) {
105+
return true
106+
}
107+
}
108+
}
109+
110+
return false
111+
}
112+
113+
// injectCrossplaneContainer adds an ephemeral container that contains crossplane for parsing
114+
// nginx config. It attaches to the nginx container and shares volumes with it.
115+
func injectCrossplaneContainer(
116+
k8sClient kubernetes.Interface,
117+
timeout time.Duration,
118+
ngfPodName,
119+
namespace string,
120+
) error {
121+
ctx, cancel := context.WithTimeout(context.Background(), timeout)
122+
defer cancel()
123+
124+
pod := &core.Pod{
125+
ObjectMeta: metav1.ObjectMeta{
126+
Name: ngfPodName,
127+
Namespace: namespace,
128+
},
129+
Spec: core.PodSpec{
130+
EphemeralContainers: []core.EphemeralContainer{
131+
{
132+
TargetContainerName: "nginx",
133+
EphemeralContainerCommon: core.EphemeralContainerCommon{
134+
Name: "crossplane",
135+
Image: "nginx-crossplane:latest",
136+
ImagePullPolicy: "Never",
137+
Stdin: true,
138+
VolumeMounts: []core.VolumeMount{
139+
{
140+
MountPath: "/etc/nginx/conf.d",
141+
Name: "nginx-conf",
142+
},
143+
{
144+
MountPath: "/etc/nginx/stream-conf.d",
145+
Name: "nginx-stream-conf",
146+
},
147+
{
148+
MountPath: "/etc/nginx/module-includes",
149+
Name: "module-includes",
150+
},
151+
{
152+
MountPath: "/etc/nginx/secrets",
153+
Name: "nginx-secrets",
154+
},
155+
{
156+
MountPath: "/etc/nginx/includes",
157+
Name: "nginx-includes",
158+
},
159+
},
160+
},
161+
},
162+
},
163+
},
164+
}
165+
166+
podClient := k8sClient.CoreV1().Pods(namespace)
167+
if _, err := podClient.UpdateEphemeralContainers(ctx, ngfPodName, pod, metav1.UpdateOptions{}); err != nil {
168+
return fmt.Errorf("error adding ephemeral container: %w", err)
169+
}
170+
171+
return nil
172+
}
173+
174+
// createCrossplaneExecutor creates the executor for the crossplane command.
175+
func createCrossplaneExecutor(
176+
k8sClient kubernetes.Interface,
177+
k8sConfig *rest.Config,
178+
ngfPodName,
179+
namespace string,
180+
) (remotecommand.Executor, error) {
181+
cmd := []string{"./crossplane", "/etc/nginx/nginx.conf"}
182+
opts := &core.PodExecOptions{
183+
Command: cmd,
184+
Container: "crossplane",
185+
Stdout: true,
186+
Stderr: true,
187+
}
188+
189+
req := k8sClient.CoreV1().RESTClient().Post().
190+
Resource("pods").
191+
SubResource("exec").
192+
Name(ngfPodName).
193+
Namespace(namespace).
194+
VersionedParams(opts, scheme.ParameterCodec)
195+
196+
exec, err := remotecommand.NewSPDYExecutor(k8sConfig, http.MethodPost, req.URL())
197+
if err != nil {
198+
return nil, fmt.Errorf("error creating executor: %w", err)
199+
}
200+
201+
return exec, nil
202+
}
203+
204+
// The following types are copied from https://github.com/nginxinc/nginx-go-crossplane,
205+
// with unnecessary fields stripped out.
206+
type Payload struct {
207+
Config []Config `json:"config"`
208+
}
209+
210+
type Config struct {
211+
File string `json:"file"`
212+
Parsed Directives `json:"parsed"`
213+
}
214+
215+
type Directive struct {
216+
Comment *string `json:"comment,omitempty"`
217+
Directive string `json:"directive"`
218+
File string `json:"file,omitempty"`
219+
Args []string `json:"args"`
220+
Includes []int `json:"includes,omitempty"`
221+
Block Directives `json:"block,omitempty"`
222+
Line int `json:"line"`
223+
}
224+
225+
type Directives []*Directive

0 commit comments

Comments
 (0)