Add Conformance Test Framework

This commit adds the initial framework we need to run conformance tests. ALL CREDIT to sig-network-gatewaya-api community and folks who added conformance there. We have blatantly copied from them and adapted the suite to our needs. https://github.com/kubernetes-sigs/gateway-api/tree/main/conformance Signed-off-by: Surya Seetharaman <suryaseetharaman.9@gmail.com>
kubernetes-sigs · May 22, 2023 · 00bf1ff · 00bf1ff
1 parent 2f547fe
commit 00bf1ff
Show file tree

Hide file tree

Showing 12 changed files with 1,092 additions and 0 deletions.
diff --git a/Makefile b/Makefile
@@ -26,6 +26,10 @@ help: ## Display this help.
 
 ##@ Development
 
+# Command-line flags passed to "go test" for the conformance
+# test. These are passed after the "-args" flag.
+CONFORMANCE_FLAGS ?=
+
 .PHONY: fmt
 fmt: ## Run go fmt against code.
 	go fmt ./...
@@ -47,6 +51,10 @@ verify:
 crd-e2e:
 	hack/crd-e2e.sh -v
 
+.PHONY: conformance
+conformance:
+	go test -v ./conformance/... -args ${CONFORMANCE_FLAGS}
+
 ##@ Deployment
 install: generate ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	kubectl kustomize config/crd | kubectl apply -f -

diff --git a/conformance/base/manifests.yaml b/conformance/base/manifests.yaml
@@ -0,0 +1,204 @@
+# This file contains the base resources that most conformance tests will rely
+# on. This includes 4 namespaces along with Server and Client Deployments in
+# each of them that can be used as backends and probes for testing traffic.
+# Create 4 namespaces
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: network-policy-conformance-gryffindor
+  labels:
+    conformance-house: gryffindor
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: network-policy-conformance-slytherin
+  labels:
+    conformance-house: slytherin
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: network-policy-conformance-hufflepuff
+  labels:
+    conformance-house: hufflepuff
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: network-policy-conformance-ravenclaw
+  labels:
+    conformance-house: ravenclaw
+# Create 4 deployments; 2 pods each under these namespaces
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: harry-potter
+  namespace: network-policy-conformance-gryffindor
+spec:
+  selector:
+    matchLabels:
+      conformance-house: gryffindor
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        conformance-house: gryffindor
+    spec:
+      containers:
+        - name: harry-potter-client
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+        - name: harry-potter-80
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 80"]
+        - name: harry-potter-8080
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 8080"]
+        - name: harry-potter-5353
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 5353"]
+        - name: harry-potter-53
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 53"]
+        - name: harry-potter-9003
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost porter"]
+          env:
+          - name: SERVE_SCTP_PORT_9003
+            value: "foo"
+        - name: harry-potter-9005
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost porter"]
+          env:
+          - name: SERVE_SCTP_PORT_9005
+            value: "foo"
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: draco-malfoy
+  namespace: network-policy-conformance-slytherin
+spec:
+  selector:
+    matchLabels:
+      conformance-house: slytherin
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        conformance-house: slytherin
+    spec:
+      containers:
+        - name: draco-malfoy-client
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+        - name: draco-malfoy-80
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 80"]
+        - name: draco-malfoy-8080
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 8080"]
+        - name: draco-malfoy-5353
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 5353"]
+        - name: draco-malfoy-53
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 53"]
+        - name: draco-malfoy-9003
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost porter"]
+          env:
+          - name: SERVE_SCTP_PORT_9003
+            value: "foo"
+        - name: draco-malfoy-9005
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost porter"]
+          env:
+          - name: SERVE_SCTP_PORT_9005
+            value: "foo"
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: cedric-diggory
+  namespace: network-policy-conformance-hufflepuff
+spec:
+  selector:
+    matchLabels:
+      conformance-house: hufflepuff
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        conformance-house: hufflepuff
+    spec:
+      containers:
+        - name: cedric-diggory-client
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+        - name: cedric-diggory-80
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 80"]
+        - name: cedric-diggory-8080
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 8080"]
+        - name: cedric-diggory-5353
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 5353"]
+        - name: cedric-diggory-53
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 53"]
+        - name: cedric-diggory-9003
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost porter"]
+          env:
+          - name: SERVE_SCTP_PORT_9003
+            value: "foo"
+        - name: cedric-diggory-9005
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost porter"]
+          env:
+          - name: SERVE_SCTP_PORT_9005
+            value: "foo"
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: luna-lovegood
+  namespace: network-policy-conformance-ravenclaw
+spec:
+  selector:
+    matchLabels:
+      conformance-house: ravenclaw
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        conformance-house: ravenclaw
+    spec:
+      containers:
+        - name: luna-lovegood-client
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+        - name: luna-lovegood-80
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 80"]
+        - name: luna-lovegood-8080
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 8080"]
+        - name: luna-lovegood-5353
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 5353"]
+        - name: luna-lovegood-53
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 53"]
+        - name: luna-lovegood-9003
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost porter"]
+          env:
+          - name: SERVE_SCTP_PORT_9003
+            value: "foo"
+        - name: luna-lovegood-9005
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost porter"]
+          env:
+          - name: SERVE_SCTP_PORT_9005
+            value: "foo"
diff --git a/conformance/conformance_test.go b/conformance/conformance_test.go
@@ -0,0 +1,90 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// conformance_test contains code to run the conformance tests. This is in its own package to avoid circular imports.
+package conformance_test
+
+import (
+	"flag"
+	"strings"
+	"testing"
+
+	"sigs.k8s.io/network-policy-api/apis/v1alpha1"
+	"sigs.k8s.io/network-policy-api/conformance/tests"
+	"sigs.k8s.io/network-policy-api/conformance/utils/flags"
+	"sigs.k8s.io/network-policy-api/conformance/utils/suite"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2econfig "k8s.io/kubernetes/test/e2e/framework/config"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+)
+
+func TestConformance(t *testing.T) {
+	cfg, err := config.GetConfig()
+	if err != nil {
+		t.Fatalf("Error loading Kubernetes config: %v", err)
+	}
+	client, err := client.New(cfg, client.Options{})
+	if err != nil {
+		t.Fatalf("Error initializing Kubernetes client: %v", err)
+	}
+
+	v1alpha1.AddToScheme(client.Scheme())
+
+	supportedFeatures := parseSupportedFeatures(*flags.SupportedFeatures)
+	exemptFeatures := parseSupportedFeatures(*flags.ExemptFeatures)
+
+	// Register test flags, then parse flags.
+	handleFlags()
+
+	t.Logf("Running conformance tests with cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]",
+		*flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures)
+
+	cSuite := suite.New(suite.Options{
+		Client:                     client,
+		Debug:                      *flags.ShowDebug,
+		CleanupBaseResources:       *flags.CleanupBaseResources,
+		SupportedFeatures:          supportedFeatures,
+		ExemptFeatures:             exemptFeatures,
+		EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
+	})
+	cSuite.Setup(t)
+
+	cSuite.Run(t, tests.ConformanceTests)
+}
+
+// parseSupportedFeatures parses flag arguments and converts the string to
+// sets.Set[suite.SupportedFeature]
+func parseSupportedFeatures(f string) sets.Set[suite.SupportedFeature] {
+	if f == "" {
+		return nil
+	}
+	res := sets.Set[suite.SupportedFeature]{}
+	for _, value := range strings.Split(f, ",") {
+		res.Insert(suite.SupportedFeature(value))
+	}
+	return res
+}
+
+// handleFlags sets up all flags and parses the command line.
+func handleFlags() {
+	e2econfig.CopyFlags(e2econfig.Flags, flag.CommandLine)
+	framework.RegisterCommonFlags(flag.CommandLine)
+	flag.Parse()
+}
diff --git a/conformance/embed.go b/conformance/embed.go
@@ -0,0 +1,22 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package conformance
+
+import "embed"
+
+//go:embed tests/* base/*
+var Manifests embed.FS
diff --git a/conformance/tests/main.go b/conformance/tests/main.go
@@ -0,0 +1,21 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tests
+
+import "sigs.k8s.io/network-policy-api/conformance/utils/suite"
+
+var ConformanceTests []suite.ConformanceTest