Add Conformance Test Framework

This commit adds the initial framework we need
to run conformance tests.
ALL CREDIT to sig-network-gatewaya-api community and
folks who added conformance there. We have blatantly
copied from them and adapted the suite to our needs.
https://github.com/kubernetes-sigs/gateway-api/tree/main/conformance

Signed-off-by: Surya Seetharaman <suryaseetharaman.9@gmail.com>

Makefile

@@ -26,6 +26,10 @@ help: ## Display this help.
 
 ##@ Development
 
+# Command-line flags passed to "go test" for the conformance
+# test. These are passed after the "-args" flag.
+CONFORMANCE_FLAGS ?=
+
 .PHONY: fmt
 fmt: ## Run go fmt against code.
 	go fmt ./...
@@ -47,6 +51,10 @@ verify:
 crd-e2e:
 	hack/crd-e2e.sh -v
 
+.PHONY: conformance
+conformance:
+	go test -v ./conformance/... -args ${CONFORMANCE_FLAGS}
+
 ##@ Deployment
 install: generate ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	kubectl kustomize config/crd | kubectl apply -f -

conformance/base/manifests.yaml

@@ -0,0 +1,135 @@
+# This file contains the base resources that most conformance tests will rely
+# on. This includes 4 namespaces along with Server and Client Deployments in
+# each of them that can be used as backends and probes for testing traffic.
+# Create 4 namespaces
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: network-policy-conformance-gryffindor
+  labels:
+    conformance-house: gryffindor
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: network-policy-conformance-slytherin
+  labels:
+    conformance-house: slytherin
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: network-policy-conformance-hufflepuff
+  labels:
+    conformance-house: hufflepuff
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: network-policy-conformance-ravenclaw
+  labels:
+    conformance-house: ravenclaw
+# Create 4 deployments; 2 pods each under these namespaces
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: harry-potter
+  namespace: network-policy-conformance-gryffindor
+spec:
+  selector:
+    matchLabels:
+      conformance-house: gryffindor
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        conformance-house: gryffindor
+    spec:
+      containers:
+        - name: harry-potter-client
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+        - name: harry-potter-80
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost netexec --http-port 80"]
+        - name: harry-potter-8080
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --tcp --http=false --port 8080"]
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: draco-malfoy
+  namespace: network-policy-conformance-slytherin
+spec:
+  selector:
+    matchLabels:
+      conformance-house: slytherin
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        conformance-house: slytherin
+    spec:
+      containers:
+        - name: draco-malfoy-client
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+        - name: draco-malfoy-80
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost netexec --http-port 80"]
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: cedric-diggory
+  namespace: network-policy-conformance-hufflepuff
+spec:
+  selector:
+    matchLabels:
+      conformance-house: hufflepuff
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        conformance-house: hufflepuff
+    spec:
+      containers:
+        - name: cedric-diggory-client
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+        - name: cedric-diggory-5353
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost netexec --udp-port 5353"]
+        - name: cedric-diggory-53
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost serve-hostname --udp --http=false --port 53"]
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: luna-lovegood
+  namespace: network-policy-conformance-ravenclaw
+spec:
+  selector:
+    matchLabels:
+      conformance-house: ravenclaw
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        conformance-house: ravenclaw
+    spec:
+      containers:
+        - name: luna-lovegood-client
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+        - name: luna-lovegood-9003
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost porter"]
+          env:
+          - name: SERVE_SCTP_PORT_9003
+            value: "foo"
+        - name: luna-lovegood-9005
+          image: registry.k8s.io/e2e-test-images/agnhost:2.43
+          command: ["/bin/bash", "-c", "/agnhost porter"]
+          env:
+          - name: SERVE_SCTP_PORT_9005
+            value: "foo"

conformance/conformance_test.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// conformance_test contains code to run the conformance tests. This is in its own package to avoid circular imports.
+package conformance_test
+
+import (
+	"flag"
+	"strings"
+	"testing"
+
+	"sigs.k8s.io/network-policy-api/apis/v1alpha1"
+	"sigs.k8s.io/network-policy-api/conformance/tests"
+	"sigs.k8s.io/network-policy-api/conformance/utils/flags"
+	"sigs.k8s.io/network-policy-api/conformance/utils/suite"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2econfig "k8s.io/kubernetes/test/e2e/framework/config"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+)
+
+func TestConformance(t *testing.T) {
+	cfg, err := config.GetConfig()
+	if err != nil {
+		t.Fatalf("Error loading Kubernetes config: %v", err)
+	}
+	client, err := client.New(cfg, client.Options{})
+	if err != nil {
+		t.Fatalf("Error initializing Kubernetes client: %v", err)
+	}
+
+	v1alpha1.AddToScheme(client.Scheme())
+
+	supportedFeatures := parseSupportedFeatures(*flags.SupportedFeatures)
+	exemptFeatures := parseSupportedFeatures(*flags.ExemptFeatures)
+
+	// Register test flags, then parse flags.
+	handleFlags()
+
+	t.Logf("Running conformance tests with cleanup: %t\n debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]",
+		*flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures)
+
+	cSuite := suite.New(suite.Options{
+		Client:                     client,
+		Debug:                      *flags.ShowDebug,
+		CleanupBaseResources:       *flags.CleanupBaseResources,
+		SupportedFeatures:          supportedFeatures,
+		ExemptFeatures:             exemptFeatures,
+		EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
+	})
+	cSuite.Setup(t)
+
+	cSuite.Run(t, tests.ConformanceTests)
+}
+
+// parseSupportedFeatures parses flag arguments and converts the string to
+// sets.Set[suite.SupportedFeature]
+func parseSupportedFeatures(f string) sets.Set[suite.SupportedFeature] {
+	if f == "" {
+		return nil
+	}
+	res := sets.Set[suite.SupportedFeature]{}
+	for _, value := range strings.Split(f, ",") {
+		res.Insert(suite.SupportedFeature(value))
+	}
+	return res
+}
+
+// handleFlags sets up all flags and parses the command line.
+func handleFlags() {
+	e2econfig.CopyFlags(e2econfig.Flags, flag.CommandLine)
+	framework.RegisterCommonFlags(flag.CommandLine)
+	flag.Parse()
+}

conformance/embed.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package conformance
+
+import "embed"
+
+//go:embed tests/* base/*
+var Manifests embed.FS

conformance/tests/main.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tests
+
+import "sigs.k8s.io/network-policy-api/conformance/utils/suite"
+
+var ConformanceTests []suite.ConformanceTest

conformance/utils/config/timeout.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import "time"
+
+type TimeoutConfig struct {
+	// CreateTimeout represents the maximum time for a Kubernetes object to be created.
+	// Max value for conformant implementation: None
+	CreateTimeout time.Duration
+
+	// DeleteTimeout represents the maximum time for a Kubernetes object to be deleted.
+	// Max value for conformant implementation: None
+	DeleteTimeout time.Duration
+
+	// GetTimeout represents the maximum time to get a Kubernetes object.
+	// Max value for conformant implementation: None
+	GetTimeout time.Duration
+
+	// ManifestFetchTimeout represents the maximum time for getting content from a https:// URL.
+	// Max value for conformant implementation: None
+	ManifestFetchTimeout time.Duration
+
+	// NamespacesMustBeReady represents the maximum time for the following to happen within
+	// specified namespace(s):
+	// * All Pods to be marked as "Ready"
+	// * All Gateways to be marked as "Accepted" and "Programmed"
+	// Max value for conformant implementation: None
+	NamespacesMustBeReady time.Duration
+
+	// RequestTimeout represents the maximum time before which the connection attempt from client to server will timeout.
+	// Max value for conformant implementation: None
+	RequestTimeout time.Duration
+}
+
+// DefaultTimeoutConfig populates a TimeoutConfig with the default values.
+func DefaultTimeoutConfig() TimeoutConfig {
+	return TimeoutConfig{
+		CreateTimeout:         60 * time.Second,
+		DeleteTimeout:         10 * time.Second,
+		GetTimeout:            10 * time.Second,
+		ManifestFetchTimeout:  10 * time.Second,
+		NamespacesMustBeReady: 300 * time.Second,
+		RequestTimeout:        3 * time.Second,
+	}
+}
+
+func SetupTimeoutConfig(timeoutConfig *TimeoutConfig) {
+	defaultTimeoutConfig := DefaultTimeoutConfig()
+	if timeoutConfig.CreateTimeout == 0 {
+		timeoutConfig.CreateTimeout = defaultTimeoutConfig.CreateTimeout
+	}
+	if timeoutConfig.DeleteTimeout == 0 {
+		timeoutConfig.DeleteTimeout = defaultTimeoutConfig.DeleteTimeout
+	}
+	if timeoutConfig.GetTimeout == 0 {
+		timeoutConfig.GetTimeout = defaultTimeoutConfig.GetTimeout
+	}
+	if timeoutConfig.ManifestFetchTimeout == 0 {
+		timeoutConfig.ManifestFetchTimeout = defaultTimeoutConfig.ManifestFetchTimeout
+	}
+	if timeoutConfig.NamespacesMustBeReady == 0 {
+		timeoutConfig.NamespacesMustBeReady = defaultTimeoutConfig.NamespacesMustBeReady
+	}
+	if timeoutConfig.RequestTimeout == 0 {
+		timeoutConfig.RequestTimeout = defaultTimeoutConfig.RequestTimeout
+	}
+}