[API Gateway] API Gateway Binding Logic (#2142)

* initial commit

* Add additional TODO

* Add some basic lifecycle unit tests

* split up implementation

* Add more tests and fix some bugs

* remove one parallel call in a loop

* Fix binding

* Add resolvedRefs statuses for routes

* Fix issue with empty parent ref that k8s doesn't like

* Fix up updates/status ordering

* Add basic gateway status setting

* Finish up first pass on gateway statuses

* Re-organize and begin adding comments

* More comments

* More comments

* More comments

* More comments

* More comments

* Add file that wasn't saved

* Add utils unit tests

* Add more tests

* Final tests

* Fix tests

* Fix up gateway annotation with binding logic

* Update doc comments for linter

* Add forgotten file

* Fix block in tests due to buffered channel size and better handle context cancelation

charts/consul/templates/connect-inject-clusterrole.yaml

@@ -137,12 +137,18 @@ rules:
   - gateway.networking.k8s.io
   resources:
   - gatewayclasses/finalizers
+  - gateways/finalizers
+  - httproutes/finalizers
+  - tcproutes/finalizers
   verbs:
   - update
 - apiGroups:
   - gateway.networking.k8s.io
   resources:
   - gatewayclasses/status
+  - gateways/status
+  - httproutes/status
+  - tcproutes/status
   verbs:
   - get
   - patch

control-plane/api-gateway/binding/annotations.go

@@ -0,0 +1,38 @@
+package binding
+
+import (
+	"encoding/json"
+
+	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+
+	"github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1"
+)
+
+const (
+	group               = "api-gateway.consul.hashicorp.com"
+	annotationConfigKey = "api-gateway.consul.hashicorp.com/config"
+)
+
+func serializeGatewayClassConfig(gw *gwv1beta1.Gateway, gwcc *v1alpha1.GatewayClassConfig) (*v1alpha1.GatewayClassConfig, bool) {
+	if gwcc == nil {
+		return nil, false
+	}
+
+	if gw.Annotations == nil {
+		gw.Annotations = make(map[string]string)
+	}
+
+	if annotatedConfig, ok := gw.Annotations[annotationConfigKey]; ok {
+		var config v1alpha1.GatewayClassConfig
+		if err := json.Unmarshal([]byte(annotatedConfig), &config.Spec); err == nil {
+			// if we can unmarshal the gateway, return it
+			return &config, false
+		}
+	}
+
+	// otherwise if we failed to unmarshal or there was no annotation, marshal it onto
+	// the gateway
+	marshaled, _ := json.Marshal(gwcc.Spec)
+	gw.Annotations[annotationConfigKey] = string(marshaled)
+	return gwcc, true
+}

control-plane/api-gateway/binding/annotations_test.go

@@ -0,0 +1,203 @@
+package binding
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+
+	"github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1"
+)
+
+func TestSerializeGatewayClassConfig_HappyPath(t *testing.T) {
+	t.Parallel()
+
+	type args struct {
+		gw   *gwv1beta1.Gateway
+		gwcc *v1alpha1.GatewayClassConfig
+	}
+	tests := []struct {
+		name              string
+		args              args
+		expectedDidUpdate bool
+	}{
+		{
+			name: "when gateway has not been annotated yet and annotations are nil",
+			args: args{
+				gw: &gwv1beta1.Gateway{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "my-gw",
+					},
+					Spec:   gwv1beta1.GatewaySpec{},
+					Status: gwv1beta1.GatewayStatus{},
+				},
+				gwcc: &v1alpha1.GatewayClassConfig{
+					TypeMeta: metav1.TypeMeta{},
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "the config",
+					},
+					Spec: v1alpha1.GatewayClassConfigSpec{
+						ServiceType: pointerTo(corev1.ServiceType("serviceType")),
+						NodeSelector: map[string]string{
+							"selector": "of node",
+						},
+						Tolerations: []v1.Toleration{
+							{
+								Key:               "key",
+								Operator:          "op",
+								Value:             "120",
+								Effect:            "to the moon",
+								TolerationSeconds: new(int64),
+							},
+						},
+						CopyAnnotations: v1alpha1.CopyAnnotationsSpec{
+							Service: []string{"service"},
+						},
+					},
+				},
+			},
+			expectedDidUpdate: true,
+		},
+		{
+			name: "when gateway has not been annotated yet but annotations are empty",
+			args: args{
+				gw: &gwv1beta1.Gateway{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:        "my-gw",
+						Annotations: make(map[string]string),
+					},
+					Spec:   gwv1beta1.GatewaySpec{},
+					Status: gwv1beta1.GatewayStatus{},
+				},
+				gwcc: &v1alpha1.GatewayClassConfig{
+					TypeMeta: metav1.TypeMeta{},
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "the config",
+					},
+					Spec: v1alpha1.GatewayClassConfigSpec{
+						ServiceType: pointerTo(corev1.ServiceType("serviceType")),
+						NodeSelector: map[string]string{
+							"selector": "of node",
+						},
+						Tolerations: []v1.Toleration{
+							{
+								Key:               "key",
+								Operator:          "op",
+								Value:             "120",
+								Effect:            "to the moon",
+								TolerationSeconds: new(int64),
+							},
+						},
+						CopyAnnotations: v1alpha1.CopyAnnotationsSpec{
+							Service: []string{"service"},
+						},
+					},
+				},
+			},
+			expectedDidUpdate: true,
+		},
+		{
+			name: "when gateway has been annotated",
+			args: args{
+				gw: &gwv1beta1.Gateway{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "my-gw",
+						Annotations: map[string]string{
+							annotationConfigKey: `{"serviceType":"serviceType","nodeSelector":{"selector":"of node"},"tolerations":[{"key":"key","operator":"op","value":"120","effect":"to the moon","tolerationSeconds":0}],"copyAnnotations":{"service":["service"]}}`,
+						},
+					},
+					Spec:   gwv1beta1.GatewaySpec{},
+					Status: gwv1beta1.GatewayStatus{},
+				},
+				gwcc: &v1alpha1.GatewayClassConfig{
+					TypeMeta: metav1.TypeMeta{},
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "the config",
+					},
+					Spec: v1alpha1.GatewayClassConfigSpec{
+						ServiceType: pointerTo(corev1.ServiceType("serviceType")),
+						NodeSelector: map[string]string{
+							"selector": "of node",
+						},
+						Tolerations: []v1.Toleration{
+							{
+								Key:               "key",
+								Operator:          "op",
+								Value:             "120",
+								Effect:            "to the moon",
+								TolerationSeconds: new(int64),
+							},
+						},
+						CopyAnnotations: v1alpha1.CopyAnnotationsSpec{
+							Service: []string{"service"},
+						},
+					},
+				},
+			},
+			expectedDidUpdate: false,
+		},
+		{
+			name: "when gateway has been annotated but the serialization was invalid",
+			args: args{
+				gw: &gwv1beta1.Gateway{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "my-gw",
+						Annotations: map[string]string{
+							// we remove the opening brace to make unmarshalling fail
+							annotationConfigKey: `"serviceType":"serviceType","nodeSelector":{"selector":"of node"},"tolerations":[{"key":"key","operator":"op","value":"120","effect":"to the moon","tolerationSeconds":0}],"copyAnnotations":{"service":["service"]}}`,
+						},
+					},
+					Spec:   gwv1beta1.GatewaySpec{},
+					Status: gwv1beta1.GatewayStatus{},
+				},
+				gwcc: &v1alpha1.GatewayClassConfig{
+					TypeMeta: metav1.TypeMeta{},
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "the config",
+					},
+					Spec: v1alpha1.GatewayClassConfigSpec{
+						ServiceType: pointerTo(corev1.ServiceType("serviceType")),
+						NodeSelector: map[string]string{
+							"selector": "of node",
+						},
+						Tolerations: []v1.Toleration{
+							{
+								Key:               "key",
+								Operator:          "op",
+								Value:             "120",
+								Effect:            "to the moon",
+								TolerationSeconds: new(int64),
+							},
+						},
+						CopyAnnotations: v1alpha1.CopyAnnotationsSpec{
+							Service: []string{"service"},
+						},
+					},
+				},
+			},
+			expectedDidUpdate: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, actualDidUpdate := serializeGatewayClassConfig(tt.args.gw, tt.args.gwcc)
+
+			if actualDidUpdate != tt.expectedDidUpdate {
+				t.Errorf("SerializeGatewayClassConfig() = %v, want %v", actualDidUpdate, tt.expectedDidUpdate)
+			}
+
+			var config v1alpha1.GatewayClassConfig
+			err := json.Unmarshal([]byte(tt.args.gw.Annotations[annotationConfigKey]), &config.Spec)
+			require.NoError(t, err)
+
+			if diff := cmp.Diff(config.Spec, tt.args.gwcc.Spec); diff != "" {
+				t.Errorf("Expected gwconfig spec to match serialized version (-want,+got):\n%s", diff)
+			}
+		})
+	}
+}