Add Gatekeeper for managing gateway deployment resources

charts/consul/values.yaml

@@ -1938,6 +1938,90 @@ connectInject:
     # @type: integer
     minAvailable: null
 
+  # Configuration settings for the Consul API Gateway integration.
+  apiGateway:
+    # Configuration settings for the optional GatewayClass installed by Consul on Kubernetes.
+    managedGatewayClass:
+      # This value defines [`nodeSelector`](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector)
+      # labels for gateway pod assignment, formatted as a multi-line string.
+      #
+      # Example:
+      #
+      # ```yaml
+      # nodeSelector: |
+      #   beta.kubernetes.io/arch: amd64
+      # ```
+      #
+      # @type: string
+      nodeSelector: null
+
+      # Toleration settings for gateway pods created with the managed gateway class. 
+      # This should be a multi-line string matching the
+      # [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec.
+      #
+      # @type: string
+      tolerations: null
+
+      # This value defines the type of Service created for gateways (e.g. LoadBalancer, ClusterIP)
+      serviceType: LoadBalancer
+
+      # This value toggles if the gateway ports should be mapped to host ports
+      useHostPorts: false
+
+      # Configuration settings for annotations to be copied from the Gateway to other child resources.
+      copyAnnotations:
+        # This value defines a list of annotations to be copied from the Gateway to the Service created, formatted as a multi-line string.
+        #
+        # Example:
+        #
+        # ```yaml
+        # service:
+        #   annotations: |
+        #     - external-dns.alpha.kubernetes.io/hostname
+        # ```
+        #
+        # @type: string
+        service: null
+
+      # This value defines the number of pods to deploy for each Gateway as well as a min and max number of pods for all Gateways
+      #
+      # Example:
+      #
+      # ```yaml
+      # deployment:
+      #   defaultInstances: 3
+      #   maxInstances: 8
+      #   minInstances: 1
+      # ```
+      #
+      # @type: map
+      deployment: null
+
+    # Configuration for the ServiceAccount created for the api-gateway component
+    serviceAccount:
+      # This value defines additional annotations for the client service account. This should be formatted as a multi-line
+      # string.
+      #
+      # ```yaml
+      # annotations: |
+      #   "sample/annotation1": "foo"
+      #   "sample/annotation2": "bar"
+      # ```
+      #
+      # @type: string
+      annotations: null
+
+    # The resource settings for Pods handling traffic for Gateway API.
+    # @recurse: false
+    # @type: map
+    resources:
+      requests:
+        memory: "100Mi"
+        cpu: "100m"
+      limits:
+        memory: "100Mi"
+        cpu: "100m"
+
   # Configures consul-cni plugin for Consul Service mesh services
   cni:
     # If true, then all traffic redirection setup uses the consul-cni plugin.
@@ -2867,6 +2951,7 @@ terminatingGateways:
   gateways:
   - name: terminating-gateway
 
+# [DEPRECATED] Use connectInject.apiGateway instead.
 # Configuration settings for the Consul API Gateway integration
 apiGateway:
   # When true the helm chart will install the Consul API Gateway controller

control-plane/api-gateway/controllers/gateway_controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/go-logr/logr"
+	apigateway "github.com/hashicorp/consul-k8s/control-plane/api-gateway"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -41,8 +42,9 @@ type GatewayControllerConfig struct {
 // GatewayController reconciles a Gateway object.
 // The Gateway is responsible for defining the behavior of API gateways.
 type GatewayController struct {
-	cache *cache.Cache
-	Log   logr.Logger
+	HelmConfig apigateway.HelmConfig
+	Log        logr.Logger
+	cache      *cache.Cache
 	client.Client
 }
 

control-plane/api-gateway/controllers/gateway_controller_test.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 )
 

control-plane/api-gateway/gatekeeper/deployment.go

@@ -0,0 +1,159 @@
+package gatekeeper
+
+import (
+	"context"
+
+	apigateway "github.com/hashicorp/consul-k8s/control-plane/api-gateway"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+)
+
+func (g *Gatekeeper) upsertDeployment(ctx context.Context) error {
+	// Get Deployment if it exists.
+	existingDeployment := &appsv1.Deployment{}
+	exists := false
+
+	err := g.Client.Get(ctx, g.namespacedName(), existingDeployment)
+	if err != nil && !k8serrors.IsNotFound(err) {
+		return err
+	} else if k8serrors.IsNotFound(err) {
+		exists = false
+	} else {
+		exists = true
+	}
+
+	deployment := g.deployment()
+
+	if exists {
+		g.Log.Info("Existing Gateway Deployment found.")
+
+		// If the user has set the number of replicas, let's respect that.
+		deployment.Spec.Replicas = existingDeployment.Spec.Replicas
+	}
+
+	mutated := deployment.DeepCopy()
+	mutator := newDeploymentMutator(deployment, mutated, g.Gateway, g.Client.Scheme())
+
+	result, err := controllerutil.CreateOrUpdate(ctx, g.Client, mutated, mutator)
+	if err != nil {
+		return err
+	}
+
+	switch result {
+	case controllerutil.OperationResultCreated:
+		g.Log.Info("Created Deployment")
+	case controllerutil.OperationResultUpdated:
+		g.Log.Info("Updated Deployment")
+	case controllerutil.OperationResultNone:
+		g.Log.Info("No change to deployment")
+	}
+
+	return nil
+}
+
+func (g *Gatekeeper) deleteDeployment(ctx context.Context) error {
+	err := g.Client.Delete(ctx, g.deployment())
+	if k8serrors.IsNotFound(err) {
+		return nil
+	}
+
+	return err
+}
+
+func (g *Gatekeeper) deployment() *appsv1.Deployment {
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        g.Gateway.Name,
+			Namespace:   g.Gateway.Namespace,
+			Labels:      apigateway.LabelsForGateway(&g.Gateway),
+			Annotations: g.HelmConfig.CopyAnnotations,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: &g.HelmConfig.Replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: apigateway.LabelsForGateway(&g.Gateway),
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: apigateway.LabelsForGateway(&g.Gateway),
+					Annotations: map[string]string{
+						"consul.hashicorp.com/connect-inject": "false",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Image: g.HelmConfig.Image,
+						},
+					},
+					Affinity: &corev1.Affinity{
+						PodAntiAffinity: &corev1.PodAntiAffinity{
+							PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+								{
+									Weight: 1,
+									PodAffinityTerm: corev1.PodAffinityTerm{
+										LabelSelector: &metav1.LabelSelector{
+											MatchLabels: apigateway.LabelsForGateway(&g.Gateway),
+										},
+										TopologyKey: "kubernetes.io/hostname",
+									},
+								},
+							},
+						},
+					},
+					NodeSelector:       g.GatewayClassConfig.Spec.NodeSelector, // TODO should I grab this from here or Helm?
+					Tolerations:        g.GatewayClassConfig.Spec.Tolerations,
+					ServiceAccountName: g.serviceAccountName(),
+				},
+			},
+		},
+	}
+}
+
+func mergeDeployments(a, b *appsv1.Deployment) *appsv1.Deployment {
+	if !compareDeployments(a, b) {
+		b.Spec.Template = a.Spec.Template
+		b.Spec.Replicas = a.Spec.Replicas
+	}
+
+	return b
+}
+
+func compareDeployments(a, b *appsv1.Deployment) bool {
+	// since K8s adds a bunch of defaults when we create a deployment, check that
+	// they don't differ by the things that we may actually change, namely container
+	// ports
+	if len(b.Spec.Template.Spec.Containers) != len(a.Spec.Template.Spec.Containers) {
+		return false
+	}
+	for i, container := range a.Spec.Template.Spec.Containers {
+		otherPorts := b.Spec.Template.Spec.Containers[i].Ports
+		if len(container.Ports) != len(otherPorts) {
+			return false
+		}
+		for j, port := range container.Ports {
+			otherPort := otherPorts[j]
+			if port.ContainerPort != otherPort.ContainerPort {
+				return false
+			}
+			if port.Protocol != otherPort.Protocol {
+				return false
+			}
+		}
+	}
+
+	return *b.Spec.Replicas == *a.Spec.Replicas
+}
+
+func newDeploymentMutator(deployment, mutated *appsv1.Deployment, gateway gwv1beta1.Gateway, scheme *runtime.Scheme) resourceMutator {
+	return func() error {
+		mutated = mergeDeployments(deployment, mutated)
+		return ctrl.SetControllerReference(&gateway, mutated, scheme)
+	}
+}

control-plane/api-gateway/gatekeeper/gatekeeper.go

@@ -0,0 +1,95 @@
+package gatekeeper
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	apigateway "github.com/hashicorp/consul-k8s/control-plane/api-gateway"
+	"github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+)
+
+// Gatekeeper is used to manage the lifecycle of Gateway deployments and services.
+type Gatekeeper struct {
+	Log    logr.Logger
+	Client client.Client
+
+	Gateway            gwv1beta1.Gateway
+	GatewayClassConfig v1alpha1.GatewayClassConfig
+	HelmConfig         apigateway.HelmConfig
+}
+
+// New creates a new Gatekeeper from the Config.
+func New(log logr.Logger, client client.Client, gateway gwv1beta1.Gateway, gatewayClassConfig v1alpha1.GatewayClassConfig, helmConfig apigateway.HelmConfig) *Gatekeeper {
+	return &Gatekeeper{
+		Log:                log,
+		Client:             client,
+		Gateway:            gateway,
+		GatewayClassConfig: gatewayClassConfig,
+		HelmConfig:         helmConfig,
+	}
+}
+
+// Upsert creates or updates the resources for handling routing of network traffic.
+func (g *Gatekeeper) Upsert(ctx context.Context) error {
+	g.Log.Info(fmt.Sprintf("Upsert Gateway Deployment %s/%s", g.Gateway.Namespace, g.Gateway.Name))
+
+	if err := g.upsertRole(ctx); err != nil {
+		return err
+	}
+
+	if err := g.upsertServiceAccount(ctx); err != nil {
+		return err
+	}
+
+	if err := g.upsertService(ctx); err != nil {
+		return err
+	}
+
+	if err := g.upsertDeployment(ctx); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Delete removes the resources for handling routing of network traffic.
+func (g *Gatekeeper) Delete(ctx context.Context) error {
+	if err := g.deleteRole(ctx); err != nil {
+		return err
+	}
+
+	if err := g.deleteServiceAccount(ctx); err != nil {
+		return err
+	}
+
+	if err := g.deleteService(ctx); err != nil {
+		return err
+	}
+
+	if err := g.deleteDeployment(ctx); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// resourceMutator is passed to create or update functions to mutate Kubernetes resources.
+type resourceMutator = func() error
+
+func (g Gatekeeper) namespacedName() types.NamespacedName {
+	return types.NamespacedName{
+		Namespace: g.Gateway.Namespace,
+		Name:      g.Gateway.Name,
+	}
+}
+
+func (g Gatekeeper) serviceAccountName() string {
+	authspecaccount := "" // TODO do I need to add this to GatewayClassConfig?
+	fmt.Println(authspecaccount)
+
+	return ""
+}