feat: allow assigning KongPluginInstallation to GatewayConfiguration …

…and DataPlane (#476)


Co-authored-by: Patryk Małek <patryk.malek@konghq.com>
Co-authored-by: Grzegorz Burzyński <czeslavo@gmail.com>

CHANGELOG.md

@@ -31,7 +31,7 @@
   [#387](https://github.com/Kong/gateway-operator/pull/387)
 - Introduce `KongPluginInstallation` CRD to allow installing custom Kong
   plugins distributed as container images.
-  [#400](https://github.com/Kong/gateway-operator/pull/400), [#424](https://github.com/Kong/gateway-operator/pull/424), [#474](https://github.com/Kong/gateway-operator/pull/474), [#560](https://github.com/Kong/gateway-operator/pull/560), [#615](https://github.com/Kong/gateway-operator/pull/615)
+  [#400](https://github.com/Kong/gateway-operator/pull/400), [#424](https://github.com/Kong/gateway-operator/pull/424), [#474](https://github.com/Kong/gateway-operator/pull/474), [#560](https://github.com/Kong/gateway-operator/pull/560), [#615](https://github.com/Kong/gateway-operator/pull/615), [#476](https://github.com/Kong/gateway-operator/pull/476)
 - Extended `DataPlane` API with a possibility to specify `PodDisruptionBudget` to be
   created for the `DataPlane` deployments via `spec.resources.podDisruptionBudget`.
   [#464](https://github.com/Kong/gateway-operator/pull/464)

api/v1beta1/dataplane_types.go

@@ -80,6 +80,10 @@ type DataPlaneOptions struct {
 	// +kubebuilder:validation:MinItems=0
 	// +kubebuilder:validation:MaxItems=1
 	Extensions []v1alpha1.ExtensionRef `json:"extensions,omitempty"`
+	// PluginsToInstall is a list of KongPluginInstallation resources that
+	// will be installed and available in the DataPlane.
+	// +optional
+	PluginsToInstall []NamespacedName `json:"pluginsToInstall,omitempty"`
 }
 
 // DataPlaneResources defines the resources that will be created and managed

api/v1beta1/gatewayconfiguration_types.go

@@ -73,6 +73,11 @@ type GatewayConfigDataPlaneOptions struct {
 	// +kubebuilder:validation:MinItems=0
 	// +kubebuilder:validation:MaxItems=1
 	Extensions []v1alpha1.ExtensionRef `json:"extensions,omitempty"`
+	// PluginsToInstall is a list of KongPluginInstallation resources that
+	// will be installed and available in the Gateways (DataPlanes) that
+	// use this GatewayConfig.
+	// +optional
+	PluginsToInstall []NamespacedName `json:"pluginsToInstall,omitempty"`
 }
 
 // GatewayConfigDataPlaneNetworkOptions defines network related options for a DataPlane.

config/crd/bases/gateway-operator.konghq.com_dataplanes.yaml

@@ -8943,6 +8943,22 @@ spec:
                         type: object
                     type: object
                 type: object
+              pluginsToInstall:
+                description: |-
+                  PluginsToInstall is a list of KongPluginInstallation resources that
+                  will be installed and available in the DataPlane.
+                items:
+                  description: NamespacedName is a resource identified by name and
+                    optional namespace.
+                  properties:
+                    name:
+                      type: string
+                    namespace:
+                      type: string
+                  required:
+                  - name
+                  type: object
+                type: array
               resources:
                 description: |-
                   DataPlaneResources defines the resources that will be created and managed

config/crd/bases/gateway-operator.konghq.com_gatewayconfigurations.yaml

@@ -17174,6 +17174,23 @@ spec:
                             type: object
                         type: object
                     type: object
+                  pluginsToInstall:
+                    description: |-
+                      PluginsToInstall is a list of KongPluginInstallation resources that
+                      will be installed and available in the Gateways (DataPlanes) that
+                      use this GatewayConfig.
+                    items:
+                      description: NamespacedName is a resource identified by name
+                        and optional namespace.
+                      properties:
+                        name:
+                          type: string
+                        namespace:
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    type: array
                 type: object
             type: object
           status:

config/crd/dataplane/gateway-operator.konghq.com_dataplanes.yaml

@@ -8943,6 +8943,22 @@ spec:
                         type: object
                     type: object
                 type: object
+              pluginsToInstall:
+                description: |-
+                  PluginsToInstall is a list of KongPluginInstallation resources that
+                  will be installed and available in the DataPlane.
+                items:
+                  description: NamespacedName is a resource identified by name and
+                    optional namespace.
+                  properties:
+                    name:
+                      type: string
+                    namespace:
+                      type: string
+                  required:
+                  - name
+                  type: object
+                type: array
               resources:
                 description: |-
                   DataPlaneResources defines the resources that will be created and managed

config/samples/gateway-kongplugininstallation-httproute.yaml

@@ -0,0 +1,170 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: additional
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: echo
+spec:
+  ports:
+    - protocol: TCP
+      name: http
+      port: 80
+      targetPort: http
+  selector:
+    app: echo
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: echo
+  name: echo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: echo
+  template:
+    metadata:
+      labels:
+        app: echo
+    spec:
+      containers:
+        - name: echo
+          image: registry.k8s.io/e2e-test-images/agnhost:2.40
+          command:
+            - /agnhost
+            - netexec
+            - --http-port=8080
+          ports:
+            - containerPort: 8080
+              name: http
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          resources:
+            requests:
+              cpu: 10m
+---
+kind: GatewayConfiguration
+apiVersion: gateway-operator.konghq.com/v1beta1
+metadata:
+  name: kong
+  namespace: default
+spec:
+  dataPlaneOptions:
+    deployment:
+      replicas: 2
+      podTemplateSpec:
+        spec:
+          containers:
+            - name: proxy
+              # renovate: datasource=docker versioning=docker
+              image: kong/kong-gateway:3.7
+              readinessProbe:
+                initialDelaySeconds: 1
+                periodSeconds: 1
+    pluginsToInstall:
+      - name: additional-custom-plugin
+        namespace: additional
+      - name: additional-custom-plugin-2
+  controlPlaneOptions:
+    deployment:
+      podTemplateSpec:
+        spec:
+          containers:
+            - name: controller
+              # renovate: datasource=docker versioning=docker
+              image: kong/kubernetes-ingress-controller:3.2.3
+              readinessProbe:
+                initialDelaySeconds: 1
+                periodSeconds: 1
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: GatewayClass
+metadata:
+  name: kong
+spec:
+  controllerName: konghq.com/gateway-operator
+  parametersRef:
+    group: gateway-operator.konghq.com
+    kind: GatewayConfiguration
+    name: kong
+    namespace: default
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: kong
+  namespace: default
+spec:
+  gatewayClassName: kong
+  listeners:
+    - name: http
+      protocol: HTTP
+      port: 80
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: httproute-echo
+  namespace: default
+  annotations:
+    konghq.com/strip-path: "true"
+    konghq.com/plugins: kong-custom-plugin,kong-custom-plugin-2
+spec:
+  parentRefs:
+    - name: kong
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /echo
+      backendRefs:
+        - name: echo
+          kind: Service
+          port: 80
+---
+apiVersion: configuration.konghq.com/v1
+kind: KongPlugin
+metadata:
+  name: kong-custom-plugin
+plugin: additional-custom-plugin
+---
+kind: KongPluginInstallation
+apiVersion: gateway-operator.konghq.com/v1alpha1
+metadata:
+  name: additional-custom-plugin
+  namespace: additional
+spec:
+  image: northamerica-northeast1-docker.pkg.dev/k8s-team-playground/plugin-example/myheader
+---
+apiVersion: configuration.konghq.com/v1
+kind: KongPlugin
+metadata:
+  name: kong-custom-plugin-2
+plugin: additional-custom-plugin-2
+---
+kind: KongPluginInstallation
+apiVersion: gateway-operator.konghq.com/v1alpha1
+metadata:
+  name: additional-custom-plugin-2
+spec:
+  image: northamerica-northeast1-docker.pkg.dev/k8s-team-playground/plugin-example/myheader-2

controller/dataplane/controller.go

@@ -86,7 +86,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	if err != nil {
 		log.Info(logger, "failed to validate dataplane: "+err.Error(), dataplane)
 		r.eventRecorder.Event(dataplane, "Warning", "ValidationFailed", err.Error())
-		markErr := r.ensureDataPlaneIsMarkedNotReady(ctx, logger, dataplane, DataPlaneConditionValidationFailed, err.Error())
+		markErr := ensureDataPlaneIsMarkedNotReady(ctx, logger, r.Client, dataplane, DataPlaneConditionValidationFailed, err.Error())
 		return ctrl.Result{}, markErr
 	}
 
@@ -162,7 +162,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, nil // no need to requeue, the update will trigger.
 	}
 
-	log.Trace(logger, "ensuring DataPlane has service addesses in status", dataplaneIngressService)
+	log.Trace(logger, "ensuring DataPlane has service addresses in status", dataplaneIngressService)
 	if updated, err := r.ensureDataPlaneAddressesStatus(ctx, logger, dataplane, dataplaneIngressService); err != nil {
 		return ctrl.Result{}, err
 	} else if updated {
@@ -176,6 +176,17 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	deploymentOpts := []k8sresources.DeploymentOpt{
 		labelSelectorFromDataPlaneStatusSelectorDeploymentOpt(dataplane),
 	}
+
+	log.Trace(logger, "ensuring generation of deployment configuration for KongPluginInstallations configured for DataPlane", dataplane)
+	kpisForDeployment, requeue, err := ensureMappedConfigMapToKongPluginInstallationForDataPlane(ctx, logger, r.Client, dataplane)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("cannot ensure KongPluginInstallation for DataPlane: %w", err)
+	}
+	if requeue {
+		return ctrl.Result{Requeue: true}, nil
+	}
+	deploymentOpts = append(deploymentOpts, withCustomPlugins(kpisForDeployment...))
+
 	deploymentBuilder := NewDeploymentBuilder(logger.WithName("deployment_builder"), r.Client).
 		WithBeforeCallbacks(r.Callbacks.BeforeDeployment).
 		WithAfterCallbacks(r.Callbacks.AfterDeployment).
@@ -186,8 +197,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 
 	deployment, res, err := deploymentBuilder.BuildAndDeploy(ctx, dataplane, r.DevelopmentMode)
 	if err != nil {
-		return ctrl.Result{}, fmt.Errorf("could not build Deployment for DataPlane %s: %w",
-			dpNn, err)
+		return ctrl.Result{}, fmt.Errorf("could not build Deployment for DataPlane %s: %w", dpNn, err)
 	}
 	if res != op.Noop {
 		return ctrl.Result{}, nil

controller/dataplane/controller_conditions.go

@@ -10,4 +10,9 @@ const (
 	// DataPlaneConditionValidationFailed is a reason which indicates validation of
 	// a dataplane is failed.
 	DataPlaneConditionValidationFailed consts.ConditionReason = "ValidationFailed"
+
+	// DataPlaneConditionReferencedResourcesNotAvailable is a reason which indicates
+	// that the referenced resources in DataPlane configuration (e.g. KongPluginInstallation)
+	// are not available.
+	DataPlaneConditionReferencedResourcesNotAvailable consts.ConditionReason = "ReferencedResourcesNotAvailable"
 )