From c4742a824d834a39d1a4ce51330aec3f9a572a44 Mon Sep 17 00:00:00 2001 From: Jakub Warczarek Date: Mon, 12 Aug 2024 13:39:41 +0200 Subject: [PATCH 1/2] feat: allow assigning KongPluginInstallation to GatewayConfiguration and DataPlane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Patryk Małek Co-authored-by: Grzegorz Burzyński --- CHANGELOG.md | 2 +- api/v1beta1/dataplane_types.go | 4 + api/v1beta1/gatewayconfiguration_types.go | 5 + api/v1beta1/zz_generated.deepcopy.go | 10 + ...ateway-operator.konghq.com_dataplanes.yaml | 16 + ...ator.konghq.com_gatewayconfigurations.yaml | 17 + ...ateway-operator.konghq.com_dataplanes.yaml | 16 + ...eway-kongplugininstallation-httproute.yaml | 170 ++++++++ controller/dataplane/controller.go | 24 +- controller/dataplane/controller_conditions.go | 5 + .../dataplane/controller_reconciler_utils.go | 186 ++++++++- controller/dataplane/owned_custom_plugins.go | 81 ++++ .../dataplane/owned_custom_plugins_test.go | 185 +++++++++ controller/dataplane/watch.go | 64 +++- controller/gateway/controller.go | 2 +- .../gateway/controller_reconciler_utils.go | 17 +- .../kongplugininstallation/controller.go | 2 +- docs/api-reference.md | 6 + internal/utils/dataplane/config.go | 35 +- pkg/consts/kongplugininstallation.go | 8 +- pkg/utils/kubernetes/resources/annotations.go | 2 +- pkg/utils/test/clients.go | 15 +- .../test_kongplugininstallation.go | 362 +++++++++++++----- 23 files changed, 1106 insertions(+), 128 deletions(-) create mode 100644 config/samples/gateway-kongplugininstallation-httproute.yaml create mode 100644 controller/dataplane/owned_custom_plugins.go create mode 100644 controller/dataplane/owned_custom_plugins_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 987cc0d04..0f964de10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ [#387](https://github.com/Kong/gateway-operator/pull/387) - Introduce `KongPluginInstallation` CRD to allow installing custom Kong plugins distributed as container images. - [#400](https://github.com/Kong/gateway-operator/pull/400), [#424](https://github.com/Kong/gateway-operator/pull/424), [#474](https://github.com/Kong/gateway-operator/pull/474), [#560](https://github.com/Kong/gateway-operator/pull/560), [#615](https://github.com/Kong/gateway-operator/pull/615) + [#400](https://github.com/Kong/gateway-operator/pull/400), [#424](https://github.com/Kong/gateway-operator/pull/424), [#474](https://github.com/Kong/gateway-operator/pull/474), [#560](https://github.com/Kong/gateway-operator/pull/560), [#615](https://github.com/Kong/gateway-operator/pull/615), [#476](https://github.com/Kong/gateway-operator/pull/476) - Extended `DataPlane` API with a possibility to specify `PodDisruptionBudget` to be created for the `DataPlane` deployments via `spec.resources.podDisruptionBudget`. [#464](https://github.com/Kong/gateway-operator/pull/464) diff --git a/api/v1beta1/dataplane_types.go b/api/v1beta1/dataplane_types.go index 7e4ed597c..4333a476b 100644 --- a/api/v1beta1/dataplane_types.go +++ b/api/v1beta1/dataplane_types.go @@ -80,6 +80,10 @@ type DataPlaneOptions struct { // +kubebuilder:validation:MinItems=0 // +kubebuilder:validation:MaxItems=1 Extensions []v1alpha1.ExtensionRef `json:"extensions,omitempty"` + // PluginsToInstall is a list of KongPluginInstallation resources that + // will be installed and available in the DataPlane. + // +optional + PluginsToInstall []NamespacedName `json:"pluginsToInstall,omitempty"` } // DataPlaneResources defines the resources that will be created and managed diff --git a/api/v1beta1/gatewayconfiguration_types.go b/api/v1beta1/gatewayconfiguration_types.go index 9bfeffe3f..376227972 100644 --- a/api/v1beta1/gatewayconfiguration_types.go +++ b/api/v1beta1/gatewayconfiguration_types.go @@ -73,6 +73,11 @@ type GatewayConfigDataPlaneOptions struct { // +kubebuilder:validation:MinItems=0 // +kubebuilder:validation:MaxItems=1 Extensions []v1alpha1.ExtensionRef `json:"extensions,omitempty"` + // PluginsToInstall is a list of KongPluginInstallation resources that + // will be installed and available in the Gateways (DataPlanes) that + // use this GatewayConfig. + // +optional + PluginsToInstall []NamespacedName `json:"pluginsToInstall,omitempty"` } // GatewayConfigDataPlaneNetworkOptions defines network related options for a DataPlane. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 5b1052697..b4c0f42b1 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -346,6 +346,11 @@ func (in *DataPlaneOptions) DeepCopyInto(out *DataPlaneOptions) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.PluginsToInstall != nil { + in, out := &in.PluginsToInstall, &out.PluginsToInstall + *out = make([]NamespacedName, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataPlaneOptions. @@ -619,6 +624,11 @@ func (in *GatewayConfigDataPlaneOptions) DeepCopyInto(out *GatewayConfigDataPlan (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.PluginsToInstall != nil { + in, out := &in.PluginsToInstall, &out.PluginsToInstall + *out = make([]NamespacedName, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayConfigDataPlaneOptions. diff --git a/config/crd/bases/gateway-operator.konghq.com_dataplanes.yaml b/config/crd/bases/gateway-operator.konghq.com_dataplanes.yaml index 4ee8b3762..247dcc14e 100644 --- a/config/crd/bases/gateway-operator.konghq.com_dataplanes.yaml +++ b/config/crd/bases/gateway-operator.konghq.com_dataplanes.yaml @@ -8943,6 +8943,22 @@ spec: type: object type: object type: object + pluginsToInstall: + description: |- + PluginsToInstall is a list of KongPluginInstallation resources that + will be installed and available in the DataPlane. + items: + description: NamespacedName is a resource identified by name and + optional namespace. + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array resources: description: |- DataPlaneResources defines the resources that will be created and managed diff --git a/config/crd/bases/gateway-operator.konghq.com_gatewayconfigurations.yaml b/config/crd/bases/gateway-operator.konghq.com_gatewayconfigurations.yaml index b51a5ed06..07bdff721 100644 --- a/config/crd/bases/gateway-operator.konghq.com_gatewayconfigurations.yaml +++ b/config/crd/bases/gateway-operator.konghq.com_gatewayconfigurations.yaml @@ -17174,6 +17174,23 @@ spec: type: object type: object type: object + pluginsToInstall: + description: |- + PluginsToInstall is a list of KongPluginInstallation resources that + will be installed and available in the Gateways (DataPlanes) that + use this GatewayConfig. + items: + description: NamespacedName is a resource identified by name + and optional namespace. + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array type: object type: object status: diff --git a/config/crd/dataplane/gateway-operator.konghq.com_dataplanes.yaml b/config/crd/dataplane/gateway-operator.konghq.com_dataplanes.yaml index 4ee8b3762..247dcc14e 100644 --- a/config/crd/dataplane/gateway-operator.konghq.com_dataplanes.yaml +++ b/config/crd/dataplane/gateway-operator.konghq.com_dataplanes.yaml @@ -8943,6 +8943,22 @@ spec: type: object type: object type: object + pluginsToInstall: + description: |- + PluginsToInstall is a list of KongPluginInstallation resources that + will be installed and available in the DataPlane. + items: + description: NamespacedName is a resource identified by name and + optional namespace. + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + type: array resources: description: |- DataPlaneResources defines the resources that will be created and managed diff --git a/config/samples/gateway-kongplugininstallation-httproute.yaml b/config/samples/gateway-kongplugininstallation-httproute.yaml new file mode 100644 index 000000000..6afc2fa7b --- /dev/null +++ b/config/samples/gateway-kongplugininstallation-httproute.yaml @@ -0,0 +1,170 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: additional +--- +apiVersion: v1 +kind: Service +metadata: + name: echo +spec: + ports: + - protocol: TCP + name: http + port: 80 + targetPort: http + selector: + app: echo +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: echo + name: echo +spec: + replicas: 1 + selector: + matchLabels: + app: echo + template: + metadata: + labels: + app: echo + spec: + containers: + - name: echo + image: registry.k8s.io/e2e-test-images/agnhost:2.40 + command: + - /agnhost + - netexec + - --http-port=8080 + ports: + - containerPort: 8080 + name: http + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + resources: + requests: + cpu: 10m +--- +kind: GatewayConfiguration +apiVersion: gateway-operator.konghq.com/v1beta1 +metadata: + name: kong + namespace: default +spec: + dataPlaneOptions: + deployment: + replicas: 2 + podTemplateSpec: + spec: + containers: + - name: proxy + # renovate: datasource=docker versioning=docker + image: kong/kong-gateway:3.7 + readinessProbe: + initialDelaySeconds: 1 + periodSeconds: 1 + pluginsToInstall: + - name: additional-custom-plugin + namespace: additional + - name: additional-custom-plugin-2 + controlPlaneOptions: + deployment: + podTemplateSpec: + spec: + containers: + - name: controller + # renovate: datasource=docker versioning=docker + image: kong/kubernetes-ingress-controller:3.2.3 + readinessProbe: + initialDelaySeconds: 1 + periodSeconds: 1 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: kong +spec: + controllerName: konghq.com/gateway-operator + parametersRef: + group: gateway-operator.konghq.com + kind: GatewayConfiguration + name: kong + namespace: default +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: kong + namespace: default +spec: + gatewayClassName: kong + listeners: + - name: http + protocol: HTTP + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute-echo + namespace: default + annotations: + konghq.com/strip-path: "true" + konghq.com/plugins: kong-custom-plugin,kong-custom-plugin-2 +spec: + parentRefs: + - name: kong + rules: + - matches: + - path: + type: PathPrefix + value: /echo + backendRefs: + - name: echo + kind: Service + port: 80 +--- +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: kong-custom-plugin +plugin: additional-custom-plugin +--- +kind: KongPluginInstallation +apiVersion: gateway-operator.konghq.com/v1alpha1 +metadata: + name: additional-custom-plugin + namespace: additional +spec: + image: northamerica-northeast1-docker.pkg.dev/k8s-team-playground/plugin-example/myheader +--- +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: kong-custom-plugin-2 +plugin: additional-custom-plugin-2 +--- +kind: KongPluginInstallation +apiVersion: gateway-operator.konghq.com/v1alpha1 +metadata: + name: additional-custom-plugin-2 +spec: + image: northamerica-northeast1-docker.pkg.dev/k8s-team-playground/plugin-example/myheader-2 diff --git a/controller/dataplane/controller.go b/controller/dataplane/controller.go index 21e81b17b..547f816de 100644 --- a/controller/dataplane/controller.go +++ b/controller/dataplane/controller.go @@ -86,7 +86,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if err != nil { log.Info(logger, "failed to validate dataplane: "+err.Error(), dataplane) r.eventRecorder.Event(dataplane, "Warning", "ValidationFailed", err.Error()) - markErr := r.ensureDataPlaneIsMarkedNotReady(ctx, logger, dataplane, DataPlaneConditionValidationFailed, err.Error()) + markErr := ensureDataPlaneIsMarkedNotReady(ctx, logger, r.Client, dataplane, DataPlaneConditionValidationFailed, err.Error()) return ctrl.Result{}, markErr } @@ -162,7 +162,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, nil // no need to requeue, the update will trigger. } - log.Trace(logger, "ensuring DataPlane has service addesses in status", dataplaneIngressService) + log.Trace(logger, "ensuring DataPlane has service addresses in status", dataplaneIngressService) if updated, err := r.ensureDataPlaneAddressesStatus(ctx, logger, dataplane, dataplaneIngressService); err != nil { return ctrl.Result{}, err } else if updated { @@ -176,6 +176,17 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu deploymentOpts := []k8sresources.DeploymentOpt{ labelSelectorFromDataPlaneStatusSelectorDeploymentOpt(dataplane), } + + log.Trace(logger, "ensuring generation of deployment configuration for KongPluginInstallations configured for DataPlane", dataplane) + kpisForDeployment, reconcileResult, err := ensureMappedConfigMapToKongPluginInstallationForDataPlane(ctx, logger, r.Client, dataplane) + if err != nil { + return ctrl.Result{}, fmt.Errorf("cannot ensure KongPluginInstallation for DataPlane: %w", err) + } + if reconcileResult.Requeue { + return reconcileResult, nil + } + deploymentOpts = append(deploymentOpts, withCustomPlugins(kpisForDeployment...)) + deploymentBuilder := NewDeploymentBuilder(logger.WithName("deployment_builder"), r.Client). WithBeforeCallbacks(r.Callbacks.BeforeDeployment). WithAfterCallbacks(r.Callbacks.AfterDeployment). @@ -186,8 +197,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu deployment, res, err := deploymentBuilder.BuildAndDeploy(ctx, dataplane, r.DevelopmentMode) if err != nil { - return ctrl.Result{}, fmt.Errorf("could not build Deployment for DataPlane %s: %w", - dpNn, err) + return ctrl.Result{}, fmt.Errorf("could not build Deployment for DataPlane %s: %w", dpNn, err) } if res != op.Noop { return ctrl.Result{}, nil @@ -210,10 +220,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, nil } - if res, err := ensureDataPlaneReadyStatus(ctx, r.Client, logger, dataplane, dataplane.Generation); err != nil { + if reconcileResult, err := ensureDataPlaneReadyStatus(ctx, r.Client, logger, dataplane, dataplane.Generation); err != nil { return ctrl.Result{}, err - } else if res.Requeue { - return res, nil + } else if reconcileResult.Requeue { + return reconcileResult, nil } log.Debug(logger, "reconciliation complete for DataPlane resource", dataplane) diff --git a/controller/dataplane/controller_conditions.go b/controller/dataplane/controller_conditions.go index 19840903b..3ff3af51e 100644 --- a/controller/dataplane/controller_conditions.go +++ b/controller/dataplane/controller_conditions.go @@ -10,4 +10,9 @@ const ( // DataPlaneConditionValidationFailed is a reason which indicates validation of // a dataplane is failed. DataPlaneConditionValidationFailed consts.ConditionReason = "ValidationFailed" + + // DataPlaneConditionReferencedResourcesNotAvailable is a reason which indicates + // that the referenced resources in DataPlane configuration (e.g. KongPluginInstallation) + // are not available. + DataPlaneConditionReferencedResourcesNotAvailable consts.ConditionReason = "ReferencedResourcesNotAvailable" ) diff --git a/controller/dataplane/controller_reconciler_utils.go b/controller/dataplane/controller_reconciler_utils.go index fc7fbffba..bab2f8762 100644 --- a/controller/dataplane/controller_reconciler_utils.go +++ b/controller/dataplane/controller_reconciler_utils.go @@ -3,19 +3,27 @@ package dataplane import ( "context" "fmt" + "maps" + "strings" "github.com/go-logr/logr" "github.com/google/go-cmp/cmp" + "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1" operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" "github.com/kong/gateway-operator/controller/pkg/address" "github.com/kong/gateway-operator/controller/pkg/log" "github.com/kong/gateway-operator/pkg/consts" k8sutils "github.com/kong/gateway-operator/pkg/utils/kubernetes" + k8sresources "github.com/kong/gateway-operator/pkg/utils/kubernetes/resources" ) // ----------------------------------------------------------------------------- @@ -51,9 +59,9 @@ func (r *Reconciler) ensureDataPlaneServiceStatus( } // ensureDataPlaneAddressesStatus ensures that provided DataPlane's status addresses -// are as expected and pathes its status if there's a difference between the +// are as expected and patches its status if there's a difference between the // current state and what's expected. -// It returns a boolean indicating if the patch has been trigerred and an error. +// It returns a boolean indicating if the patch has been triggered and an error. func (r *Reconciler) ensureDataPlaneAddressesStatus( ctx context.Context, log logr.Logger, @@ -77,6 +85,175 @@ func (r *Reconciler) ensureDataPlaneAddressesStatus( return false, nil } +// ensureMappedConfigMapToKongPluginInstallationForDataPlane ensures that the KongPluginInstallation +// resources referenced by the DataPlane are resolved and DataPlane is configured to use them. +// During resolving for each DataPlane based on each instance of KongPluginInstallation +// ConfigMap is created and mounted. The DataPlane manages its lifecycle. It returns a slice +// of custom plugins that are intended to be used to generate a Deployment. +func ensureMappedConfigMapToKongPluginInstallationForDataPlane( + ctx context.Context, logger logr.Logger, c client.Client, dataplane *operatorv1beta1.DataPlane, +) ([]customPlugin, ctrl.Result, error) { + configMapsOwned, err := findCustomPluginConfigMapsOwnedByDataPlane(ctx, c, dataplane) + if err != nil { + return nil, ctrl.Result{}, err + } + configMapsToRetain := make(map[types.NamespacedName]struct{}, len(configMapsOwned)) + + var cps []customPlugin + for _, kpiNN := range dataplane.Spec.PluginsToInstall { + kpiNN := types.NamespacedName(kpiNN) + if kpiNN.Namespace == "" { + kpiNN.Namespace = dataplane.Namespace + } + + cp, reconciliationResult, err := populateDedicatedConfigMapForKongPluginInstallation( + ctx, logger, c, configMapsOwned, kpiNN, dataplane, + ) + if err != nil || reconciliationResult.Requeue { + return nil, reconciliationResult, err + } + configMapsToRetain[cp.ConfigMapNN] = struct{}{} + cps = append(cps, cp) + } + for _, cm := range configMapsOwned { + if _, retain := configMapsToRetain[client.ObjectKeyFromObject(&cm)]; !retain { + if err := c.Delete(ctx, &cm); client.IgnoreNotFound(err) != nil { + return nil, ctrl.Result{}, err + } + } + } + + return cps, ctrl.Result{}, nil +} + +func findCustomPluginConfigMapsOwnedByDataPlane( + ctx context.Context, c client.Client, dataplane *operatorv1beta1.DataPlane, +) ([]corev1.ConfigMap, error) { + cms, err := k8sutils.ListConfigMapsForOwner(ctx, c, dataplane.GetUID()) + if err != nil { + return nil, err + } + return lo.Filter(cms, func(cm corev1.ConfigMap, _ int) bool { + _, isForKPI := cm.Annotations[consts.AnnotationMappedToKongPluginInstallation] + return isForKPI + }), nil +} + +func populateDedicatedConfigMapForKongPluginInstallation( + ctx context.Context, + logger logr.Logger, + c client.Client, + cms []corev1.ConfigMap, + kpiNN types.NamespacedName, + dataplane *operatorv1beta1.DataPlane, +) (customPlugin, ctrl.Result, error) { + kpi, ready, err := verifyKPIReadinessForDataPlane(ctx, logger, c, dataplane, kpiNN) + if err != nil { + return customPlugin{}, ctrl.Result{}, err + } + if !ready { + return customPlugin{}, ctrl.Result{Requeue: true}, nil + } + + var underlyingCM corev1.ConfigMap + backingCMNN := types.NamespacedName{ + Namespace: kpi.Namespace, + Name: kpi.Status.UnderlyingConfigMapName, + } + log.Trace(logger, fmt.Sprintf("Fetch underlying ConfigMap %s for KongPluginInstallation", backingCMNN), kpi) + if err := c.Get(ctx, backingCMNN, &underlyingCM); err != nil { + return customPlugin{}, ctrl.Result{}, fmt.Errorf("could not fetch underlying ConfigMap to clone %s: %w", backingCMNN, err) + } + + log.Trace(logger, "Find ConfigMap mapped to KongPluginInstallation", kpi) + mappedConfigMapForKPI := lo.Filter(cms, func(cm corev1.ConfigMap, _ int) bool { + kpiNN := cm.Annotations[consts.AnnotationMappedToKongPluginInstallation] + return kpiNN == client.ObjectKeyFromObject(&kpi).String() + }) + var cm corev1.ConfigMap + switch len(mappedConfigMapForKPI) { + case 0: + log.Trace(logger, "Create new ConfigMap for KongPluginInstallation", kpi) + cm.GenerateName = dataplane.Name + "-" + cm.Namespace = dataplane.Namespace + k8sutils.SetOwnerForObject(&cm, dataplane) + k8sresources.LabelObjectAsDataPlaneManaged(&cm) + k8sresources.AnnotateConfigMapWithKongPluginInstallation(&cm, kpi) + cm.Data = underlyingCM.Data + if err := c.Create(ctx, &cm); err != nil { + return customPlugin{}, ctrl.Result{}, fmt.Errorf("could not create new ConfigMap for KongPluginInstallation: %w", err) + } + case 1: + log.Trace(logger, fmt.Sprintf("Check if update existing ConfigMap %s for KongPluginInstallation", client.ObjectKeyFromObject(&cm)), kpi) + cm = mappedConfigMapForKPI[0] + if maps.Equal(cm.Data, underlyingCM.Data) { + log.Trace(logger, fmt.Sprintf("Nothing to update in existing ConfigMap %s for KongPluginInstallation", client.ObjectKeyFromObject(&cm)), kpi) + return customPlugin{}, ctrl.Result{}, nil + } + log.Trace(logger, fmt.Sprintf("Update existing ConfigMap %s for KongPluginInstallation", client.ObjectKeyFromObject(&cm)), kpi) + cm.Data = underlyingCM.Data + if err := c.Update(ctx, &cm); err != nil { + if k8serrors.IsConflict(err) { + return customPlugin{}, ctrl.Result{Requeue: true}, nil + } + return customPlugin{}, ctrl.Result{}, fmt.Errorf("could not update mapped: %w", err) + } + default: + // It should never happen. + names := strings.Join(lo.Map(mappedConfigMapForKPI, func(cm corev1.ConfigMap, _ int) string { + return client.ObjectKeyFromObject(&cm).String() + }), ", ") + return customPlugin{}, ctrl.Result{}, fmt.Errorf("unexpected error happened - more than one ConfigMap found: %s", names) + } + return customPlugin{ + Name: kpi.Name, + ConfigMapNN: client.ObjectKeyFromObject(&cm), + Generation: kpi.Generation, + }, ctrl.Result{}, nil +} + +// verifyKPIReadinessForDataPlane updates DataPlane status conditions based on status of KPI object. +// Possible states: it does not exist or it hasn't been fully reconciled yet, or it's failing. Those +// problems can be fixed by the user or they're transient. Use returned kpi only when ready is true. +func verifyKPIReadinessForDataPlane( + ctx context.Context, logger logr.Logger, c client.Client, dataplane *operatorv1beta1.DataPlane, kpiNN types.NamespacedName, +) (kpi operatorv1alpha1.KongPluginInstallation, ready bool, err error) { + // Report to user when KPI does not exist or it hasn't been fully reconciled yet. + // It can be fixed by the user or it's transient. + if err := c.Get(ctx, kpiNN, &kpi); err != nil { + if k8serrors.IsNotFound(err) { + msg := fmt.Sprintf("referenced KongPluginInstallation %s not found", kpiNN) + markErr := ensureDataPlaneIsMarkedNotReady(ctx, logger, c, dataplane, DataPlaneConditionReferencedResourcesNotAvailable, msg) + return kpi, false, markErr + } else { + return kpi, true, err + } + } + if len(kpi.Status.Conditions) == 0 || lo.ContainsBy(kpi.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == string(operatorv1alpha1.KongPluginInstallationConditionStatusAccepted) && + c.Status == metav1.ConditionFalse && + c.Reason == string(operatorv1alpha1.KongPluginInstallationReasonPending) + }) { + msgPending := fmt.Sprintf("please wait, referenced KongPluginInstallation %s has not been fully reconciled yet", kpiNN) + markErr := ensureDataPlaneIsMarkedNotReady( + ctx, logger, c, dataplane, DataPlaneConditionReferencedResourcesNotAvailable, msgPending, + ) + return kpi, false, markErr + } + if lo.ContainsBy(kpi.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == string(operatorv1alpha1.KongPluginInstallationConditionStatusAccepted) && + c.Status == metav1.ConditionFalse && + c.Reason == string(operatorv1alpha1.KongPluginInstallationReasonFailed) + }) { + msgFailed := fmt.Sprintf("something wrong with referenced KongPluginInstallation %s, please check it", kpiNN) + markErr := ensureDataPlaneIsMarkedNotReady( + ctx, logger, c, dataplane, DataPlaneConditionReferencedResourcesNotAvailable, msgFailed, + ) + return kpi, false, markErr + } + return kpi, true, nil +} + // isSameDataPlaneCondition returns true if two `metav1.Condition`s // indicates the same condition of a `DataPlane` resource. func isSameDataPlaneCondition(condition1, condition2 metav1.Condition) bool { @@ -86,9 +263,10 @@ func isSameDataPlaneCondition(condition1, condition2 metav1.Condition) bool { condition1.Message == condition2.Message } -func (r *Reconciler) ensureDataPlaneIsMarkedNotReady( +func ensureDataPlaneIsMarkedNotReady( ctx context.Context, log logr.Logger, + c client.Client, dataplane *operatorv1beta1.DataPlane, reason consts.ConditionReason, message string, ) error { @@ -122,7 +300,7 @@ func (r *Reconciler) ensureDataPlaneIsMarkedNotReady( } if shouldUpdate { - _, err := patchDataPlaneStatus(ctx, r.Client, log, dataplane) + _, err := patchDataPlaneStatus(ctx, c, log, dataplane) return err } return nil diff --git a/controller/dataplane/owned_custom_plugins.go b/controller/dataplane/owned_custom_plugins.go new file mode 100644 index 000000000..24fddcda1 --- /dev/null +++ b/controller/dataplane/owned_custom_plugins.go @@ -0,0 +1,81 @@ +package dataplane + +import ( + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/kong/gateway-operator/internal/utils/dataplane" + "github.com/kong/gateway-operator/pkg/consts" + k8sresources "github.com/kong/gateway-operator/pkg/utils/kubernetes/resources" +) + +type customPlugin struct { + // Name of the KongPluginInstallation resource. + Name string + // ConfigMapNN is the namespace/name of the ConfigMap that contains the plugin. + ConfigMapNN types.NamespacedName + // Generation is the generation of the KongPluginInstallation that contains the plugin. + Generation int64 +} + +func withCustomPlugins(customPlugins ...customPlugin) k8sresources.DeploymentOpt { + // Noop/cleanup operation that is safe to execute if no plugins are provided. + if len(customPlugins) == 0 { + return func(d *appsv1.Deployment) { + // It's safe to perform a delete operation on a nil map. + delete( + d.Spec.Template.Annotations, + consts.AnnotationKongPluginInstallationGenerationInternal, + ) + } + } + + var ( + kpisNames = make([]string, 0, len(customPlugins)) + kpisGenerations = make([]string, 0, len(customPlugins)) + kpisVolumeMounts = make([]corev1.VolumeMount, 0, len(customPlugins)) + kpisVolumes = make([]corev1.Volume, 0, len(customPlugins)) + ) + + for _, cp := range customPlugins { + kpisNames = append(kpisNames, cp.Name) + kpisGenerations = append(kpisGenerations, fmt.Sprintf("%s:%d", cp.Name, cp.Generation)) + kpisVolumeMounts = append(kpisVolumeMounts, corev1.VolumeMount{ + Name: cp.Name, + MountPath: "/opt/kong/plugins/" + cp.Name, + }) + kpisVolumes = append(kpisVolumes, corev1.Volume{ + Name: cp.Name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cp.ConfigMapNN.Name, + }, + }, + }, + }) + } + + return func(deployment *appsv1.Deployment) { + if deployment.Spec.Template.Annotations == nil { + deployment.Spec.Template.Annotations = make(map[string]string) + } + deployment.Spec.Template.Annotations[consts.AnnotationKongPluginInstallationGenerationInternal] = strings.Join(kpisGenerations, ",") + deployment.Spec.Template.Spec.Containers[0].Env = append( + deployment.Spec.Template.Spec.Containers[0].Env, + dataplane.ConfigureKongPluginRelatedEnvVars(kpisNames)..., + ) + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append( + deployment.Spec.Template.Spec.Containers[0].VolumeMounts, + kpisVolumeMounts..., + ) + deployment.Spec.Template.Spec.Volumes = append( + deployment.Spec.Template.Spec.Volumes, + kpisVolumes..., + ) + } +} diff --git a/controller/dataplane/owned_custom_plugins_test.go b/controller/dataplane/owned_custom_plugins_test.go new file mode 100644 index 000000000..bec48613e --- /dev/null +++ b/controller/dataplane/owned_custom_plugins_test.go @@ -0,0 +1,185 @@ +package dataplane + +import ( + "testing" + + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/kong/gateway-operator/pkg/consts" +) + +func TestOptWithCustomPlugin(t *testing.T) { + testCases := []struct { + name string + customPlugins []customPlugin + expectedEnv []corev1.EnvVar + expectedVolumes []corev1.Volume + expectedVolumeMounts []corev1.VolumeMount + expectedAnnotations map[string]string + }{ + { + name: "no custom plugins", + customPlugins: []customPlugin{}, + }, + { + name: "one custom plugin", + customPlugins: []customPlugin{ + { + Name: "plugin1", + ConfigMapNN: types.NamespacedName{ + Name: "configmap1", + }, + Generation: 1, + }, + }, + expectedEnv: []corev1.EnvVar{ + { + Name: "KONG_PLUGINS", + Value: "bundled,plugin1", + }, + { + Name: "KONG_LUA_PACKAGE_PATH", + Value: "/opt/?.lua;;", + }, + }, + expectedVolumes: []corev1.Volume{ + { + Name: "plugin1", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "configmap1", + }, + }, + }, + }, + }, + expectedVolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin1", + MountPath: "/opt/kong/plugins/plugin1", + }, + }, + expectedAnnotations: map[string]string{ + consts.AnnotationKongPluginInstallationGenerationInternal: "plugin1:1", + }, + }, + { + name: "multiple custom plugins", + customPlugins: []customPlugin{ + { + Name: "plugin1", + ConfigMapNN: types.NamespacedName{ + Name: "configmap1", + }, + Generation: 1, + }, + { + Name: "plugin2", + ConfigMapNN: types.NamespacedName{ + Name: "configmap2", + }, + Generation: 2, + }, + }, + expectedEnv: []corev1.EnvVar{ + { + Name: "KONG_PLUGINS", + Value: "bundled,plugin1,plugin2", + }, + { + Name: "KONG_LUA_PACKAGE_PATH", + Value: "/opt/?.lua;;", + }, + }, + expectedVolumes: []corev1.Volume{ + { + Name: "plugin1", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "configmap1", + }, + }, + }, + }, + { + Name: "plugin2", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "configmap2", + }, + }, + }, + }, + }, + expectedVolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin1", + MountPath: "/opt/kong/plugins/plugin1", + }, + { + Name: "plugin2", + MountPath: "/opt/kong/plugins/plugin2", + }, + }, + expectedAnnotations: map[string]string{ + consts.AnnotationKongPluginInstallationGenerationInternal: "plugin1:1,plugin2:2", + }, + }, + } + + for _, tt := range testCases { + t.Run( + tt.name, + func(t *testing.T) { + deployment := &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {}, + }, + }, + }, + }, + } + withCustomPlugins(tt.customPlugins...)(deployment) + + require.Equal(t, tt.expectedAnnotations, deployment.Spec.Template.Annotations) + require.Equal(t, tt.expectedEnv, deployment.Spec.Template.Spec.Containers[0].Env) + require.Equal(t, tt.expectedVolumeMounts, deployment.Spec.Template.Spec.Containers[0].VolumeMounts) + require.Equal(t, tt.expectedVolumes, deployment.Spec.Template.Spec.Volumes) + }, + ) + } +} + +func TestOptNoopWithCustomPlugin(t *testing.T) { + const ( + annotationThatShouldBePreservedKey = "annotation-to-preserve" + annotationThatShouldBePreservedValue = "this-is-it" + ) + deployment := &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + consts.AnnotationKongPluginInstallationGenerationInternal: "plugin1:1", + annotationThatShouldBePreservedKey: annotationThatShouldBePreservedValue, + }, + }, + }, + }, + } + withCustomPlugins()(deployment) + + require.Equal( + t, map[string]string{annotationThatShouldBePreservedKey: annotationThatShouldBePreservedValue}, deployment.Spec.Template.Annotations, + ) +} diff --git a/controller/dataplane/watch.go b/controller/dataplane/watch.go index 475c1f414..55af38902 100644 --- a/controller/dataplane/watch.go +++ b/controller/dataplane/watch.go @@ -1,14 +1,23 @@ package dataplane import ( + "context" + appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" + k8stypes "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" + "github.com/kong/gateway-operator/pkg/consts" ) // DataPlaneWatchBuilder creates a controller builder pre-configured with @@ -27,5 +36,58 @@ func DataPlaneWatchBuilder(mgr ctrl.Manager) *builder.Builder { // Watch for changes in HPA created by the dataplane controller. Owns(&autoscalingv2.HorizontalPodAutoscaler{}). // Watch for changes in PodDisruptionBudgets created by the dataplane controller. - Owns(&policyv1.PodDisruptionBudget{}) + Owns(&policyv1.PodDisruptionBudget{}). + // Watch for changes in ConfigMaps created by the dataplane controller. + Owns(&corev1.ConfigMap{}). + // Watch for changes in ConfigMaps that are mapped to KongPluginInstallation objects. + // They may trigger reconciliation of DataPlane resources. + WatchesRawSource( + source.Kind( + mgr.GetCache(), + &corev1.ConfigMap{}, + handler.TypedEnqueueRequestsFromMapFunc(listDataPlanesReferencingKongPluginInstallation(mgr.GetClient())), + ), + ) +} + +func listDataPlanesReferencingKongPluginInstallation( + c client.Client, +) handler.TypedMapFunc[*corev1.ConfigMap, reconcile.Request] { + return func( + ctx context.Context, kpiCM *corev1.ConfigMap, + ) []reconcile.Request { + logger := ctrllog.FromContext(ctx) + + // Find all DataPlane resources referencing KongPluginInstallation + // that maps to the ConfigMap enqueued for reconciliation. + kpiToFind := kpiCM.Annotations[consts.AnnotationMappedToKongPluginInstallation] + if kpiToFind == "" { + return nil + } + + var dataPlaneList operatorv1beta1.DataPlaneList + if err := c.List(ctx, &dataPlaneList); client.IgnoreNotFound(err) != nil { + logger.Error(err, "Failed to list DataPlanes in watch", "KongPluginInstallation", kpiToFind) + return nil + } + var dataPlanesToReconcile []reconcile.Request + for _, dp := range dataPlaneList.Items { + for _, ptiNN := range dp.Spec.PluginsToInstall { + kpiNN := k8stypes.NamespacedName(ptiNN) + if kpiNN.Namespace == "" { + kpiNN.Namespace = dp.Namespace + } + + if kpiNN.String() == kpiToFind { + dataPlanesToReconcile = append( + dataPlanesToReconcile, + reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&dp), + }, + ) + } + } + } + return dataPlanesToReconcile + } } diff --git a/controller/gateway/controller.go b/controller/gateway/controller.go index 8bda0c09f..444695a6f 100644 --- a/controller/gateway/controller.go +++ b/controller/gateway/controller.go @@ -441,7 +441,7 @@ func (r *Reconciler) provisionDataPlane( // if not configured in gatewayconfiguration, compare deployment option of dataplane with an empty one. expectedDataPlaneOptions := &operatorv1beta1.DataPlaneOptions{} if gatewayConfig.Spec.DataPlaneOptions != nil { - expectedDataPlaneOptions = gatewayConfigDataPlaneOptionsToDataPlaneOptions(*gatewayConfig.Spec.DataPlaneOptions) + expectedDataPlaneOptions = gatewayConfigDataPlaneOptionsToDataPlaneOptions(gatewayConfig.Namespace, *gatewayConfig.Spec.DataPlaneOptions) } // Don't require setting defaults for DataPlane when using Gateway CRD. setDataPlaneOptionsDefaults(expectedDataPlaneOptions, r.DefaultDataPlaneImage) diff --git a/controller/gateway/controller_reconciler_utils.go b/controller/gateway/controller_reconciler_utils.go index 5cd7798cd..8978d9edb 100644 --- a/controller/gateway/controller_reconciler_utils.go +++ b/controller/gateway/controller_reconciler_utils.go @@ -51,7 +51,7 @@ func (r *Reconciler) createDataPlane(ctx context.Context, }, } if gatewayConfig.Spec.DataPlaneOptions != nil { - dataplane.Spec.DataPlaneOptions = *gatewayConfigDataPlaneOptionsToDataPlaneOptions(*gatewayConfig.Spec.DataPlaneOptions) + dataplane.Spec.DataPlaneOptions = *gatewayConfigDataPlaneOptionsToDataPlaneOptions(gatewayConfig.Namespace, *gatewayConfig.Spec.DataPlaneOptions) } setDataPlaneOptionsDefaults(&dataplane.Spec.DataPlaneOptions, r.DefaultDataPlaneImage) if err := setDataPlaneIngressServicePorts(&dataplane.Spec.DataPlaneOptions, gateway.Spec.Listeners); err != nil { @@ -126,9 +126,20 @@ func (r *Reconciler) getGatewayAddresses( return gatewayAddressesFromService(services[0]) } -func gatewayConfigDataPlaneOptionsToDataPlaneOptions(opts operatorv1beta1.GatewayConfigDataPlaneOptions) *operatorv1beta1.DataPlaneOptions { +func gatewayConfigDataPlaneOptionsToDataPlaneOptions( + gatewayConfigNamespace string, opts operatorv1beta1.GatewayConfigDataPlaneOptions, +) *operatorv1beta1.DataPlaneOptions { + // When Namespace is not provided, the GatewayConfiguration's namespace is assumed. + pluginsToInstall := lo.Map(opts.PluginsToInstall, func(pluginReference operatorv1beta1.NamespacedName, _ int) operatorv1beta1.NamespacedName { + if pluginReference.Namespace == "" { + pluginReference.Namespace = gatewayConfigNamespace + } + return pluginReference + }) + dataPlaneOptions := &operatorv1beta1.DataPlaneOptions{ - Deployment: opts.Deployment, + Deployment: opts.Deployment, + PluginsToInstall: pluginsToInstall, } if opts.Network.Services != nil && opts.Network.Services.Ingress != nil { diff --git a/controller/kongplugininstallation/controller.go b/controller/kongplugininstallation/controller.go index dc66d5700..579479f0f 100644 --- a/controller/kongplugininstallation/controller.go +++ b/controller/kongplugininstallation/controller.go @@ -129,7 +129,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu &secret, ); err != nil { if k8serrors.IsNotFound(err) { - return ctrl.Result{}, setStatusConditionFailedForKongPluginInstallation(ctx, r.Client, &kpi, fmt.Sprintf("cannot retrieve secret %q, because: %s", secretNN, err)) + return ctrl.Result{}, setStatusConditionFailedForKongPluginInstallation(ctx, r.Client, &kpi, fmt.Sprintf("referenced Secret %q not found", secretNN)) } return ctrl.Result{}, fmt.Errorf("something unexpected during fetching secret %s: %w", secretNN, err) } diff --git a/docs/api-reference.md b/docs/api-reference.md index 721f779a6..86cd427e4 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -834,6 +834,7 @@ deploy the DataPlane. | `network` _[DataPlaneNetworkOptions](#dataplanenetworkoptions)_ | | | `resources` _[DataPlaneResources](#dataplaneresources)_ | | | `extensions` _[ExtensionRef](#extensionref) array_ | Extensions provide additional or replacement features for the DataPlane resources to influence or enhance functionality. NOTE: since we have one extension only (DataPlaneKonnectExtension), we limit the amount of extensions to 1. | +| `pluginsToInstall` _[NamespacedName](#namespacedname) array_ | PluginsToInstall is a list of KongPluginInstallation resources that will be installed and available in the DataPlane. | _Appears in:_ @@ -969,6 +970,7 @@ DataPlaneSpec defines the desired state of DataPlane | `network` _[DataPlaneNetworkOptions](#dataplanenetworkoptions)_ | | | `resources` _[DataPlaneResources](#dataplaneresources)_ | | | `extensions` _[ExtensionRef](#extensionref) array_ | Extensions provide additional or replacement features for the DataPlane resources to influence or enhance functionality. NOTE: since we have one extension only (DataPlaneKonnectExtension), we limit the amount of extensions to 1. | +| `pluginsToInstall` _[NamespacedName](#namespacedname) array_ | PluginsToInstall is a list of KongPluginInstallation resources that will be installed and available in the DataPlane. | _Appears in:_ @@ -1044,6 +1046,7 @@ configure and deploy a DataPlane object. | `deployment` _[DataPlaneDeploymentOptions](#dataplanedeploymentoptions)_ | | | `network` _[GatewayConfigDataPlaneNetworkOptions](#gatewayconfigdataplanenetworkoptions)_ | | | `extensions` _[ExtensionRef](#extensionref) array_ | Extensions provide additional or replacement features for the DataPlane resources to influence or enhance functionality. NOTE: since we have one extension only (DataPlaneKonnectExtension), we limit the amount of extensions to 1. | +| `pluginsToInstall` _[NamespacedName](#namespacedname) array_ | PluginsToInstall is a list of KongPluginInstallation resources that will be installed and available in the Gateways (DataPlanes) that use this GatewayConfig. | _Appears in:_ @@ -1165,6 +1168,9 @@ NamespacedName is a resource identified by name and optional namespace. _Appears in:_ +- [DataPlaneOptions](#dataplaneoptions) +- [DataPlaneSpec](#dataplanespec) +- [GatewayConfigDataPlaneOptions](#gatewayconfigdataplaneoptions) - [KonnectCertificateOptions](#konnectcertificateoptions) #### PodDisruptionBudget diff --git a/internal/utils/dataplane/config.go b/internal/utils/dataplane/config.go index 1c1ce5fac..b3bf8151b 100644 --- a/internal/utils/dataplane/config.go +++ b/internal/utils/dataplane/config.go @@ -3,6 +3,7 @@ package dataplane import ( "fmt" "sort" + "strings" corev1 "k8s.io/api/core/v1" @@ -10,6 +11,14 @@ import ( k8sutils "github.com/kong/gateway-operator/pkg/utils/kubernetes" ) +const ( + kongPluginsEnvVarName = "KONG_PLUGINS" + kongPluginsDefaultValue = "bundled" + + kongLuaPackagePathVarName = "KONG_LUA_PACKAGE_PATH" + kongLuaPackagePathDefaultValue = "/opt/?.lua;;" +) + // KongDefaults are the baseline Kong proxy configuration options needed for // the proxy to function. var KongDefaults = map[string]string{ @@ -20,7 +29,7 @@ var KongDefaults = map[string]string{ "KONG_CLUSTER_LISTEN": "off", "KONG_DATABASE": "off", "KONG_NGINX_WORKER_PROCESSES": "2", - "KONG_PLUGINS": "bundled", + kongPluginsEnvVarName: kongPluginsDefaultValue, "KONG_PORTAL_API_ACCESS_LOG": "/dev/stdout", "KONG_PORTAL_API_ERROR_LOG": "/dev/stderr", "KONG_PORT_MAPS": "80:8000, 443:8443", @@ -75,3 +84,27 @@ func FillDataPlaneProxyContainerEnvs(existing []corev1.EnvVar, podTemplateSpec * } sort.Sort(k8sutils.SortableEnvVars(container.Env)) } + +// ConfigureKongPluginRelatedEnvVars returns the environment variables +// needed for configuring the Kong Gateway with the provided Kong Plugin +// names. If kongPluginNames is nil or empty, nil is returned. Kong will use bundled +// plugins by default if we do not override `KONG_PLUGINS`. +func ConfigureKongPluginRelatedEnvVars(kongPluginNames []string) []corev1.EnvVar { + if len(kongPluginNames) == 0 { + return nil + } + kpiNames := make([]string, 0, len(kongPluginNames)+1) // +1 for the default value + // Const "bundled" is required to have the default plugins enabled. + kpiNames = append(kpiNames, kongPluginsDefaultValue) + kpiNames = append(kpiNames, kongPluginNames...) + return []corev1.EnvVar{ + { + Name: kongPluginsEnvVarName, + Value: strings.Join(kpiNames, ","), + }, + { + Name: kongLuaPackagePathVarName, + Value: kongLuaPackagePathDefaultValue, + }, + } +} diff --git a/pkg/consts/kongplugininstallation.go b/pkg/consts/kongplugininstallation.go index abf16cf93..6820fe087 100644 --- a/pkg/consts/kongplugininstallation.go +++ b/pkg/consts/kongplugininstallation.go @@ -5,7 +5,11 @@ const ( // KongPluginInstallation controller. KongPluginInstallationManagedLabelValue = "kong-plugin-installation" - // AnnotationKongPluginInstallationName is the annotation key used to store the name of the KongPluginInstallation + // AnnotationMappedToKongPluginInstallation is the annotation key used to store the name of the KongPluginInstallation // that maps to particular ConfigMap. - AnnotationKongPluginInstallationMappedKongPluginInstallation = OperatorLabelPrefix + "mapped-to-kong-plugin-installation" + AnnotationMappedToKongPluginInstallation = OperatorLabelPrefix + "mapped-to-kong-plugin-installation" + + // AnnotationKongPluginInstallationGenerationInternal is the annotation key used to store KongPluginInstallation + // and its generation, internal usage to re-trigger deployment when KongPluginInstallation changes. + AnnotationKongPluginInstallationGenerationInternal = OperatorLabelPrefix + "kong-plugin-installation-generation" ) diff --git a/pkg/utils/kubernetes/resources/annotations.go b/pkg/utils/kubernetes/resources/annotations.go index dca451050..2f74601aa 100644 --- a/pkg/utils/kubernetes/resources/annotations.go +++ b/pkg/utils/kubernetes/resources/annotations.go @@ -16,6 +16,6 @@ func AnnotateConfigMapWithKongPluginInstallation(cm *corev1.ConfigMap, kpi v1alp if annotations == nil { annotations = make(map[string]string) } - annotations[consts.AnnotationKongPluginInstallationMappedKongPluginInstallation] = client.ObjectKeyFromObject(&kpi).String() + annotations[consts.AnnotationMappedToKongPluginInstallation] = client.ObjectKeyFromObject(&kpi).String() cm.SetAnnotations(annotations) } diff --git a/pkg/utils/test/clients.go b/pkg/utils/test/clients.go index 0c2e87f4c..42aad607f 100644 --- a/pkg/utils/test/clients.go +++ b/pkg/utils/test/clients.go @@ -18,14 +18,16 @@ import ( configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" configurationv1beta1 "github.com/kong/kubernetes-configuration/api/configuration/v1beta1" konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" + configurationclient "github.com/kong/kubernetes-configuration/pkg/clientset" ) // K8sClients is a struct that contains all the Kubernetes clients needed by the tests. type K8sClients struct { - K8sClient *kubernetesclient.Clientset - OperatorClient *operatorclient.Clientset - GatewayClient *gatewayclient.Clientset - MgrClient ctrlruntimeclient.Client + K8sClient *kubernetesclient.Clientset + OperatorClient *operatorclient.Clientset + GatewayClient *gatewayclient.Clientset + ConfigurationClient *configurationclient.Clientset + MgrClient ctrlruntimeclient.Client } // NewK8sClients returns a new K8sClients struct with all the clients needed by the tests. @@ -42,6 +44,11 @@ func NewK8sClients(env environments.Environment) (K8sClients, error) { if err != nil { return clients, err } + clients.ConfigurationClient, err = configurationclient.NewForConfig(env.Cluster().Config()) + if err != nil { + return clients, err + } + clients.MgrClient, err = ctrlruntimeclient.New(env.Cluster().Config(), ctrlruntimeclient.Options{}) if err != nil { return clients, err diff --git a/test/integration/test_kongplugininstallation.go b/test/integration/test_kongplugininstallation.go index a03f462bf..bf72f6d8f 100644 --- a/test/integration/test_kongplugininstallation.go +++ b/test/integration/test_kongplugininstallation.go @@ -2,197 +2,273 @@ package integration import ( "fmt" + "net/http" "strings" "testing" "time" "github.com/google/uuid" + "github.com/kong/kubernetes-testing-framework/pkg/clusters" + "github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/generators" "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - "github.com/kong/gateway-operator/api/v1alpha1" + operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1" + operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" + "github.com/kong/gateway-operator/controller/dataplane" "github.com/kong/gateway-operator/pkg/consts" + testutils "github.com/kong/gateway-operator/pkg/utils/test" "github.com/kong/gateway-operator/test/helpers" + + configurationv1 "github.com/kong/kubernetes-configuration/api/configuration/v1" ) func TestKongPluginInstallationEssentials(t *testing.T) { - t.Parallel() - + if webhookEnabled { + // It can't be tested with webhook, because it rejects resources immediately, that would + // be accepted by the controller taking into account the eventual consistency nature of K8s. + t.Skip("webhook is enabled, skipping the test (due to webhook validation limitations)") + } namespace, cleaner := helpers.SetupTestEnv(t, GetCtx(), GetEnv()) const registryUrl = "northamerica-northeast1-docker.pkg.dev/k8s-team-playground/" + + const pluginInvalidLayersImage = registryUrl + "plugin-example/invalid-layers" + + const pluginMyHeaderImage = registryUrl + "plugin-example/myheader" + expectedHeadersForMyHeader := http.Header{"myheader": {"roar"}} + + const pluginMyHeader2Image = registryUrl + "plugin-example-private/myheader-2" + expectedHeadersForMyHeader2 := http.Header{"newheader": {"amazing"}} + t.Log("deploying an invalid KongPluginInstallation resource") - kpiNN := k8stypes.NamespacedName{ + kpiPublicNN := k8stypes.NamespacedName{ Name: "test-kpi", Namespace: namespace.Name, } - kpi := &v1alpha1.KongPluginInstallation{ + kpiPublic := &operatorv1alpha1.KongPluginInstallation{ ObjectMeta: metav1.ObjectMeta{ - Name: kpiNN.Name, + Name: kpiPublicNN.Name, + Namespace: kpiPublicNN.Namespace, }, - Spec: v1alpha1.KongPluginInstallationSpec{ - Image: registryUrl + "plugin-example/invalid-layers", + Spec: operatorv1alpha1.KongPluginInstallationSpec{ + Image: pluginInvalidLayersImage, }, } - kpi, err := GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiNN.Namespace).Create(GetCtx(), kpi, metav1.CreateOptions{}) + kpiPublic, err := GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(namespace.Name).Create(GetCtx(), kpiPublic, metav1.CreateOptions{}) require.NoError(t, err) - cleaner.Add(kpi) + cleaner.Add(kpiPublic) + t.Log("waiting for the KongPluginInstallation resource to be rejected, because of the invalid image") checkKongPluginInstallationConditions( - t, - kpiNN, - metav1.ConditionFalse, - `problem with the image: "northamerica-northeast1-docker.pkg.dev/k8s-team-playground/plugin-example/invalid-layers" error: expected exactly one layer with plugin, found 2 layers`) + t, kpiPublicNN, metav1.ConditionFalse, + fmt.Sprintf(`problem with the image: "%s" error: expected exactly one layer with plugin, found 2 layers`, pluginInvalidLayersImage), + ) + + t.Log("deploy Gateway with example service and HTTPRoute") + ip, gatewayConfigNN, httpRouteNN := deployGatewayWithKPI(t, cleaner, namespace.Name) + t.Log("attach broken KPI to the Gateway") + attachKPI(t, gatewayConfigNN, kpiPublicNN) + t.Log("ensure that status of the DataPlane is not ready with proper description of the issue") + checkDataPlaneStatus( + t, namespace.Name, metav1.ConditionFalse, dataplane.DataPlaneConditionReferencedResourcesNotAvailable, + fmt.Sprintf("something wrong with referenced KongPluginInstallation %s, please check it", client.ObjectKeyFromObject(kpiPublic)), + ) t.Log("updating KongPluginInstallation resource to a valid image") - kpi, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiNN.Namespace).Get(GetCtx(), kpiNN.Name, metav1.GetOptions{}) - kpi.Spec.Image = registryUrl + "plugin-example/valid:0.1.0" + kpiPublic, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiPublicNN.Namespace).Get(GetCtx(), kpiPublicNN.Name, metav1.GetOptions{}) + kpiPublic.Spec.Image = pluginMyHeaderImage require.NoError(t, err) - _, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiNN.Namespace).Update(GetCtx(), kpi, metav1.UpdateOptions{}) + _, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiPublicNN.Namespace).Update(GetCtx(), kpiPublic, metav1.UpdateOptions{}) require.NoError(t, err) t.Log("waiting for the KongPluginInstallation resource to be accepted") - checkKongPluginInstallationConditions(t, kpiNN, metav1.ConditionTrue, "plugin successfully saved in cluster as ConfigMap") + checkKongPluginInstallationConditions(t, kpiPublicNN, metav1.ConditionTrue, "plugin successfully saved in cluster as ConfigMap") - var respectiveCM corev1.ConfigMap - t.Log("check creation and content of respective ConfigMap") - require.EventuallyWithT(t, func(c *assert.CollectT) { - configMaps, err := GetClients().K8sClient.CoreV1().ConfigMaps(namespace.Name).List(GetCtx(), metav1.ListOptions{}) - if !assert.NoError(c, err) { - return - } - var found bool - respectiveCM, found = lo.Find(configMaps.Items, func(cm corev1.ConfigMap) bool { - return cm.Labels[consts.GatewayOperatorManagedByLabel] == consts.KongPluginInstallationManagedLabelValue && - cm.Annotations[consts.AnnotationKongPluginInstallationMappedKongPluginInstallation] == kpiNN.String() && - strings.HasPrefix(cm.Name, kpiNN.Name) - }) - if !assert.True(c, found) { - return - } - }, 15*time.Second, time.Second) - require.Equal(t, pluginExpectedContent(), respectiveCM.Data) + t.Log("waiting for the DataPlane that reference KongPluginInstallation to be ready") + checkDataPlaneStatus(t, namespace.Name, metav1.ConditionTrue, consts.ResourceReadyReason, "") + t.Log("attach configured KongPlugin with KongPluginInstallation to the HTTPRoute") + attachKongPluginBasedOnKPIToRoute(t, cleaner, httpRouteNN, kpiPublicNN) - t.Log("delete respective ConfigMap to check if it will be recreated") - var respectiveCMName = respectiveCM.Name - err = GetClients().K8sClient.CoreV1().ConfigMaps(namespace.Name).Delete(GetCtx(), respectiveCMName, metav1.DeleteOptions{}) - require.NoError(t, err) - t.Log("check recreation of respective ConfigMap") - var recreatedCM *corev1.ConfigMap - require.EventuallyWithT(t, func(c *assert.CollectT) { - recreatedCM, err = GetClients().K8sClient.CoreV1().ConfigMaps(kpiNN.Namespace).Get(GetCtx(), respectiveCMName, metav1.GetOptions{}) - assert.NoError(c, err) - }, 15*time.Second, time.Second) - require.Equal(t, pluginExpectedContent(), recreatedCM.Data) + t.Log("verify that plugin is in properly configured and works") + verifyCustomPlugins(t, ip, expectedHeadersForMyHeader) if registryCreds := GetKongPluginImageRegistryCredentialsForTests(); registryCreds != "" { - // Create secondNamespace with K8s client to check cross-namespace capabilities. - secondNamespace := &corev1.Namespace{ + // Create kpiPrivateNamespace with K8s client to check cross-namespace capabilities. + t.Log("add additional KongPluginInstallation resource from a private image") + kpiPrivateNN := k8stypes.NamespacedName{ + Name: "test-kpi-private", + Namespace: createRandomNamespace(t), + } + kpiPrivate := &operatorv1alpha1.KongPluginInstallation{ ObjectMeta: metav1.ObjectMeta{ - Name: uuid.NewString(), + Name: kpiPrivateNN.Name, + Namespace: kpiPrivateNN.Namespace, + }, + Spec: operatorv1alpha1.KongPluginInstallationSpec{ + Image: pluginMyHeader2Image, }, } - _, err := GetClients().K8sClient.CoreV1().Namespaces().Create(GetCtx(), secondNamespace, metav1.CreateOptions{}) require.NoError(t, err) - cleaner.Add(secondNamespace) - - t.Log("update KongPluginInstallation resource to a private image") - kpi, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiNN.Namespace).Get(GetCtx(), kpiNN.Name, metav1.GetOptions{}) - kpi.Spec.Image = registryUrl + "plugin-example-private/valid:0.1.0" - require.NoError(t, err) - _, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiNN.Namespace).Update(GetCtx(), kpi, metav1.UpdateOptions{}) + _, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiPrivateNN.Namespace).Create(GetCtx(), kpiPrivate, metav1.CreateOptions{}) require.NoError(t, err) t.Log("waiting for the KongPluginInstallation resource to be reconciled and report unauthenticated request") checkKongPluginInstallationConditions( - t, kpiNN, metav1.ConditionFalse, "response status code 403: denied: Unauthenticated request. Unauthenticated requests do not have permission", + t, kpiPrivateNN, metav1.ConditionFalse, "response status code 403: denied: Unauthenticated request. Unauthenticated requests do not have permission", ) - t.Log("update KongPluginInstallation resource with credentials reference in other namespace") - kpi, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiNN.Namespace).Get(GetCtx(), kpiNN.Name, metav1.GetOptions{}) + t.Log("update KongPluginInstallation resource with credentials reference in other namespace than KongPluginInstallation") + namespaceForSecret := createRandomNamespace(t) + kpiPrivate, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiPrivateNN.Namespace).Get(GetCtx(), kpiPrivateNN.Name, metav1.GetOptions{}) require.NoError(t, err) + const kindSecret = gatewayv1.Kind("Secret") secretRef := gatewayv1.SecretObjectReference{ - Kind: lo.ToPtr(gatewayv1.Kind("Secret")), - Namespace: lo.ToPtr(gatewayv1.Namespace(secondNamespace.Name)), + Kind: lo.ToPtr(kindSecret), + Namespace: lo.ToPtr(gatewayv1.Namespace(namespaceForSecret)), Name: "kong-plugin-image-registry-credentials", } - kpi.Spec.ImagePullSecretRef = &secretRef - _, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiNN.Namespace).Update(GetCtx(), kpi, metav1.UpdateOptions{}) + kpiPrivate.Spec.ImagePullSecretRef = &secretRef + _, err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiPrivateNN.Namespace).Update(GetCtx(), kpiPrivate, metav1.UpdateOptions{}) require.NoError(t, err) t.Log("waiting for the KongPluginInstallation resource to be reconciled and report missing ReferenceGrant for the Secret with credentials") checkKongPluginInstallationConditions( - t, kpiNN, metav1.ConditionFalse, fmt.Sprintf("Secret %s/%s reference not allowed by any ReferenceGrant", *secretRef.Namespace, secretRef.Name), + t, kpiPrivateNN, metav1.ConditionFalse, fmt.Sprintf("Secret %s/%s reference not allowed by any ReferenceGrant", *secretRef.Namespace, secretRef.Name), + ) + attachKPI(t, gatewayConfigNN, kpiPrivateNN) + checkDataPlaneStatus( + t, namespace.Name, metav1.ConditionFalse, dataplane.DataPlaneConditionReferencedResourcesNotAvailable, + fmt.Sprintf("something wrong with referenced KongPluginInstallation %s, please check it", client.ObjectKeyFromObject(kpiPrivate)), ) + t.Log("add missing ReferenceGrant for the Secret with credentials") refGrant := &gatewayv1beta1.ReferenceGrant{ ObjectMeta: metav1.ObjectMeta{ Name: "kong-plugin-image-registry-credentials", - Namespace: secondNamespace.Name, + Namespace: namespaceForSecret, }, Spec: gatewayv1beta1.ReferenceGrantSpec{ To: []gatewayv1beta1.ReferenceGrantTo{ { - Kind: gatewayv1.Kind("Secret"), + Kind: kindSecret, Name: lo.ToPtr(secretRef.Name), }, }, From: []gatewayv1beta1.ReferenceGrantFrom{ { - Group: gatewayv1.Group(v1alpha1.SchemeGroupVersion.Group), + Group: gatewayv1.Group(operatorv1alpha1.SchemeGroupVersion.Group), Kind: gatewayv1.Kind("KongPluginInstallation"), - Namespace: gatewayv1.Namespace(namespace.Name), + Namespace: gatewayv1.Namespace(kpiPrivate.Namespace), }, }, }, } - _, err = GetClients().GatewayClient.GatewayV1beta1().ReferenceGrants(secondNamespace.Name).Create(GetCtx(), refGrant, metav1.CreateOptions{}) + _, err = GetClients().GatewayClient.GatewayV1beta1().ReferenceGrants(namespaceForSecret).Create(GetCtx(), refGrant, metav1.CreateOptions{}) require.NoError(t, err) t.Log("waiting for the KongPluginInstallation resource to be reconciled and report missing Secret with credentials") checkKongPluginInstallationConditions( - t, kpiNN, metav1.ConditionFalse, fmt.Sprintf(`cannot retrieve secret "%s/%s"`, *secretRef.Namespace, secretRef.Name), + t, kpiPrivateNN, metav1.ConditionFalse, + fmt.Sprintf(`referenced Secret "%s/%s" not found`, *secretRef.Namespace, secretRef.Name), + ) + checkDataPlaneStatus( + t, namespace.Name, metav1.ConditionFalse, dataplane.DataPlaneConditionReferencedResourcesNotAvailable, + fmt.Sprintf("something wrong with referenced KongPluginInstallation %s, please check it", client.ObjectKeyFromObject(kpiPrivate)), ) t.Log("add missing Secret with credentials") secret := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: string(secretRef.Name), - Namespace: secondNamespace.Name, + Name: string(secretRef.Name), }, Type: corev1.SecretTypeDockerConfigJson, StringData: map[string]string{ ".dockerconfigjson": registryCreds, }, } - _, err = GetClients().K8sClient.CoreV1().Secrets(secondNamespace.Name).Create(GetCtx(), &secret, metav1.CreateOptions{}) + _, err = GetClients().K8sClient.CoreV1().Secrets(string(*secretRef.Namespace)).Create(GetCtx(), &secret, metav1.CreateOptions{}) require.NoError(t, err) t.Log("waiting for the KongPluginInstallation resource to be reconciled successfully") checkKongPluginInstallationConditions( - t, kpiNN, metav1.ConditionTrue, "plugin successfully saved in cluster as ConfigMap", + t, kpiPrivateNN, metav1.ConditionTrue, "plugin successfully saved in cluster as ConfigMap", ) - var updatedCM *corev1.ConfigMap - require.EventuallyWithT(t, func(c *assert.CollectT) { - updatedCM, err = GetClients().K8sClient.CoreV1().ConfigMaps(kpiNN.Namespace).Get(GetCtx(), respectiveCMName, metav1.GetOptions{}) - assert.NoError(c, err) - assert.Equal(c, privatePluginExpectedContent(), updatedCM.Data) - }, 15*time.Second, time.Second) + + t.Log("waiting for the DataPlane that reference KongPluginInstallation to be ready") + checkDataPlaneStatus(t, namespace.Name, metav1.ConditionTrue, consts.ResourceReadyReason, "") + t.Log("attach configured KongPlugin to the HTTPRoute") + attachKongPluginBasedOnKPIToRoute(t, cleaner, httpRouteNN, kpiPrivateNN) + verifyCustomPlugins(t, ip, expectedHeadersForMyHeader, expectedHeadersForMyHeader2) } else { t.Log("skipping private image test - no credentials provided") } +} - t.Log("delete KongPluginInstallation resource") - err = GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(kpiNN.Namespace).Delete(GetCtx(), kpiNN.Name, metav1.DeleteOptions{}) +func deployGatewayWithKPI( + t *testing.T, cleaner *clusters.Cleaner, namespace string, +) (gatewayIPAddress string, gatewayConfigNN, httpRouteNN k8stypes.NamespacedName) { + gatewayConfig := helpers.GenerateGatewayConfiguration(namespace) + t.Logf("deploying GatewayConfiguration %s/%s", gatewayConfig.Namespace, gatewayConfig.Name) + gatewayConfig, err := GetClients().OperatorClient.ApisV1beta1().GatewayConfigurations(namespace).Create(GetCtx(), gatewayConfig, metav1.CreateOptions{}) require.NoError(t, err) - t.Log("check deletion of respective ConfigMap") - require.EventuallyWithT(t, func(c *assert.CollectT) { - _, err := GetClients().K8sClient.CoreV1().ConfigMaps(kpiNN.Namespace).Get(GetCtx(), respectiveCM.Name, metav1.GetOptions{}) - assert.True(c, apierrors.IsNotFound(err), "ConfigMap not deleted") - }, 15*time.Second, time.Second) + cleaner.Add(gatewayConfig) + + gatewayClass := helpers.MustGenerateGatewayClass(t, gatewayv1.ParametersReference{ + Group: gatewayv1.Group(operatorv1beta1.SchemeGroupVersion.Group), + Kind: gatewayv1.Kind("GatewayConfiguration"), + Namespace: (*gatewayv1.Namespace)(&gatewayConfig.Namespace), + Name: gatewayConfig.Name, + }) + t.Logf("deploying GatewayClass %s", gatewayClass.Name) + gatewayClass, err = GetClients().GatewayClient.GatewayV1().GatewayClasses().Create(GetCtx(), gatewayClass, metav1.CreateOptions{}) + require.NoError(t, err) + cleaner.Add(gatewayClass) + + gatewayNSN := k8stypes.NamespacedName{ + Name: uuid.NewString(), + Namespace: namespace, + } + gateway := helpers.GenerateGateway(gatewayNSN, gatewayClass) + t.Logf("deploying Gateway %s/%s", gateway.Namespace, gateway.Name) + gateway, err = GetClients().GatewayClient.GatewayV1().Gateways(namespace).Create(GetCtx(), gateway, metav1.CreateOptions{}) + require.NoError(t, err) + cleaner.Add(gateway) + + t.Logf("verifying Gateway %s/%s gets an IP address", gateway.Namespace, gateway.Name) + require.Eventually(t, testutils.GatewayIPAddressExist(t, GetCtx(), gatewayNSN, clients), 4*testutils.SubresourceReadinessWait, time.Second) + gateway = testutils.MustGetGateway(t, GetCtx(), gatewayNSN, clients) + + t.Log("deploying backend deployment (httpbin) of HTTPRoute") + container := generators.NewContainer("httpbin", testutils.HTTPBinImage, 80) + deployment := generators.NewDeploymentForContainer(container) + deployment, err = GetEnv().Cluster().Client().AppsV1().Deployments(namespace).Create(GetCtx(), deployment, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("exposing deployment %s via service", deployment.Name) + service := generators.NewServiceForDeployment(deployment, corev1.ServiceTypeClusterIP) + _, err = GetEnv().Cluster().Client().CoreV1().Services(namespace).Create(GetCtx(), service, metav1.CreateOptions{}) + require.NoError(t, err) + + httpRoute := helpers.GenerateHTTPRoute(namespace, gateway.Name, service.Name) + t.Logf("creating httproute %s/%s to access deployment %s via kong", httpRoute.Namespace, httpRoute.Name, deployment.Name) + require.EventuallyWithT(t, + func(c *assert.CollectT) { + result, err := GetClients().GatewayClient.GatewayV1().HTTPRoutes(namespace).Create(GetCtx(), httpRoute, metav1.CreateOptions{}) + if err != nil { + t.Logf("failed to deploy httproute: %v", err) + c.Errorf("failed to deploy httproute: %v", err) + return + } + cleaner.Add(result) + }, + testutils.DefaultIngressWait, testutils.WaitIngressTick, + ) + + return gateway.Status.Addresses[0].Value, client.ObjectKeyFromObject(gatewayConfig), client.ObjectKeyFromObject(httpRoute) } func checkKongPluginInstallationConditions( @@ -202,7 +278,6 @@ func checkKongPluginInstallationConditions( expectedMessage string, ) { t.Helper() - require.EventuallyWithT(t, func(c *assert.CollectT) { kpi, err := GetClients().OperatorClient.ApisV1alpha1().KongPluginInstallations(namespacedName.Namespace).Get(GetCtx(), namespacedName.Name, metav1.GetOptions{}) if !assert.NoError(c, err) { @@ -212,27 +287,110 @@ func checkKongPluginInstallationConditions( return } status := kpi.Status.Conditions[0] - assert.EqualValues(c, v1alpha1.KongPluginInstallationConditionStatusAccepted, status.Type) + assert.EqualValues(c, operatorv1alpha1.KongPluginInstallationConditionStatusAccepted, status.Type) assert.EqualValues(c, conditionStatus, status.Status) if conditionStatus == metav1.ConditionTrue { - assert.EqualValues(c, v1alpha1.KongPluginInstallationReasonReady, status.Reason) + assert.EqualValues(c, operatorv1alpha1.KongPluginInstallationReasonReady, status.Reason) } else { - assert.EqualValues(c, v1alpha1.KongPluginInstallationReasonFailed, status.Reason) + assert.EqualValues(c, operatorv1alpha1.KongPluginInstallationReasonFailed, status.Reason) } assert.Contains(c, status.Message, expectedMessage) }, 15*time.Second, time.Second) } -func pluginExpectedContent() map[string]string { - return map[string]string{ - "handler.lua": "handler-content\n", - "schema.lua": "schema-content\n", +func attachKPI(t *testing.T, gatewayConfigNN k8stypes.NamespacedName, kpiNN k8stypes.NamespacedName) { + t.Helper() + gatewayConfig, err := GetClients().OperatorClient.ApisV1beta1().GatewayConfigurations(gatewayConfigNN.Namespace).Get(GetCtx(), gatewayConfigNN.Name, metav1.GetOptions{}) + require.NoError(t, err) + gatewayConfig.Spec.DataPlaneOptions.PluginsToInstall = append(gatewayConfig.Spec.DataPlaneOptions.PluginsToInstall, operatorv1beta1.NamespacedName(kpiNN)) + _, err = GetClients().OperatorClient.ApisV1beta1().GatewayConfigurations(gatewayConfigNN.Namespace).Update(GetCtx(), gatewayConfig, metav1.UpdateOptions{}) + require.NoError(t, err) +} + +func attachKongPluginBasedOnKPIToRoute(t *testing.T, cleaner *clusters.Cleaner, httpRouteNN, kpiNN k8stypes.NamespacedName) { + kongPluginName := kpiNN.Name + "-plugin" + // To have it in the same namespace as the HTTPRoute to which it is attached. + kongPluginNamespace := httpRouteNN.Namespace + kongPlugin := configurationv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: kongPluginName, + Namespace: kongPluginNamespace, + }, + PluginName: kpiNN.Name, } + _, err := GetClients().ConfigurationClient.ConfigurationV1().KongPlugins(kongPluginNamespace).Create(GetCtx(), &kongPlugin, metav1.CreateOptions{}) + require.NoError(t, err) + cleaner.Add(&kongPlugin) + + t.Logf("attaching KongPlugin to GatewayConfiguration") + // Update httpRoute with KongPlugin + httpRoute, err := GetClients().GatewayClient.GatewayV1().HTTPRoutes(httpRouteNN.Namespace).Get(GetCtx(), httpRouteNN.Name, metav1.GetOptions{}) + require.NoError(t, err) + const kpAnnotation = "konghq.com/plugins" + httpRoute.Annotations[kpAnnotation] = strings.Join( + append(strings.Split(httpRoute.Annotations[kpAnnotation], ","), kongPluginName), ",", + ) + _, err = GetClients().GatewayClient.GatewayV1().HTTPRoutes(httpRouteNN.Namespace).Update(GetCtx(), httpRoute, metav1.UpdateOptions{}) + require.NoError(t, err) } -func privatePluginExpectedContent() map[string]string { - return map[string]string{ - "handler.lua": "handler-content-private\n", - "schema.lua": "schema-content-private\n", +func checkDataPlaneStatus( + t *testing.T, + namespace string, + expectedConditionStatus metav1.ConditionStatus, + expectedConditionReason consts.ConditionReason, + expectedConditionMessage string, +) { + t.Helper() + var dp operatorv1beta1.DataPlane + require.EventuallyWithT(t, func(c *assert.CollectT) { + dps, err := GetClients().OperatorClient.ApisV1beta1().DataPlanes(namespace).List(GetCtx(), metav1.ListOptions{}) + if !assert.NoError(c, err) { + return + } + if assert.Len(c, dps.Items, 1) { + dp = dps.Items[0] + } + if !assert.Len(c, dp.Status.Conditions, 1) { + return + } + + condition := dp.Status.Conditions[0] + assert.EqualValues(c, consts.ReadyType, condition.Type) + assert.EqualValues(c, expectedConditionStatus, condition.Status) + assert.EqualValues(c, expectedConditionReason, condition.Reason) + assert.Equal(c, expectedConditionMessage, condition.Message) + }, 2*time.Minute, time.Second) +} + +func verifyCustomPlugins(t *testing.T, ip string, expectedHeaders ...http.Header) { + t.Log("verify that plugin is in place and works") + httpClient, err := helpers.CreateHTTPClient(nil, "") + require.NoError(t, err) + require.EventuallyWithT(t, func(c *assert.CollectT) { + resp, err := httpClient.Get(fmt.Sprintf("http://%s/test", ip)) + if !assert.NoError(c, err) { + return + } + defer resp.Body.Close() + if !assert.Equal(c, http.StatusOK, resp.StatusCode) { + return + } + for _, h := range expectedHeaders { + for k, v := range h { + assert.Equal(c, v, resp.Header.Values(k)) + } + } + }, 15*time.Second, time.Second) +} + +func createRandomNamespace(t *testing.T) string { + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: uuid.NewString(), + }, } + _, err := GetClients().K8sClient.CoreV1().Namespaces().Create(GetCtx(), namespace, metav1.CreateOptions{}) + require.NoError(t, err) + return namespace.Name } From 3f14182a263d2474e3d43680154176af17644b79 Mon Sep 17 00:00:00 2001 From: Jakub Warczarek Date: Mon, 23 Sep 2024 17:14:10 +0200 Subject: [PATCH 2/2] chore(refactor): requeue boolean instead of ctrl.Result --- controller/dataplane/controller.go | 12 ++--- .../dataplane/controller_reconciler_utils.go | 48 +++++++++---------- .../test_kongplugininstallation.go | 5 +- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/controller/dataplane/controller.go b/controller/dataplane/controller.go index 547f816de..635466213 100644 --- a/controller/dataplane/controller.go +++ b/controller/dataplane/controller.go @@ -178,12 +178,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } log.Trace(logger, "ensuring generation of deployment configuration for KongPluginInstallations configured for DataPlane", dataplane) - kpisForDeployment, reconcileResult, err := ensureMappedConfigMapToKongPluginInstallationForDataPlane(ctx, logger, r.Client, dataplane) + kpisForDeployment, requeue, err := ensureMappedConfigMapToKongPluginInstallationForDataPlane(ctx, logger, r.Client, dataplane) if err != nil { return ctrl.Result{}, fmt.Errorf("cannot ensure KongPluginInstallation for DataPlane: %w", err) } - if reconcileResult.Requeue { - return reconcileResult, nil + if requeue { + return ctrl.Result{Requeue: true}, nil } deploymentOpts = append(deploymentOpts, withCustomPlugins(kpisForDeployment...)) @@ -220,10 +220,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, nil } - if reconcileResult, err := ensureDataPlaneReadyStatus(ctx, r.Client, logger, dataplane, dataplane.Generation); err != nil { + if res, err := ensureDataPlaneReadyStatus(ctx, r.Client, logger, dataplane, dataplane.Generation); err != nil { return ctrl.Result{}, err - } else if reconcileResult.Requeue { - return reconcileResult, nil + } else if res.Requeue { + return res, nil } log.Debug(logger, "reconciliation complete for DataPlane resource", dataplane) diff --git a/controller/dataplane/controller_reconciler_utils.go b/controller/dataplane/controller_reconciler_utils.go index bab2f8762..8f5aafa80 100644 --- a/controller/dataplane/controller_reconciler_utils.go +++ b/controller/dataplane/controller_reconciler_utils.go @@ -14,7 +14,6 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1" @@ -92,25 +91,25 @@ func (r *Reconciler) ensureDataPlaneAddressesStatus( // of custom plugins that are intended to be used to generate a Deployment. func ensureMappedConfigMapToKongPluginInstallationForDataPlane( ctx context.Context, logger logr.Logger, c client.Client, dataplane *operatorv1beta1.DataPlane, -) ([]customPlugin, ctrl.Result, error) { +) (cps []customPlugin, requeue bool, err error) { configMapsOwned, err := findCustomPluginConfigMapsOwnedByDataPlane(ctx, c, dataplane) if err != nil { - return nil, ctrl.Result{}, err + return nil, false, err } configMapsToRetain := make(map[types.NamespacedName]struct{}, len(configMapsOwned)) - var cps []customPlugin for _, kpiNN := range dataplane.Spec.PluginsToInstall { kpiNN := types.NamespacedName(kpiNN) if kpiNN.Namespace == "" { kpiNN.Namespace = dataplane.Namespace } - cp, reconciliationResult, err := populateDedicatedConfigMapForKongPluginInstallation( + var cp customPlugin + cp, requeue, err = populateDedicatedConfigMapForKongPluginInstallation( ctx, logger, c, configMapsOwned, kpiNN, dataplane, ) - if err != nil || reconciliationResult.Requeue { - return nil, reconciliationResult, err + if err != nil || requeue { + return nil, requeue, err } configMapsToRetain[cp.ConfigMapNN] = struct{}{} cps = append(cps, cp) @@ -118,12 +117,12 @@ func ensureMappedConfigMapToKongPluginInstallationForDataPlane( for _, cm := range configMapsOwned { if _, retain := configMapsToRetain[client.ObjectKeyFromObject(&cm)]; !retain { if err := c.Delete(ctx, &cm); client.IgnoreNotFound(err) != nil { - return nil, ctrl.Result{}, err + return nil, false, err } } } - return cps, ctrl.Result{}, nil + return cps, false, nil } func findCustomPluginConfigMapsOwnedByDataPlane( @@ -146,13 +145,13 @@ func populateDedicatedConfigMapForKongPluginInstallation( cms []corev1.ConfigMap, kpiNN types.NamespacedName, dataplane *operatorv1beta1.DataPlane, -) (customPlugin, ctrl.Result, error) { +) (cp customPlugin, requeue bool, err error) { kpi, ready, err := verifyKPIReadinessForDataPlane(ctx, logger, c, dataplane, kpiNN) if err != nil { - return customPlugin{}, ctrl.Result{}, err + return customPlugin{}, false, err } if !ready { - return customPlugin{}, ctrl.Result{Requeue: true}, nil + return customPlugin{}, true, nil } var underlyingCM corev1.ConfigMap @@ -162,7 +161,7 @@ func populateDedicatedConfigMapForKongPluginInstallation( } log.Trace(logger, fmt.Sprintf("Fetch underlying ConfigMap %s for KongPluginInstallation", backingCMNN), kpi) if err := c.Get(ctx, backingCMNN, &underlyingCM); err != nil { - return customPlugin{}, ctrl.Result{}, fmt.Errorf("could not fetch underlying ConfigMap to clone %s: %w", backingCMNN, err) + return customPlugin{}, false, fmt.Errorf("could not fetch underlying ConfigMap to clone %s: %w", backingCMNN, err) } log.Trace(logger, "Find ConfigMap mapped to KongPluginInstallation", kpi) @@ -181,35 +180,36 @@ func populateDedicatedConfigMapForKongPluginInstallation( k8sresources.AnnotateConfigMapWithKongPluginInstallation(&cm, kpi) cm.Data = underlyingCM.Data if err := c.Create(ctx, &cm); err != nil { - return customPlugin{}, ctrl.Result{}, fmt.Errorf("could not create new ConfigMap for KongPluginInstallation: %w", err) + return customPlugin{}, false, fmt.Errorf("could not create new ConfigMap for KongPluginInstallation: %w", err) } case 1: log.Trace(logger, fmt.Sprintf("Check if update existing ConfigMap %s for KongPluginInstallation", client.ObjectKeyFromObject(&cm)), kpi) cm = mappedConfigMapForKPI[0] if maps.Equal(cm.Data, underlyingCM.Data) { log.Trace(logger, fmt.Sprintf("Nothing to update in existing ConfigMap %s for KongPluginInstallation", client.ObjectKeyFromObject(&cm)), kpi) - return customPlugin{}, ctrl.Result{}, nil - } - log.Trace(logger, fmt.Sprintf("Update existing ConfigMap %s for KongPluginInstallation", client.ObjectKeyFromObject(&cm)), kpi) - cm.Data = underlyingCM.Data - if err := c.Update(ctx, &cm); err != nil { - if k8serrors.IsConflict(err) { - return customPlugin{}, ctrl.Result{Requeue: true}, nil + } else { + log.Trace(logger, fmt.Sprintf("Update existing ConfigMap %s for KongPluginInstallation", client.ObjectKeyFromObject(&cm)), kpi) + cm.Data = underlyingCM.Data + if err := c.Update(ctx, &cm); err != nil { + if k8serrors.IsConflict(err) { + return customPlugin{}, true, nil + } + return customPlugin{}, false, fmt.Errorf("could not update mapped: %w", err) } - return customPlugin{}, ctrl.Result{}, fmt.Errorf("could not update mapped: %w", err) } + default: // It should never happen. names := strings.Join(lo.Map(mappedConfigMapForKPI, func(cm corev1.ConfigMap, _ int) string { return client.ObjectKeyFromObject(&cm).String() }), ", ") - return customPlugin{}, ctrl.Result{}, fmt.Errorf("unexpected error happened - more than one ConfigMap found: %s", names) + return customPlugin{}, false, fmt.Errorf("unexpected error happened - more than one ConfigMap found: %s", names) } return customPlugin{ Name: kpi.Name, ConfigMapNN: client.ObjectKeyFromObject(&cm), Generation: kpi.Generation, - }, ctrl.Result{}, nil + }, false, nil } // verifyKPIReadinessForDataPlane updates DataPlane status conditions based on status of KPI object. diff --git a/test/integration/test_kongplugininstallation.go b/test/integration/test_kongplugininstallation.go index bf72f6d8f..b40ba7dba 100644 --- a/test/integration/test_kongplugininstallation.go +++ b/test/integration/test_kongplugininstallation.go @@ -96,7 +96,7 @@ func TestKongPluginInstallationEssentials(t *testing.T) { t.Log("attach configured KongPlugin with KongPluginInstallation to the HTTPRoute") attachKongPluginBasedOnKPIToRoute(t, cleaner, httpRouteNN, kpiPublicNN) - t.Log("verify that plugin is in properly configured and works") + t.Log("verify that plugin is properly configured and works") verifyCustomPlugins(t, ip, expectedHeadersForMyHeader) if registryCreds := GetKongPluginImageRegistryCredentialsForTests(); registryCreds != "" { @@ -202,6 +202,7 @@ func TestKongPluginInstallationEssentials(t *testing.T) { checkDataPlaneStatus(t, namespace.Name, metav1.ConditionTrue, consts.ResourceReadyReason, "") t.Log("attach configured KongPlugin to the HTTPRoute") attachKongPluginBasedOnKPIToRoute(t, cleaner, httpRouteNN, kpiPrivateNN) + t.Log("verify that plugin is properly configured and works") verifyCustomPlugins(t, ip, expectedHeadersForMyHeader, expectedHeadersForMyHeader2) } else { t.Log("skipping private image test - no credentials provided") @@ -364,7 +365,7 @@ func checkDataPlaneStatus( } func verifyCustomPlugins(t *testing.T, ip string, expectedHeaders ...http.Header) { - t.Log("verify that plugin is in place and works") + t.Helper() httpClient, err := helpers.CreateHTTPClient(nil, "") require.NoError(t, err) require.EventuallyWithT(t, func(c *assert.CollectT) {