Skip to content

Commit

Permalink
feat: allow assigning KongPluginInstallation to GatewayConfiguration …
Browse files Browse the repository at this point in the history
…and DataPlane
  • Loading branch information
programmer04 committed Aug 12, 2024
1 parent 9f4550a commit 1c8f88f
Show file tree
Hide file tree
Showing 13 changed files with 319 additions and 6 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
[#387](https://github.com/Kong/gateway-operator/pull/387)
- Introduce `KongPluginInstallation` CRD to allow installing custom Kong
plugins distributed as container images.
[#400](https://github.com/Kong/gateway-operator/pull/400), [#424](https://github.com/Kong/gateway-operator/pull/424), [#474](https://github.com/Kong/gateway-operator/pull/474)
[#400](https://github.com/Kong/gateway-operator/pull/400), [#424](https://github.com/Kong/gateway-operator/pull/424), [#474](https://github.com/Kong/gateway-operator/pull/474), [#476](https://github.com/Kong/gateway-operator/pull/476)
- Extended `DataPlane` API with a possibility to specify `PodDisruptionBudget` to be
created for the `DataPlane` deployments via `spec.resources.podDisruptionBudget`.
[#464](https://github.com/Kong/gateway-operator/pull/464)
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/dataplane_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,11 @@ type DataPlaneOptions struct {

// +optional
Resources DataPlaneResources `json:"resources"`

// PluginToInstall is a list of KongPluginInstallation resources that
// will be installed and available in the DataPlane.
// +optional
PluginsToInstall []NamespacedName `json:"pluginsToInstall,omitempty"`
}

// DataPlaneResources defines the resources that will be created and managed
Expand Down
6 changes: 6 additions & 0 deletions api/v1beta1/gatewayconfiguration_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ type GatewayConfigDataPlaneOptions struct {

// +optional
Network GatewayConfigDataPlaneNetworkOptions `json:"network"`

// PluginToInstall is a list of KongPluginInstallation resources that
// will be installed and available in the Gateways (DataPlanes) that
// use this GatewayConfig.
// +optional
PluginsToInstall []NamespacedName `json:"pluginsToInstall,omitempty"`
}

// GatewayConfigDataPlaneNetworkOptions defines network related options for a DataPlane.
Expand Down
10 changes: 10 additions & 0 deletions api/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 16 additions & 0 deletions config/crd/bases/gateway-operator.konghq.com_dataplanes.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8933,6 +8933,22 @@ spec:
type: object
type: object
type: object
pluginsToInstall:
description: |-
PluginToInstall is a list of KongPluginInstallation resources that
will be installed and available in the DataPlane.
items:
description: NamespacedName is a resource identified by name and
optional namespace.
properties:
name:
type: string
namespace:
type: string
required:
- name
type: object
type: array
resources:
description: |-
DataPlaneResources defines the resources that will be created and managed
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17190,6 +17190,23 @@ spec:
type: object
type: object
type: object
pluginsToInstall:
description: |-
PluginToInstall is a list of KongPluginInstallation resources that
will be installed and available in the Gateways (DataPlanes) that
use this GatewayConfig.
items:
description: NamespacedName is a resource identified by name
and optional namespace.
properties:
name:
type: string
namespace:
type: string
required:
- name
type: object
type: array
type: object
type: object
status:
Expand Down
16 changes: 16 additions & 0 deletions config/crd/dataplane/gateway-operator.konghq.com_dataplanes.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8933,6 +8933,22 @@ spec:
type: object
type: object
type: object
pluginsToInstall:
description: |-
PluginToInstall is a list of KongPluginInstallation resources that
will be installed and available in the DataPlane.
items:
description: NamespacedName is a resource identified by name and
optional namespace.
properties:
name:
type: string
namespace:
type: string
required:
- name
type: object
type: array
resources:
description: |-
DataPlaneResources defines the resources that will be created and managed
Expand Down
149 changes: 149 additions & 0 deletions config/samples/gateway-kongplugininstallation-httproute.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
apiVersion: v1
kind: Service
metadata:
name: echo
spec:
ports:
- protocol: TCP
name: http
port: 80
targetPort: http
selector:
app: echo
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: echo
name: echo
spec:
replicas: 1
selector:
matchLabels:
app: echo
template:
metadata:
labels:
app: echo
spec:
containers:
- name: echo
image: registry.k8s.io/e2e-test-images/agnhost:2.40
command:
- /agnhost
- netexec
- --http-port=8080
ports:
- containerPort: 8080
name: http
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
resources:
requests:
cpu: 10m
---
kind: GatewayConfiguration
apiVersion: gateway-operator.konghq.com/v1beta1
metadata:
name: kong
namespace: default
spec:
dataPlaneOptions:
deployment:
replicas: 2
podTemplateSpec:
spec:
containers:
- name: proxy
# renovate: datasource=docker versioning=docker
image: kong/kong-gateway:3.7
readinessProbe:
initialDelaySeconds: 1
periodSeconds: 1
pluginsToInstall:
- name: additional-custom-plugin
controlPlaneOptions:
deployment:
podTemplateSpec:
spec:
containers:
- name: controller
# renovate: datasource=docker versioning=docker
image: kong/kubernetes-ingress-controller:3.2.3
readinessProbe:
initialDelaySeconds: 1
periodSeconds: 1
---
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
name: kong
spec:
controllerName: konghq.com/gateway-operator
parametersRef:
group: gateway-operator.konghq.com
kind: GatewayConfiguration
name: kong
namespace: default
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: kong
namespace: default
spec:
gatewayClassName: kong
listeners:
- name: http
protocol: HTTP
port: 80
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: httproute-echo
namespace: default
annotations:
konghq.com/strip-path: "true"
konghq.com/plugins: kong-custom-plugin
spec:
parentRefs:
- name: kong
rules:
- matches:
- path:
type: PathPrefix
value: /echo
backendRefs:
- name: echo
kind: Service
port: 80
---
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
name: kong-custom-plugin
plugin: additional-custom-plugin
---
kind: KongPluginInstallation
apiVersion: gateway-operator.konghq.com/v1alpha1
metadata:
name: additional-custom-plugin
spec:
image: northamerica-northeast1-docker.pkg.dev/k8s-team-playground/plugin-example/myheader
19 changes: 18 additions & 1 deletion controller/dataplane/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1"
operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1"
"github.com/kong/gateway-operator/controller/pkg/ctxinjector"
"github.com/kong/gateway-operator/controller/pkg/log"
Expand Down Expand Up @@ -166,7 +167,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
return ctrl.Result{}, nil // no need to requeue, the update will trigger.
}

log.Trace(logger, "ensuring DataPlane has service addesses in status", dataplaneIngressService)
log.Trace(logger, "ensuring DataPlane has service addresses in status", dataplaneIngressService)
if updated, err := r.ensureDataPlaneAddressesStatus(ctx, logger, dataplane, dataplaneIngressService); err != nil {
return ctrl.Result{}, err
} else if updated {
Expand All @@ -180,6 +181,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
deploymentOpts := []k8sresources.DeploymentOpt{
labelSelectorFromDataPlaneStatusSelectorDeploymentOpt(dataplane),
}

var kpis []operatorv1alpha1.KongPluginInstallation
for _, plugin := range dataplane.Spec.PluginsToInstall {
var kpi operatorv1alpha1.KongPluginInstallation
if err := r.Get(ctx, types.NamespacedName(plugin), &kpi); err != nil {
if k8serrors.IsNotFound(err) {
return ctrl.Result{}, nil
}
return ctrl.Result{}, fmt.Errorf("could not fetch KongPluginInstallation %s: %w", plugin, err)
}
kpis = append(kpis, kpi)
}
if len(kpis) > 0 {
deploymentOpts = append(deploymentOpts, withCustomPlugins(kpis))
}

deploymentBuilder := NewDeploymentBuilder(logger.WithName("deployment_builder"), r.Client).
WithBeforeCallbacks(r.Callbacks.BeforeDeployment).
WithAfterCallbacks(r.Callbacks.AfterDeployment).
Expand Down
59 changes: 59 additions & 0 deletions controller/dataplane/custom_plugin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package dataplane

import (
"strings"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"

operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1"
k8sresources "github.com/kong/gateway-operator/pkg/utils/kubernetes/resources"
)

func withCustomPlugins(kpis []operatorv1alpha1.KongPluginInstallation) k8sresources.DeploymentOpt {
var (
kpisNames = make([]string, 0, len(kpis)+1)
kpisVolumeMounts = make([]corev1.VolumeMount, 0, len(kpis))
kpisVolumes = make([]corev1.Volume, 0, len(kpis))
)
kpisNames = append(kpisNames, "bundled") // To preserve plugins available in the default image.
for _, kpi := range kpis {
kpisNames = append(kpisNames, kpi.Name)
kpisVolumeMounts = append(kpisVolumeMounts, corev1.VolumeMount{
Name: kpi.Name,
MountPath: "/opt/kong/plugins/" + kpi.Name,
})
kpisVolumes = append(kpisVolumes, corev1.Volume{
Name: kpi.Name,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: kpi.Status.UnderlyingConfigMapName,
},
},
},
})
}

return func(deployment *appsv1.Deployment) {
deployment.Spec.Template.Spec.Containers[0].Env = append(
deployment.Spec.Template.Spec.Containers[0].Env,
corev1.EnvVar{
Name: "KONG_PLUGINS",
Value: strings.Join(kpisNames, ","),
},
corev1.EnvVar{
Name: "KONG_LUA_PACKAGE_PATH",
Value: "/opt/?.lua;;",
},
)
deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(
deployment.Spec.Template.Spec.Containers[0].VolumeMounts,
kpisVolumeMounts...,
)
deployment.Spec.Template.Spec.Volumes = append(
deployment.Spec.Template.Spec.Volumes,
kpisVolumes...,
)
}
}
2 changes: 1 addition & 1 deletion controller/gateway/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -444,7 +444,7 @@ func (r *Reconciler) provisionDataPlane(
// if not configured in gatewayconfiguration, compare deployment option of dataplane with an empty one.
expectedDataPlaneOptions := &operatorv1beta1.DataPlaneOptions{}
if gatewayConfig.Spec.DataPlaneOptions != nil {
expectedDataPlaneOptions = gatewayConfigDataPlaneOptionsToDataPlaneOptions(*gatewayConfig.Spec.DataPlaneOptions)
expectedDataPlaneOptions = gatewayConfigDataPlaneOptionsToDataPlaneOptions(gatewayConfig.Namespace, *gatewayConfig.Spec.DataPlaneOptions)
}
// Don't require setting defaults for DataPlane when using Gateway CRD.
setDataPlaneOptionsDefaults(expectedDataPlaneOptions, r.DefaultDataPlaneImage)
Expand Down
Loading

0 comments on commit 1c8f88f

Please sign in to comment.