feat(crd) migrate to Kubebuilder (#1971)

* Add dedicated types for KongIngress subsections Add new KongIngressUpstream, KongIngressService, and KongIngressRoute types for the KongIngress subsections, rather than using the go-kong types directly. These allow for kubebuilder validation without much annotation of go-kong types, and avoid unwanted fields (ID, name, etc.) that aren't actually exposed by KongIngress. * Enable all-Kubebuilder CRD generation CRD manifests are now entirely generated using stock Kubebuilder tools and type annotations. * Remove unnecessary meta fields from ConfigSource types The ConfigSource types used in KongPlugin/KongClusterPlugin were previously full Kubernetes objects, with associated metadata. These resources are not actually Kubernetes objects (they are never created independent of plugins) and should not have included this metadata. * Add upgrade test Add a new E2E test that performs an upgrade from the previous minor version's latest patch, and ensures that routes work as expected both before and after the upgrade.
Kong · Dec 15, 2021 · 9c2bc19 · 9c2bc19
1 parent 4ad6442
commit 9c2bc19
Show file tree

Hide file tree

Showing 36 changed files with 1,199 additions and 597 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -49,6 +49,20 @@ you should edit it (`kubectl edit validatingwebhookconfiguration
 kong-validations`) and add `kongclusterplugins` under the `resources` block for
 the `configuration.konghq.com` API group.
 
+#### Breaking changes
+
+- You must upgrade to 2.0.x before upgrading to 2.1.x to properly handle the
+  transition from apiextensions.k8s.io/v1beta1 CRDs to apiextensions.k8s.io/v1
+  CRDSs. CRDs are now generated from their underlying Go structures to avoid
+  accidental mismatches between implementation and Kubernetes configuration.
+  KongIngresses previously included `healthchecks.passive.unhealthy.timeout`
+  and `healthchecks.active.unhealthy.timeout` fields that did not match the
+  corresponding Kong configuration and had no effect. These are now
+  `healthchecks.passive.unhealthy.timeouts` and
+  `healthchecks.active.unhealthy.timeouts`, respectively. If you use these
+  fields, you must rename them in your KongIngresses before upgrading.
+  [#1971](https://github.com/Kong/kubernetes-ingress-controller/pull/1971)
+
 #### Added
 
 - Added validation for `Gateway` objects in the admission webhook

diff --git a/Makefile b/Makefile
@@ -24,7 +24,7 @@ export GO111MODULE=on
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.2)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0)
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
@@ -52,7 +52,7 @@ endef
 # Build
 # ------------------------------------------------------------------------------
 
-CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false,allowDangerousTypes=true"
+CRD_OPTIONS ?= "+crd:allowDangerousTypes=true"
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -119,8 +119,7 @@ manifests: manifests.crds manifests.single
 
 .PHONY: manifests.crds
 manifests.crds: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=kong-ingress webhook paths="./..." output:crd:artifacts:config=build/config/crd/bases
-	go run hack/generators/manifests/main.go --input-directory build/config/crd/bases/ --output-directory config/crd/bases
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=kong-ingress webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
 .PHONY: manifests.single
 manifests.single: kustomize ## Compose single-file deployment manifests from building blocks

diff --git a/config/crd/bases/configuration.konghq.com_kongclusterplugins.yaml b/config/crd/bases/configuration.konghq.com_kongclusterplugins.yaml
@@ -1,6 +1,11 @@
+
+---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
   name: kongclusterplugins.configuration.konghq.com
 spec:
   group: configuration.konghq.com
@@ -11,7 +16,6 @@ spec:
     shortNames:
     - kcp
     singular: kongclusterplugin
-  preserveUnknownFields: false
   scope: Cluster
   versions:
   - additionalPrinterColumns:
@@ -38,6 +42,11 @@ spec:
       openAPIV3Schema:
         description: KongClusterPlugin is the Schema for the kongclusterplugins API
         properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
           config:
             description: Config contains the plugin configuration.
             type: object
@@ -70,6 +79,13 @@ spec:
           disabled:
             description: Disabled set if the plugin is disabled or not
             type: boolean
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
           plugin:
             description: PluginName is the name of the plugin to which to apply the
               config

diff --git a/config/crd/bases/configuration.konghq.com_kongconsumers.yaml b/config/crd/bases/configuration.konghq.com_kongconsumers.yaml
@@ -1,6 +1,11 @@
+
+---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
   name: kongconsumers.configuration.konghq.com
 spec:
   group: configuration.konghq.com
@@ -11,7 +16,6 @@ spec:
     shortNames:
     - kc
     singular: kongconsumer
-  preserveUnknownFields: false
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
@@ -28,6 +32,11 @@ spec:
       openAPIV3Schema:
         description: KongConsumer is the Schema for the kongconsumers API
         properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
           credentials:
             description: Credentials are references to secrets containing a credential
               to be provisioned in Kong.
@@ -38,6 +47,13 @@ spec:
             description: CustomID existing unique ID for the consumer - useful for
               mapping Kong with users in your existing database
             type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
           username:
             description: Username unique username of the consumer.
             type: string

diff --git a/config/crd/bases/configuration.konghq.com_kongingresses.yaml b/config/crd/bases/configuration.konghq.com_kongingresses.yaml
@@ -1,6 +1,11 @@
+
+---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
   name: kongingresses.configuration.konghq.com
 spec:
   group: configuration.konghq.com
@@ -11,22 +16,34 @@ spec:
     shortNames:
     - ki
     singular: kongingress
-  preserveUnknownFields: false
   scope: Namespaced
   versions:
   - name: v1
     schema:
       openAPIV3Schema:
         description: KongIngress is the Schema for the kongingresses API
         properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
           proxy:
+            description: KongIngressService contains KongIngress service configuration
             properties:
-              path:
-                type: string
-                pattern: ^/.*$
               connect_timeout:
                 minimum: 0
                 type: integer
+              path:
+                pattern: ^/.*$
+                type: string
               protocol:
                 enum:
                 - http
@@ -48,7 +65,7 @@ spec:
                 type: integer
             type: object
           route:
-            description: Route represents a Route in Kong. Read https://getkong.org/docs/0.13.x/admin-api/#Route-object
+            description: KongIngressRoute contains KongIngress route configuration
             properties:
               headers:
                 additionalProperties:
@@ -84,12 +101,6 @@ spec:
               regex_priority:
                 type: integer
               request_buffering:
-                description: "Kong buffers requests and responses by default. Buffering
-                  is not always desired, for instance if large payloads are being
-                  proxied using HTTP 1.1 chunked encoding. \n The request and response
-                  route buffering options are enabled by default and allow the user
-                  to disable buffering if desired for their use case. \n SEE ALSO:
-                  - https://github.com/Kong/kong/pull/6057 - https://docs.konghq.com/2.2.x/admin-api/#route-object"
                 type: boolean
               response_buffering:
                 type: boolean
@@ -101,7 +112,7 @@ spec:
                 type: boolean
             type: object
           upstream:
-            description: Upstream represents an Upstream in Kong.
+            description: KongIngressUpstream contains KongIngress upstream configuration
             properties:
               algorithm:
                 enum:
@@ -150,6 +161,10 @@ spec:
                       http_path:
                         pattern: ^/.*$
                         type: string
+                      https_sni:
+                        type: string
+                      https_verify_certificate:
+                        type: boolean
                       timeout:
                         minimum: 0
                         type: integer
@@ -172,7 +187,7 @@ spec:
                           tcp_failures:
                             minimum: 0
                             type: integer
-                          timeout:
+                          timeouts:
                             minimum: 0
                             type: integer
                         type: object
@@ -196,6 +211,8 @@ spec:
                             minimum: 0
                             type: integer
                         type: object
+                      type:
+                        type: string
                       unhealthy:
                         description: Unhealthy configures thresholds and HTTP status
                           codes to mark targets unhealthy.
@@ -213,13 +230,13 @@ spec:
                           tcp_failures:
                             minimum: 0
                             type: integer
-                          timeout:
+                          timeouts:
                             minimum: 0
                             type: integer
                         type: object
                     type: object
                   threshold:
-                    type: integer
+                    type: number
                 type: object
               host_header:
                 type: string

diff --git a/config/crd/bases/configuration.konghq.com_kongplugins.yaml b/config/crd/bases/configuration.konghq.com_kongplugins.yaml
@@ -1,6 +1,11 @@
+
+---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
   name: kongplugins.configuration.konghq.com
 spec:
   group: configuration.konghq.com
@@ -11,7 +16,6 @@ spec:
     shortNames:
     - kp
     singular: kongplugin
-  preserveUnknownFields: false
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
@@ -38,6 +42,11 @@ spec:
       openAPIV3Schema:
         description: KongPlugin is the Schema for the kongplugins API
         properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
           config:
             description: Config contains the plugin configuration.
             type: object
@@ -60,9 +69,19 @@ spec:
                 - name
                 type: object
             type: object
+          consumerRef:
+            description: ConsumerRef is a reference to a particular consumer
+            type: string
           disabled:
             description: Disabled set if the plugin is disabled or not
             type: boolean
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
           plugin:
             description: PluginName is the name of the plugin to which to apply the
               config

diff --git a/config/crd/bases/configuration.konghq.com_tcpingresses.yaml b/config/crd/bases/configuration.konghq.com_tcpingresses.yaml
@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.6.2
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: tcpingresses.configuration.konghq.com
 spec:
@@ -14,7 +14,6 @@ spec:
     listKind: TCPIngressList
     plural: tcpingresses
     singular: tcpingress
-  preserveUnknownFields: false
   scope: Namespaced
   versions:
   - additionalPrinterColumns:

diff --git a/config/crd/bases/configuration.konghq.com_udpingresses.yaml b/config/crd/bases/configuration.konghq.com_udpingresses.yaml
@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.6.2
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: udpingresses.configuration.konghq.com
 spec:
@@ -14,7 +14,6 @@ spec:
     listKind: UDPIngressList
     plural: udpingresses
     singular: udpingress
-  preserveUnknownFields: false
   scope: Namespaced
   versions:
   - additionalPrinterColumns: