From 8829a6744006faba268232ed9e3db12c48befdc0 Mon Sep 17 00:00:00 2001 From: Erik Godding Boye Date: Sun, 30 Jun 2024 09:30:56 +0200 Subject: [PATCH] feat: configure excluded propagate labels/annotations --- charts/accurate/README.md | 32 +++++++++--------- charts/accurate/templates/configmap.yaml | 6 ++++ charts/accurate/values.yaml | 12 +++++++ cmd/accurate-controller/sub/run.go | 7 +++- controllers/namespace_controller.go | 7 ++-- controllers/propagate.go | 28 +++++++++------- controllers/propagate_test.go | 41 +++++++++++++++++++++++- pkg/config/types.go | 30 +++++++++++++---- 8 files changed, 125 insertions(+), 38 deletions(-) diff --git a/charts/accurate/README.md b/charts/accurate/README.md index 71b02f2..44a3e87 100644 --- a/charts/accurate/README.md +++ b/charts/accurate/README.md @@ -53,21 +53,23 @@ $ helm install --create-namespace --namespace accurate accurate -f values.yaml a ## Values -| Key | Type | Default | Description | -| ---------------------------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| controller.additionalRBAC.rules | list | `[]` | Specify the RBAC rules to be added to the controller. ClusterRole and ClusterRoleBinding are created with the names `{{ release name }}-additional-resources`. The rules defined here will be used for the ClusterRole rules. | -| controller.additionalRBAC.clusterRoles | list | `[]` | Specify additional ClusterRoles to be granted to the accurate controller. "admin" is recommended to allow the controller to manage common namespace-scoped resources. | -| controller.config.annotationKeys | list | `[]` | Annotations to be propagated to sub-namespaces. It is also possible to specify a glob pattern that can be interpreted by Go's "path.Match" func. | -| controller.config.labelKeys | list | `[]` | Labels to be propagated to sub-namespaces. It is also possible to specify a glob pattern that can be interpreted by Go's "path.Match" func. | -| controller.config.watches | list | `[{"group":"rbac.authorization.k8s.io","kind":"Role","version":"v1"},{"group":"rbac.authorization.k8s.io","kind":"RoleBinding","version":"v1"},{"kind":"Secret","version":"v1"}]` | List of GVK for namespace-scoped resources that can be propagated. Any namespace-scoped resource is allowed. | -| controller.extraArgs | list | `[]` | Optional additional arguments. | -| controller.replicas | int | `2` | Specify the number of replicas of the controller Pod. | -| controller.resources | object | `{"requests":{"cpu":"100m","memory":"20Mi"}}` | Specify resources. | -| controller.terminationGracePeriodSeconds | int | `10` | Specify terminationGracePeriodSeconds. | -| image.pullPolicy | string | `nil` | Accurate image pullPolicy. | -| image.repository | string | `"ghcr.io/cybozu-go/accurate"` | Accurate image repository to use. | -| image.tag | string | `{{ .Chart.AppVersion }}` | Accurate image tag to use. | -| installCRDs | bool | `true` | Controls if CRDs are automatically installed and managed as part of your Helm release. | +| Key | Type | Default | Description | +|--------------------------------------------------|--------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| controller.additionalRBAC.rules | list | `[]` | Specify the RBAC rules to be added to the controller. ClusterRole and ClusterRoleBinding are created with the names `{{ release name }}-additional-resources`. The rules defined here will be used for the ClusterRole rules. | +| controller.additionalRBAC.clusterRoles | list | `[]` | Specify additional ClusterRoles to be granted to the accurate controller. "admin" is recommended to allow the controller to manage common namespace-scoped resources. | +| controller.config.annotationKeys | list | `[]` | Annotations to be propagated to sub-namespaces. It is also possible to specify a glob pattern that can be interpreted by Go's "path.Match" func. | +| controller.config.labelKeys | list | `[]` | Labels to be propagated to sub-namespaces. It is also possible to specify a glob pattern that can be interpreted by Go's "path.Match" func. | +| controller.config.watches | list | `[{"group":"rbac.authorization.k8s.io","kind":"Role","version":"v1"},{"group":"rbac.authorization.k8s.io","kind":"RoleBinding","version":"v1"},{"kind":"Secret","version":"v1"}]` | List of GVK for namespace-scoped resources that can be propagated. Any namespace-scoped resource is allowed. | +| controller.config.propagateAnnotationKeyExcludes | list | `["*kubernetes.io/*"]` | Annotations to exclude when propagating resources. It is also possible to specify a glob pattern that can be interpreted by Go's "path.Match" func. | +| controller.config.propagateLabelKeyExcludes | list | `["*kubernetes.io/*"]` | Labels to exclude when propagating resources. It is also possible to specify a glob pattern that can be interpreted by Go's "path.Match" func. | +| controller.extraArgs | list | `[]` | Optional additional arguments. | +| controller.replicas | int | `2` | Specify the number of replicas of the controller Pod. | +| controller.resources | object | `{"requests":{"cpu":"100m","memory":"20Mi"}}` | Specify resources. | +| controller.terminationGracePeriodSeconds | int | `10` | Specify terminationGracePeriodSeconds. | +| image.pullPolicy | string | `nil` | Accurate image pullPolicy. | +| image.repository | string | `"ghcr.io/cybozu-go/accurate"` | Accurate image repository to use. | +| image.tag | string | `{{ .Chart.AppVersion }}` | Accurate image tag to use. | +| installCRDs | bool | `true` | Controls if CRDs are automatically installed and managed as part of your Helm release. | ## Generate Manifests diff --git a/charts/accurate/templates/configmap.yaml b/charts/accurate/templates/configmap.yaml index b438dad..152b53f 100644 --- a/charts/accurate/templates/configmap.yaml +++ b/charts/accurate/templates/configmap.yaml @@ -21,5 +21,11 @@ data: {{- end }} watches: {{ toYaml .Values.controller.config.watches | nindent 6 }} {{- with .Values.controller.config.namingPolicies }} + {{- with .Values.controller.config.propagateLabelKeyExcludes }} + propagateLabelKeyExcludes: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.controller.config.propagateAnnotationKeyExcludes }} + propagateAnnotationKeyExcludes: {{ toYaml . | nindent 6 }} + {{- end }} namingPolicies: {{ toYaml . | nindent 6 }} {{- end }} diff --git a/charts/accurate/values.yaml b/charts/accurate/values.yaml index 1429a24..6b8d4d4 100644 --- a/charts/accurate/values.yaml +++ b/charts/accurate/values.yaml @@ -68,6 +68,18 @@ controller: - version: v1 kind: ResourceQuota + # controller.config.propagateLabelKeyExcludes -- Labels to exclude when propagating resources. + # It is also possible to specify a glob pattern that can be interpreted by Go's "path.Match" func. + ## https://pkg.go.dev/path#Match + propagateLabelKeyExcludes: + - '*kubernetes.io/*' + + # controller.config.propagateAnnotationKeyExcludes -- Annotations to exclude when propagating resources. + # It is also possible to specify a glob pattern that can be interpreted by Go's "path.Match" func. + ## https://pkg.go.dev/path#Match + propagateAnnotationKeyExcludes: + - '*kubernetes.io/*' + # controller.config.namingPolicies -- List of nameing policy for SubNamespaces. # root and match are both regular expressions. # When a SubNamespace is created in a tree starting from a root namespace and the root namespace's name matches the "root" regular expression, the SubNamespace name is validated with the "match" regular expression. diff --git a/cmd/accurate-controller/sub/run.go b/cmd/accurate-controller/sub/run.go index 1f0d1d3..a8b58e4 100644 --- a/cmd/accurate-controller/sub/run.go +++ b/cmd/accurate-controller/sub/run.go @@ -109,6 +109,10 @@ func subMain(ns, addr string, port int) error { }) } + cloner := controllers.ResourceCloner{ + LabelKeyExcludes: cfg.PropagateLabelKeyExcludes, + AnnotationKeyExcludes: cfg.PropagateAnnotationKeyExcludes, + } dec := admission.NewDecoder(scheme) // Namespace reconciler & webhook @@ -117,6 +121,7 @@ func subMain(ns, addr string, port int) error { } if err := (&controllers.NamespaceReconciler{ Client: mgr.GetClient(), + ResourceCloner: cloner, LabelKeys: cfg.LabelKeys, AnnotationKeys: cfg.AnnotationKeys, SubNamespaceLabelKeys: cfg.SubNamespaceLabelKeys, @@ -142,7 +147,7 @@ func subMain(ns, addr string, port int) error { if err := indexing.SetupIndexForResource(ctx, mgr, res); err != nil { return fmt.Errorf("failed to setup indexer for %s: %w", res.GroupVersionKind().String(), err) } - if err := controllers.NewPropagateController(res).SetupWithManager(mgr); err != nil { + if err := controllers.NewPropagateController(res, cloner).SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create %s controller: %w", res.GroupVersionKind().String(), err) } logger.Info("watching", "gvk", res.GroupVersionKind().String()) diff --git a/controllers/namespace_controller.go b/controllers/namespace_controller.go index 7cbb17b..ceb3f98 100644 --- a/controllers/namespace_controller.go +++ b/controllers/namespace_controller.go @@ -26,6 +26,7 @@ import ( // NamespaceReconciler reconciles a Namespace object type NamespaceReconciler struct { client.Client + ResourceCloner LabelKeys []string AnnotationKeys []string SubNamespaceLabelKeys []string @@ -229,7 +230,7 @@ func (r *NamespaceReconciler) propagateCreate(ctx context.Context, res *unstruct return err } - if err := r.Create(ctx, cloneResource(res, ns)); err != nil { + if err := r.Create(ctx, r.cloneResource(res, ns)); err != nil { return utilerrors.Ignore(err, utilerrors.IsNamespaceTerminating) } @@ -249,14 +250,14 @@ func (r *NamespaceReconciler) propagateUpdate(ctx context.Context, res *unstruct if !apierrors.IsNotFound(err) { return err } - if err := r.Create(ctx, cloneResource(res, ns)); err != nil { + if err := r.Create(ctx, r.cloneResource(res, ns)); err != nil { return utilerrors.Ignore(err, utilerrors.IsNamespaceTerminating) } logger.Info("created a resource", "namespace", ns, "name", res.GetName(), "gvk", gvk.String()) return nil } - c2 := cloneResource(res, ns) + c2 := r.cloneResource(res, ns) // Ensure that managed fields are upgraded to SSA before the following SSA. // TODO(migration): This code could be removed after a couple of releases. diff --git a/controllers/propagate.go b/controllers/propagate.go index 41877ee..2ee27b3 100644 --- a/controllers/propagate.go +++ b/controllers/propagate.go @@ -3,7 +3,6 @@ package controllers import ( "context" "fmt" - "strings" utilerrors "github.com/cybozu-go/accurate/internal/util/errors" "github.com/cybozu-go/accurate/pkg/config" @@ -26,7 +25,12 @@ import ( // removal soon. const notGenerated = "false" -func cloneResource(res *unstructured.Unstructured, ns string) *unstructured.Unstructured { +type ResourceCloner struct { + LabelKeyExcludes []string + AnnotationKeyExcludes []string +} + +func (rc *ResourceCloner) cloneResource(res *unstructured.Unstructured, ns string) *unstructured.Unstructured { c := res.DeepCopy() delete(c.Object, "metadata") delete(c.Object, "status") @@ -34,7 +38,7 @@ func cloneResource(res *unstructured.Unstructured, ns string) *unstructured.Unst c.SetName(res.GetName()) labels := make(map[string]string) for k, v := range res.GetLabels() { - if strings.Contains(k, "kubernetes.io/") { + if matchKey(k, rc.LabelKeyExcludes) { continue } labels[k] = v @@ -43,7 +47,7 @@ func cloneResource(res *unstructured.Unstructured, ns string) *unstructured.Unst c.SetLabels(labels) annotations := make(map[string]string) for k, v := range res.GetAnnotations() { - if strings.Contains(k, "kubernetes.io/") { + if matchKey(k, rc.AnnotationKeyExcludes) { continue } annotations[k] = v @@ -62,18 +66,20 @@ func cloneResource(res *unstructured.Unstructured, ns string) *unstructured.Unst // PropagateController propagates objects of a namespace-scoped resource. type PropagateController struct { client.Client + ResourceCloner reader client.Reader res *unstructured.Unstructured } // NewPropagateController creates a new PropagateController. // The GroupVersionKind of `res` must be set. -func NewPropagateController(res *unstructured.Unstructured) *PropagateController { +func NewPropagateController(res *unstructured.Unstructured, cloner ResourceCloner) *PropagateController { if res.GetKind() == "" { panic("no group version kind") } return &PropagateController{ - res: res.DeepCopy(), + res: res.DeepCopy(), + ResourceCloner: cloner, } } @@ -200,7 +206,7 @@ func (r *PropagateController) handleDelete(ctx context.Context, req ctrl.Request } else { switch obj.GetAnnotations()[constants.AnnPropagate] { case constants.PropagateCreate, constants.PropagateUpdate: - if err := r.Create(ctx, cloneResource(obj, req.Namespace)); err != nil { + if err := r.Create(ctx, r.cloneResource(obj, req.Namespace)); err != nil { if utilerrors.IsNamespaceTerminating(err) { return nil } @@ -257,7 +263,7 @@ func (r *PropagateController) propagateCreate(ctx context.Context, obj *unstruct return fmt.Errorf("failed to look up %s/%s: %w", child.Name, name, err) } - if err := r.Create(ctx, cloneResource(obj, child.Name)); err != nil { + if err := r.Create(ctx, r.cloneResource(obj, child.Name)); err != nil { if utilerrors.IsNamespaceTerminating(err) { return nil } @@ -275,7 +281,7 @@ func (r *PropagateController) propagateUpdate(ctx context.Context, obj, parent * name := obj.GetName() if parent != nil { - clone := cloneResource(parent, obj.GetNamespace()) + clone := r.cloneResource(parent, obj.GetNamespace()) // Ensure that managed fields are upgraded to SSA before the following SSA. // TODO(migration): This code could be removed after a couple of releases. @@ -306,7 +312,7 @@ func (r *PropagateController) propagateUpdate(ctx context.Context, obj, parent * return fmt.Errorf("failed to lookup %s/%s: %w", child.Name, name, err) } - clone := cloneResource(obj, child.Name) + clone := r.cloneResource(obj, child.Name) if err := r.Create(ctx, clone); err != nil { if utilerrors.IsNamespaceTerminating(err) { return nil @@ -318,7 +324,7 @@ func (r *PropagateController) propagateUpdate(ctx context.Context, obj, parent * continue } - clone := cloneResource(obj, child.Name) + clone := r.cloneResource(obj, child.Name) // Ensure that managed fields are upgraded to SSA before the following SSA. // TODO(migration): This code could be removed after a couple of releases. diff --git a/controllers/propagate_test.go b/controllers/propagate_test.go index 8d0df81..a3801d3 100644 --- a/controllers/propagate_test.go +++ b/controllers/propagate_test.go @@ -62,7 +62,11 @@ var _ = Describe("SubNamespace controller", func() { err = indexing.SetupIndexForResource(ctx, mgr, svcRes) Expect(err).NotTo(HaveOccurred()) - pc := NewPropagateController(svcRes) + cloner := ResourceCloner{ + AnnotationKeyExcludes: []string{"*excluded-annotation.io/*"}, + LabelKeyExcludes: []string{"*excluded-label.io/*"}, + } + pc := NewPropagateController(svcRes, cloner) err = pc.SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred()) @@ -289,6 +293,41 @@ var _ = Describe("SubNamespace controller", func() { } }) + DescribeTable("should NOT propagate excluded resource labels/annotations", + func(sourceNs, targetNs string) { + const excludedAnnotation = "excluded-annotation.io/foo" + const excludedLabel = "excluded-label.io/foo" + + svc := &corev1.Service{} + svc.Namespace = sourceNs + svc.Name = "svc" + svc.Annotations = map[string]string{ + constants.AnnPropagate: constants.PropagateUpdate, + excludedAnnotation: "bar", + "foo.bar/baz": "baz", + } + svc.Labels = map[string]string{ + excludedLabel: "bar", + "foo.bar/baz": "baz", + } + svc.Spec.ClusterIP = "None" + svc.Spec.Ports = []corev1.ServicePort{{Port: 3333, TargetPort: intstr.FromInt32(3333)}} + Expect(k8sClient.Create(ctx, svc)).To(Succeed()) + + clone := &corev1.Service{} + clone.Name = "svc" + clone.Namespace = targetNs + Eventually(komega.Get(clone)).Should(Succeed()) + + Expect(clone.Annotations).To(Not(HaveKey(excludedAnnotation))) + Expect(clone.Annotations).To(HaveKeyWithValue("foo.bar/baz", "baz")) + Expect(clone.Labels).To(Not(HaveKey(excludedLabel))) + Expect(clone.Labels).To(HaveKeyWithValue("foo.bar/baz", "baz")) + }, + Entry("to sub-namespaces", rootNS, sub1NS), + Entry("from a template namespace", tmplNS, instanceNS), + ) + It("should manage generated resources", func() { cm1 := &corev1.ConfigMap{} cm1.Namespace = rootNS diff --git a/pkg/config/types.go b/pkg/config/types.go index 8c06358..e955824 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -31,13 +31,15 @@ type NamingPolicyRegexp struct { // Config represents the configuration file of Accurate. type Config struct { - LabelKeys []string `json:"labelKeys,omitempty"` - AnnotationKeys []string `json:"annotationKeys,omitempty"` - SubNamespaceLabelKeys []string `json:"subNamespaceLabelKeys,omitempty"` - SubNamespaceAnnotationKeys []string `json:"subNamespaceAnnotationKeys,omitempty"` - Watches []metav1.GroupVersionKind `json:"watches,omitempty"` - NamingPolicies []NamingPolicy `json:"namingPolicies,omitempty"` - NamingPolicyRegexps []NamingPolicyRegexp + LabelKeys []string `json:"labelKeys,omitempty"` + AnnotationKeys []string `json:"annotationKeys,omitempty"` + SubNamespaceLabelKeys []string `json:"subNamespaceLabelKeys,omitempty"` + SubNamespaceAnnotationKeys []string `json:"subNamespaceAnnotationKeys,omitempty"` + Watches []metav1.GroupVersionKind `json:"watches,omitempty"` + PropagateLabelKeyExcludes []string `json:"propagateLabelKeyExcludes,omitempty"` + PropagateAnnotationKeyExcludes []string `json:"propagateAnnotationKeyExcludes,omitempty"` + NamingPolicies []NamingPolicy `json:"namingPolicies,omitempty"` + NamingPolicyRegexps []NamingPolicyRegexp } // Validate validates the configurations. @@ -93,6 +95,20 @@ func (c *Config) Validate(mapper meta.RESTMapper) error { } } + for _, key := range c.PropagateLabelKeyExcludes { + // Verify that pattern is a valid format. + if _, err := path.Match(key, ""); err != nil { + return fmt.Errorf("malformed pattern for propagateLabelKeyExcludes %s: %w", key, err) + } + } + + for _, key := range c.PropagateAnnotationKeyExcludes { + // Verify that pattern is a valid format. + if _, err := path.Match(key, ""); err != nil { + return fmt.Errorf("malformed pattern for propagateAnnotationKeyExcludes %s: %w", key, err) + } + } + for _, policy := range c.NamingPolicies { root, err := regexp.Compile(policy.Root) if err != nil {