From c07dd45b6048f634845d18ff31a3039a78065e25 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 15 Sep 2024 19:31:56 +0300 Subject: [PATCH 1/6] Add sharding to the FluxInstance API Signed-off-by: Stefan Prodan --- api/v1/fluxinstance_types.go | 20 ++++++++++++++- api/v1/zz_generated.deepcopy.go | 25 +++++++++++++++++++ .../fluxcd.controlplane.io_fluxinstances.yaml | 18 +++++++++++-- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/api/v1/fluxinstance_types.go b/api/v1/fluxinstance_types.go index 8ad9d91..ff8001e 100644 --- a/api/v1/fluxinstance_types.go +++ b/api/v1/fluxinstance_types.go @@ -47,6 +47,10 @@ type FluxInstanceSpec struct { // +optional Cluster *Cluster `json:"cluster,omitempty"` + // Sharding holds the specification of the sharding configuration. + // +optional + Sharding *Sharding `json:"sharding,omitempty"` + // Storage holds the specification of the source-controller // persistent volume claim. // +optional @@ -60,12 +64,14 @@ type FluxInstanceSpec struct { // Wait instructs the controller to check the health of all the reconciled // resources. Defaults to true. // +kubebuilder:default:=true + // +optional Wait bool `json:"wait"` // MigrateResources instructs the controller to migrate the Flux custom resources // from the previous version to the latest API version specified in the CRD. // Defaults to true. // +kubebuilder:default:=true + // +optional MigrateResources bool `json:"migrateResources"` // Sync specifies the source for the cluster sync operation. @@ -73,7 +79,7 @@ type FluxInstanceSpec struct { // and Flux Kustomization are created to sync the cluster state // with the source repository. // +optional - Sync *Sync `json:"sync,omitempty"` + Sync *Sync `json:"sync"` } // Distribution specifies the version and container registry to pull images from. @@ -155,6 +161,18 @@ type Cluster struct { Type string `json:"type,omitempty"` } +type Sharding struct { + // Key is the label key used to shard the resources. + // +kubebuilder:default:=sharding.fluxcd.io/key + // +optional + Key string `json:"key,omitempty"` + + // Shards is the list of shard names. + // +kubebuilder:validation:MinItems=1 + // +required + Shards []string `json:"shards"` +} + // Storage is the specification for the persistent volume claim. type Storage struct { // Class is the storage class to use for the PVC. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 3a382e8..71e5bfb 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -161,6 +161,11 @@ func (in *FluxInstanceSpec) DeepCopyInto(out *FluxInstanceSpec) { *out = new(Cluster) **out = **in } + if in.Sharding != nil { + in, out := &in.Sharding, &out.Sharding + *out = new(Sharding) + (*in).DeepCopyInto(*out) + } if in.Storage != nil { in, out := &in.Storage, &out.Storage *out = new(Storage) @@ -437,6 +442,26 @@ func (in *ResourceRef) DeepCopy() *ResourceRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Sharding) DeepCopyInto(out *Sharding) { + *out = *in + if in.Shards != nil { + in, out := &in.Shards, &out.Shards + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Sharding. +func (in *Sharding) DeepCopy() *Sharding { + if in == nil { + return nil + } + out := new(Sharding) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Storage) DeepCopyInto(out *Storage) { *out = *in diff --git a/config/crd/bases/fluxcd.controlplane.io_fluxinstances.yaml b/config/crd/bases/fluxcd.controlplane.io_fluxinstances.yaml index 2af2f28..85d5412 100644 --- a/config/crd/bases/fluxcd.controlplane.io_fluxinstances.yaml +++ b/config/crd/bases/fluxcd.controlplane.io_fluxinstances.yaml @@ -208,6 +208,22 @@ spec: from the previous version to the latest API version specified in the CRD. Defaults to true. type: boolean + sharding: + description: Sharding holds the specification of the sharding configuration. + properties: + key: + default: sharding.fluxcd.io/key + description: Key is the label key used to shard the resources. + type: string + shards: + description: Shards is the list of shard names. + items: + type: string + minItems: 1 + type: array + required: + - shards + type: object storage: description: |- Storage holds the specification of the source-controller @@ -280,8 +296,6 @@ spec: type: boolean required: - distribution - - migrateResources - - wait type: object status: description: FluxInstanceStatus defines the observed state of FluxInstance From 8fb72a794eb0ef324c561926396d1b943a5053c7 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 15 Sep 2024 19:32:39 +0300 Subject: [PATCH 2/6] Build controller shards Signed-off-by: Stefan Prodan --- .gitignore | 1 + internal/builder/build.go | 11 + internal/builder/build_test.go | 72 ++++++ internal/builder/options.go | 4 + internal/builder/templates.go | 95 ++++++++ .../v2.3.0-golden/sharding.kustomization.yaml | 205 ++++++++++++++++++ .../v2.3.0-golden/storage.kustomization.yaml | 7 + 7 files changed, 395 insertions(+) create mode 100644 internal/builder/testdata/v2.3.0-golden/sharding.kustomization.yaml diff --git a/.gitignore b/.gitignore index bc05b88..6b0d528 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ disto/ *.tfstate.* .terraform.lock.hcl .terraformrc +/internal/builder/testdata/output/ diff --git a/internal/builder/build.go b/internal/builder/build.go index ab5f016..6ac0add 100644 --- a/internal/builder/build.go +++ b/internal/builder/build.go @@ -115,5 +115,16 @@ func generate(base string, options Options) error { return fmt.Errorf("replacing service account namespace in rbac failed: %w", err) } } + + for _, shard := range options.Shards { + options.ShardName = shard + if err := os.MkdirAll(path.Join(base, shard), os.ModePerm); err != nil { + return fmt.Errorf("generate shard dir failed: %w", err) + } + if err := execTemplate(options, kustomizationShardTmpl, path.Join(base, shard, "kustomization.yaml")); err != nil { + return fmt.Errorf("generate shard kustomization failed: %w", err) + } + } + return nil } diff --git a/internal/builder/build_test.go b/internal/builder/build_test.go index 9621f54..222024d 100644 --- a/internal/builder/build_test.go +++ b/internal/builder/build_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "github.com/fluxcd/pkg/apis/kustomize" @@ -15,6 +16,28 @@ import ( "sigs.k8s.io/yaml" ) +func TestBuild(t *testing.T) { + g := NewWithT(t) + const version = "v2.3.0" + options := MakeDefaultOptions() + options.Version = version + options.Shards = []string{"shard1", "shard2"} + options.Patches = ProfileOpenShift + GetMultitenantProfile("") + options.ArtifactStorage = &ArtifactStorage{ + Class: "standard", + Size: "10Gi", + } + + srcDir := filepath.Join("testdata", version) + dstDir := filepath.Join("testdata", "output") + err := os.RemoveAll(dstDir) + g.Expect(err).NotTo(HaveOccurred()) + + result, err := Build(srcDir, dstDir, options) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result.Objects).NotTo(BeEmpty()) +} + func TestBuild_Defaults(t *testing.T) { g := NewWithT(t) const version = "v2.3.0" @@ -299,6 +322,55 @@ func TestBuild_InvalidPatches(t *testing.T) { g.Expect(err.Error()).To(ContainSubstring("Unexpected kind: removes")) } +func TestBuild_Sharding(t *testing.T) { + g := NewWithT(t) + const version = "v2.3.0" + options := MakeDefaultOptions() + options.Version = version + options.Shards = []string{"shard1", "shard2"} + options.Patches = ProfileOpenShift + GetMultitenantProfile("") + options.ArtifactStorage = &ArtifactStorage{ + Class: "standard", + Size: "10Gi", + } + + srcDir := filepath.Join("testdata", version) + goldenFile := filepath.Join("testdata", version+"-golden", "sharding.kustomization.yaml") + + dstDir, err := testTempDir(t) + g.Expect(err).NotTo(HaveOccurred()) + + ci, err := ExtractComponentImages(srcDir, options) + g.Expect(err).NotTo(HaveOccurred()) + options.ComponentImages = ci + + result, err := Build(srcDir, dstDir, options) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result.Objects).NotTo(BeEmpty()) + + if shouldGenGolden() { + err = cp.Copy(filepath.Join(dstDir, "kustomization.yaml"), goldenFile) + g.Expect(err).NotTo(HaveOccurred()) + } + + genK, err := os.ReadFile(filepath.Join(dstDir, "kustomization.yaml")) + g.Expect(err).NotTo(HaveOccurred()) + + goldenK, err := os.ReadFile(goldenFile) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(string(genK)).To(Equal(string(goldenK))) + + found := false + for _, obj := range result.Objects { + if strings.Contains(obj.GetName(), options.Shards[0]) { + found = true + g.Expect(obj.GetAnnotations()).To(HaveKeyWithValue("sharding.fluxcd.io/role", "shard")) + } + } + g.Expect(found).To(BeTrue()) +} + func testTempDir(t *testing.T) (string, error) { tmpDir := t.TempDir() diff --git a/internal/builder/options.go b/internal/builder/options.go index b56cb5e..b75a724 100644 --- a/internal/builder/options.go +++ b/internal/builder/options.go @@ -21,6 +21,9 @@ type Options struct { Patches string ArtifactStorage *ArtifactStorage Sync *Sync + ShardingKey string + Shards []string + ShardName string } // MakeDefaultOptions returns the default builder configuration. @@ -44,6 +47,7 @@ func MakeDefaultOptions() Options { LogLevel: "info", NotificationController: "notification-controller", ClusterDomain: "cluster.local", + ShardingKey: "sharding.fluxcd.io/key", } } diff --git a/internal/builder/templates.go b/internal/builder/templates.go index e36609f..0c531a8 100644 --- a/internal/builder/templates.go +++ b/internal/builder/templates.go @@ -35,6 +35,9 @@ resources: {{- range .Components }} - {{.}}.yaml {{- end }} +{{- range .Shards }} + - {{.}} +{{- end }} {{- if $artifactStorage }} - pvc.yaml {{- end }} @@ -90,6 +93,13 @@ patches: path: /spec/template/spec/containers/0/args/6 value: --storage-adv-addr=source-controller.$(RUNTIME_NAMESPACE).svc.{{$clusterDomain}}. {{- if $artifactStorage }} +- target: + group: apps + version: v1 + kind: Deployment + name: {{$component}} + annotationSelector: "!sharding.fluxcd.io/role" + patch: |- - op: add path: '/spec/template/spec/volumes/-' value: @@ -120,9 +130,94 @@ patches: value: --log-level={{$logLevel}} {{- end }} {{- end }} +{{- if gt (len .Shards) 0 }} +- target: + kind: Deployment + name: "(source-controller|kustomize-controller|helm-controller)" + annotationSelector: "!sharding.fluxcd.io/role" + patch: | + - op: add + path: /spec/template/spec/containers/0/args/- + value: --watch-label-selector=!{{.ShardingKey}} +{{- end }} {{ .Patches }} ` +var kustomizationShardTmpl = `--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: {{.Namespace}} +resources: +{{- range $i, $component := .Components }} +{{- if eq $component "source-controller" }} + - ../{{$component}}.yaml +{{- else if eq $component "kustomize-controller" }} + - ../{{$component}}.yaml +{{- else if eq $component "helm-controller" }} + - ../{{$component}}.yaml +{{- end }} +{{- end }} +nameSuffix: "-{{.ShardName}}" +commonAnnotations: + sharding.fluxcd.io/role: "shard" +patches: + - target: + kind: (Namespace|CustomResourceDefinition|ClusterRole|ClusterRoleBinding|ServiceAccount|NetworkPolicy|ResourceQuota) + patch: | + apiVersion: v1 + kind: all + metadata: + name: all + $patch: delete + - target: + kind: Service + name: (source-controller) + patch: | + - op: replace + path: /spec/selector/app + value: source-controller-{{.ShardName}} + - target: + kind: Deployment + name: (source-controller) + patch: | + - op: replace + path: /spec/selector/matchLabels/app + value: source-controller-{{.ShardName}} + - op: replace + path: /spec/template/metadata/labels/app + value: source-controller-{{.ShardName}} + - op: add + path: /spec/template/spec/containers/0/args/- + value: --storage-adv-addr=source-controller-{{.ShardName}}.$(RUNTIME_NAMESPACE).svc.{{.ClusterDomain}}. + - target: + kind: Deployment + name: (kustomize-controller) + patch: | + - op: replace + path: /spec/selector/matchLabels/app + value: kustomize-controller-{{.ShardName}} + - op: replace + path: /spec/template/metadata/labels/app + value: kustomize-controller-{{.ShardName}} + - target: + kind: Deployment + name: (helm-controller) + patch: | + - op: replace + path: /spec/selector/matchLabels/app + value: helm-controller-{{.ShardName}} + - op: replace + path: /spec/template/metadata/labels/app + value: helm-controller-{{.ShardName}} + - target: + kind: Deployment + name: (source-controller|kustomize-controller|helm-controller) + patch: | + - op: add + path: /spec/template/spec/containers/0/args/- + value: --watch-label-selector={{.ShardingKey}}={{.ShardName}} +` + var kustomizationRolesTmpl = `--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/internal/builder/testdata/v2.3.0-golden/sharding.kustomization.yaml b/internal/builder/testdata/v2.3.0-golden/sharding.kustomization.yaml new file mode 100644 index 0000000..d0c07c2 --- /dev/null +++ b/internal/builder/testdata/v2.3.0-golden/sharding.kustomization.yaml @@ -0,0 +1,205 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: flux-system +transformers: + - annotations.yaml + - labels.yaml +resources: + - namespace.yaml + - policies.yaml + - roles + - source-controller.yaml + - kustomize-controller.yaml + - helm-controller.yaml + - notification-controller.yaml + - image-reflector-controller.yaml + - image-automation-controller.yaml + - shard1 + - shard2 + - pvc.yaml +images: + - name: fluxcd/source-controller + newName: ghcr.io/fluxcd/source-controller + newTag: v1.3.0 + - name: fluxcd/kustomize-controller + newName: ghcr.io/fluxcd/kustomize-controller + newTag: v1.3.0 + - name: fluxcd/helm-controller + newName: ghcr.io/fluxcd/helm-controller + newTag: v1.0.1 + - name: fluxcd/notification-controller + newName: ghcr.io/fluxcd/notification-controller + newTag: v1.3.0 + - name: fluxcd/image-reflector-controller + newName: ghcr.io/fluxcd/image-reflector-controller + newTag: v0.32.0 + - name: fluxcd/image-automation-controller + newName: ghcr.io/fluxcd/image-automation-controller + newTag: v0.38.0 +patches: +- path: node-selector.yaml + target: + kind: Deployment +- target: + group: apps + version: v1 + kind: Deployment + name: source-controller + patch: |- + - op: replace + path: /spec/template/spec/containers/0/args/0 + value: --events-addr=http://notification-controller.flux-system.svc.cluster.local./ + - op: replace + path: /spec/template/spec/containers/0/args/1 + value: --watch-all-namespaces=true + - op: replace + path: /spec/template/spec/containers/0/args/2 + value: --log-level=info + - op: replace + path: /spec/template/spec/containers/0/args/6 + value: --storage-adv-addr=source-controller.$(RUNTIME_NAMESPACE).svc.cluster.local. +- target: + group: apps + version: v1 + kind: Deployment + name: source-controller + annotationSelector: "!sharding.fluxcd.io/role" + patch: |- + - op: add + path: '/spec/template/spec/volumes/-' + value: + name: persistent-data + persistentVolumeClaim: + claimName: source-controller + - op: replace + path: '/spec/template/spec/containers/0/volumeMounts/0' + value: + name: persistent-data + mountPath: /data +- target: + group: apps + version: v1 + kind: Deployment + name: kustomize-controller + patch: |- + - op: replace + path: /spec/template/spec/containers/0/args/0 + value: --events-addr=http://notification-controller.flux-system.svc.cluster.local./ + - op: replace + path: /spec/template/spec/containers/0/args/1 + value: --watch-all-namespaces=true + - op: replace + path: /spec/template/spec/containers/0/args/2 + value: --log-level=info +- target: + group: apps + version: v1 + kind: Deployment + name: helm-controller + patch: |- + - op: replace + path: /spec/template/spec/containers/0/args/0 + value: --events-addr=http://notification-controller.flux-system.svc.cluster.local./ + - op: replace + path: /spec/template/spec/containers/0/args/1 + value: --watch-all-namespaces=true + - op: replace + path: /spec/template/spec/containers/0/args/2 + value: --log-level=info +- target: + group: apps + version: v1 + kind: Deployment + name: notification-controller + patch: |- + - op: replace + path: /spec/template/spec/containers/0/args/0 + value: --watch-all-namespaces=true + - op: replace + path: /spec/template/spec/containers/0/args/1 + value: --log-level=info +- target: + group: apps + version: v1 + kind: Deployment + name: image-reflector-controller + patch: |- + - op: replace + path: /spec/template/spec/containers/0/args/0 + value: --events-addr=http://notification-controller.flux-system.svc.cluster.local./ + - op: replace + path: /spec/template/spec/containers/0/args/1 + value: --watch-all-namespaces=true + - op: replace + path: /spec/template/spec/containers/0/args/2 + value: --log-level=info +- target: + group: apps + version: v1 + kind: Deployment + name: image-automation-controller + patch: |- + - op: replace + path: /spec/template/spec/containers/0/args/0 + value: --events-addr=http://notification-controller.flux-system.svc.cluster.local./ + - op: replace + path: /spec/template/spec/containers/0/args/1 + value: --watch-all-namespaces=true + - op: replace + path: /spec/template/spec/containers/0/args/2 + value: --log-level=info +- target: + kind: Deployment + name: "(source-controller|kustomize-controller|helm-controller)" + annotationSelector: "!sharding.fluxcd.io/role" + patch: | + - op: add + path: /spec/template/spec/containers/0/args/- + value: --watch-label-selector=!sharding.fluxcd.io/key + +- target: + kind: Deployment + patch: |- + - op: remove + path: /spec/template/spec/securityContext + - op: remove + path: /spec/template/spec/containers/0/securityContext/seccompProfile + - op: remove + path: /spec/template/spec/containers/0/securityContext/runAsNonRoot +- target: + kind: Namespace + patch: |- + - op: remove + path: /metadata/labels/pod-security.kubernetes.io~1warn + - op: remove + path: /metadata/labels/pod-security.kubernetes.io~1warn-version + +- target: + kind: Deployment + name: "(kustomize-controller|helm-controller|notification-controller|image-reflector-controller|image-automation-controller)" + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: --no-cross-namespace-refs=true +- target: + kind: Deployment + name: "(kustomize-controller)" + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: --no-remote-bases=true +- target: + kind: Deployment + name: "(kustomize-controller|helm-controller)" + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: --default-service-account=default +- target: + kind: Kustomization + patch: |- + - op: add + path: /spec/serviceAccountName + value: kustomize-controller + diff --git a/internal/builder/testdata/v2.3.0-golden/storage.kustomization.yaml b/internal/builder/testdata/v2.3.0-golden/storage.kustomization.yaml index e52f206..85e4fe2 100644 --- a/internal/builder/testdata/v2.3.0-golden/storage.kustomization.yaml +++ b/internal/builder/testdata/v2.3.0-golden/storage.kustomization.yaml @@ -57,6 +57,13 @@ patches: - op: replace path: /spec/template/spec/containers/0/args/6 value: --storage-adv-addr=source-controller.$(RUNTIME_NAMESPACE).svc.cluster.local. +- target: + group: apps + version: v1 + kind: Deployment + name: source-controller + annotationSelector: "!sharding.fluxcd.io/role" + patch: |- - op: add path: '/spec/template/spec/volumes/-' value: From 394faa73c3042311b80c1cb5c2fe7ef46b81b823 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 15 Sep 2024 19:33:02 +0300 Subject: [PATCH 3/6] Implement sharding reconciliation Signed-off-by: Stefan Prodan --- internal/controller/fluxinstance_controller.go | 10 +++++++++- .../controller/fluxinstance_controller_test.go | 17 ++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/internal/controller/fluxinstance_controller.go b/internal/controller/fluxinstance_controller.go index b521c0d..94b90bb 100644 --- a/internal/controller/fluxinstance_controller.go +++ b/internal/controller/fluxinstance_controller.go @@ -292,9 +292,12 @@ func (r *FluxInstanceReconciler) build(ctx context.Context, options.ImagePullSecret = obj.GetDistribution().ImagePullSecret options.Namespace = obj.GetNamespace() options.Components = obj.GetComponents() - options.ClusterDomain = obj.GetCluster().Domain options.NetworkPolicy = obj.GetCluster().NetworkPolicy + if obj.GetCluster().Domain != "" { + options.ClusterDomain = obj.GetCluster().Domain + } + if obj.GetCluster().Type == "openshift" { options.Patches += builder.ProfileOpenShift } @@ -302,6 +305,11 @@ func (r *FluxInstanceReconciler) build(ctx context.Context, options.Patches += builder.GetMultitenantProfile(obj.GetCluster().TenantDefaultServiceAccount) } + if obj.Spec.Sharding != nil { + options.ShardingKey = obj.Spec.Sharding.Key + options.Shards = obj.Spec.Sharding.Shards + } + if obj.Spec.Storage != nil { options.ArtifactStorage = &builder.ArtifactStorage{ Class: obj.Spec.Storage.Class, diff --git a/internal/controller/fluxinstance_controller_test.go b/internal/controller/fluxinstance_controller_test.go index 550a349..dbcbd9f 100644 --- a/internal/controller/fluxinstance_controller_test.go +++ b/internal/controller/fluxinstance_controller_test.go @@ -521,6 +521,9 @@ func TestFluxInstanceReconciler_Profiles(t *testing.T) { Type: "openshift", Multitenant: true, } + spec.Sharding = &fluxcdv1.Sharding{ + Shards: []string{"shard1", "shard2"}, + } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -581,6 +584,17 @@ func TestFluxInstanceReconciler_Profiles(t *testing.T) { // Check custom patches. g.Expect(*kc.Spec.Replicas).To(BeNumerically("==", 0)) + // Check if the shards were installed. + for _, shard := range spec.Sharding.Shards { + sc := &appsv1.Deployment{} + err = testClient.Get(ctx, types.NamespacedName{Name: "source-controller-" + shard, Namespace: ns.Name}, sc) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(sc.Spec.Template.Spec.Containers[0].Args).To(ContainElements( + fmt.Sprintf("--watch-label-selector=sharding.fluxcd.io/key=%s", shard), + fmt.Sprintf("--storage-adv-addr=source-controller-%s.$(RUNTIME_NAMESPACE).svc.cluster.local.", shard), + )) + } + // Uninstall the instance. err = testClient.Delete(ctx, obj) g.Expect(err).ToNot(HaveOccurred()) @@ -652,7 +666,8 @@ func TestFluxInstanceReconciler_NewVersion(t *testing.T) { func getDefaultFluxSpec() fluxcdv1.FluxInstanceSpec { return fluxcdv1.FluxInstanceSpec{ - Wait: false, + Wait: false, + MigrateResources: true, Distribution: fluxcdv1.Distribution{ Version: "v2.3.0", Registry: "ghcr.io/fluxcd", From 781ff6e2031163b9f467ab248f39abdcdc681735 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Sun, 15 Sep 2024 19:33:17 +0300 Subject: [PATCH 4/6] Add sharding to API docs Signed-off-by: Stefan Prodan --- README.md | 6 +-- api/v1/fluxinstance_types.go | 2 +- .../flux-operator.clusterserviceversion.yaml | 10 ++-- docs/api/v1/fluxinstance.md | 46 +++++++++++++++++++ 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 171d1d9..f5b609c 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ removes the operational burden of managing Flux across fleets of clusters by ful installation, configuration, and upgrade of the Flux controllers based on a declarative API. **Advanced Configuration** - The operator simplifies the configuration of Flux multi-tenancy lockdown, -vertical scaling, persistent storage, and allows fine-tuning the Flux controllers with Kustomize patches. -The operator streamlines the transition from Git as the delivery mechanism for the cluster -desired state to OCI artifacts and S3-compatible storage. +sharding, horizontal and vertical scaling, persistent storage, and allows fine-tuning the Flux +controllers with Kustomize patches. The operator streamlines the transition from Git as the delivery +mechanism for the cluster desired state to OCI artifacts and S3-compatible storage. **Deep Insights** - The operator provides deep insights into the delivery pipelines managed by Flux, including detailed reports and Prometheus metrics about the Flux controllers diff --git a/api/v1/fluxinstance_types.go b/api/v1/fluxinstance_types.go index ff8001e..8444d6d 100644 --- a/api/v1/fluxinstance_types.go +++ b/api/v1/fluxinstance_types.go @@ -79,7 +79,7 @@ type FluxInstanceSpec struct { // and Flux Kustomization are created to sync the cluster state // with the source repository. // +optional - Sync *Sync `json:"sync"` + Sync *Sync `json:"sync,omitempty"` } // Distribution specifies the version and container registry to pull images from. diff --git a/config/olm/bundle/manifests/flux-operator.clusterserviceversion.yaml b/config/olm/bundle/manifests/flux-operator.clusterserviceversion.yaml index 5c8b3c6..b5acbe1 100644 --- a/config/olm/bundle/manifests/flux-operator.clusterserviceversion.yaml +++ b/config/olm/bundle/manifests/flux-operator.clusterserviceversion.yaml @@ -111,8 +111,7 @@ metadata: spec: displayName: Flux Operator description: >- - [Flux](https://fluxcd.io) is a leading GitOps Continuous Delivery - tool + [Flux](https://fluxcd.io) is a leading GitOps Continuous Delivery tool used to streamline and automate application deployments on Kubernetes. Flux enables teams to achieve a reliable and auditable delivery process while promoting collaboration and traceability across different environments. @@ -125,17 +124,15 @@ spec: For more information about the CNCF-graduated Flux project, please see the [Flux architecture overview](https://fluxcd.control-plane.io/guides/flux-architecture/). - ### Operator Capabilities - The [Flux Operator](https://github.com/controlplaneio-fluxcd/flux-operator) provides a declarative API for the installation, configuration and upgrade of CNCF Flux and the ControlPlane [enterprise distribution](https://fluxcd.control-plane.io/). The operator allows the configuration of Flux multi-tenancy lockdown, network policies, - persistent storage, vertical scaling, custom patches, and the synchronization of the - cluster state from Git repositories, OCI artifacts and S3-compatible storage. + persistent storage, sharding, vertical scaling, custom patches, and the synchronization + of the cluster state from Git repositories, OCI artifacts and S3-compatible storage. The Flux Operator supervises the Flux controllers and provides a unified view of all the Flux resources that define the GitOps workflows for the target cluster. @@ -144,7 +141,6 @@ spec: ### OpenShift Support - The Flux Operator should be installed in a dedicated namespace, e.g. `flux-system`. To deploy Flux on OpenShift clusters, create a `FluxInstance` custom resource diff --git a/docs/api/v1/fluxinstance.md b/docs/api/v1/fluxinstance.md index 7e61785..05815ac 100644 --- a/docs/api/v1/fluxinstance.md +++ b/docs/api/v1/fluxinstance.md @@ -330,6 +330,52 @@ spec: The `.spec.storage.size` field is required and specifies the size of the persistent volume claim. +### Sharding configuration + +The `.spec.sharding` field is optional and specifies the sharding configuration for the Flux controllers. + +Example: + +```yaml +spec: + sharding: + key: "sharding.fluxcd.io/key" + shards: + - "shard1" + - "shard2" +``` + +For each shard, the operator will create a separate set of controllers, e.g.: + +```console +$ kubectl -n flux-system get deployments -l app.kubernetes.io/part-of=flux +NAME +source-controller +source-controller-shard1 +source-controller-shard2 +kustomize-controller +kustomize-controller-shard1 +kustomize-controller-shard2 +helm-controller +helm-controller-shard1 +helm-controller-shard2 +``` + +Note that only the `source-controller`, `kustomize-controller` and `helm-controller` controllers +support sharding. + +To assign a resource to a specific shard, add the `sharding.fluxcd.io/key` label with the shard value, +e.g.: `sharding.fluxcd.io/key: shard1`. + +#### Sharding key + +The `.spec.sharding.key` field is optional and specifies the sharding key label to use for the Flux controllers. +By default, the key is set to `sharding.fluxcd.io/key`. + +#### Shards + +The `.spec.sharding.shards` field is required and specifies the list of sharding values to use for the Flux controllers. + ### Kustomize patches The `.spec.kustomize.patches` field is optional and specifies the Kustomize patches to apply to the Flux controllers. From 71ada12fdf63cca280b9984cd46b0a78b946018d Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 16 Sep 2024 10:14:56 +0300 Subject: [PATCH 5/6] Add sharding to e2e tests Signed-off-by: Stefan Prodan --- config/samples/fluxcd_v1_fluxinstance.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/samples/fluxcd_v1_fluxinstance.yaml b/config/samples/fluxcd_v1_fluxinstance.yaml index 3d26d27..2103ee3 100644 --- a/config/samples/fluxcd_v1_fluxinstance.yaml +++ b/config/samples/fluxcd_v1_fluxinstance.yaml @@ -22,6 +22,8 @@ spec: storage: class: "standard" size: "1Gi" + sharding: + shards: [ "shard1" ] sync: kind: GitRepository url: "https://github.com/controlplaneio-fluxcd/distribution.git" From ad81e8c591bd815b74c3dcf65d2f6f73aa291a74 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 16 Sep 2024 10:24:40 +0300 Subject: [PATCH 6/6] Update dependencies to Kubernetes v1.31.1 Signed-off-by: Stefan Prodan --- go.mod | 28 ++++++++++++++-------------- go.sum | 56 ++++++++++++++++++++++++++++---------------------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index c918852..8bbbc40 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,12 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.27.33 github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.23.6 github.com/fluxcd/cli-utils v0.36.0-flux.9 - github.com/fluxcd/pkg/apis/kustomize v1.6.0 - github.com/fluxcd/pkg/apis/meta v1.6.0 + github.com/fluxcd/pkg/apis/kustomize v1.6.1 + github.com/fluxcd/pkg/apis/meta v1.6.1 github.com/fluxcd/pkg/kustomize v1.13.0 - github.com/fluxcd/pkg/runtime v0.49.0 - github.com/fluxcd/pkg/ssa v0.41.0 - github.com/fluxcd/pkg/tar v0.8.0 + github.com/fluxcd/pkg/runtime v0.49.1 + github.com/fluxcd/pkg/ssa v0.41.1 + github.com/fluxcd/pkg/tar v0.8.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/go-containerregistry v0.20.2 github.com/onsi/ginkgo/v2 v2.20.2 @@ -23,10 +23,10 @@ require ( github.com/prometheus/client_golang v1.20.3 github.com/spf13/pflag v1.0.5 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - k8s.io/api v0.31.0 - k8s.io/apiextensions-apiserver v0.31.0 - k8s.io/apimachinery v0.31.0 - k8s.io/client-go v0.31.0 + k8s.io/api v0.31.1 + k8s.io/apiextensions-apiserver v0.31.1 + k8s.io/apimachinery v0.31.1 + k8s.io/client-go v0.31.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.19.0 sigs.k8s.io/yaml v1.4.0 @@ -51,7 +51,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect - github.com/cyphar/filepath-securejoin v0.3.1 // indirect + github.com/cyphar/filepath-securejoin v0.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/cli v27.2.1+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect @@ -60,7 +60,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fluxcd/pkg/envsubst v1.1.0 // indirect - github.com/fluxcd/pkg/sourceignore v0.8.0 // indirect + github.com/fluxcd/pkg/sourceignore v0.8.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-errors/errors v1.5.1 // indirect @@ -133,11 +133,11 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/cli-runtime v0.31.0 // indirect - k8s.io/component-base v0.31.0 // indirect + k8s.io/cli-runtime v0.31.1 // indirect + k8s.io/component-base v0.31.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3 // indirect - k8s.io/kubectl v0.31.0 // indirect + k8s.io/kubectl v0.31.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.17.3 // indirect sigs.k8s.io/kustomize/kyaml v0.17.2 // indirect diff --git a/go.sum b/go.sum index 17d80ec..e67107e 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.3.1 h1:1V7cHiaW+C+39wEfpH6XlLBQo3j/PciWFrgfCLS8XrE= -github.com/cyphar/filepath-securejoin v0.3.1/go.mod h1:F7i41x/9cBF7lzCrVsYs9fuzwRZm4NQsGTBdpp6mETc= +github.com/cyphar/filepath-securejoin v0.3.2 h1:QhZu5AxQ+o1XZH0Ye05YzvJ0kAdK6VQc0z9NNMek7gc= +github.com/cyphar/filepath-securejoin v0.3.2/go.mod h1:F7i41x/9cBF7lzCrVsYs9fuzwRZm4NQsGTBdpp6mETc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -73,22 +73,22 @@ github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fluxcd/cli-utils v0.36.0-flux.9 h1:RITKdwIAqT3EFKXl7B91mj6usVjxcy7W8PJZlxqUa84= github.com/fluxcd/cli-utils v0.36.0-flux.9/go.mod h1:q6lXQpbAlrZmTB4Qe5oAENkv0y2kwMWcqTMDHrRo2Is= -github.com/fluxcd/pkg/apis/kustomize v1.6.0 h1:G8Nj4ec8CeReT7nhwGLOuli9s+fInEk8gG+xjpDmZWQ= -github.com/fluxcd/pkg/apis/kustomize v1.6.0/go.mod h1:uWiRCCq/HFW9UraizQKJWi0/jB3QrkYcSvXa0uVyQig= -github.com/fluxcd/pkg/apis/meta v1.6.0 h1:93TcRpiph0OCoQh+cI+PM7E35kBW9dScuas9tWc90Dw= -github.com/fluxcd/pkg/apis/meta v1.6.0/go.mod h1:ZOeHcvyVdZDC5ZOGV7YuwplIvAx6LvmpeyhfTcNZCnc= +github.com/fluxcd/pkg/apis/kustomize v1.6.1 h1:22FJc69Mq4i8aCxnKPlddHhSMyI4UPkQkqiAdWFcqe0= +github.com/fluxcd/pkg/apis/kustomize v1.6.1/go.mod h1:5dvQ4IZwz0hMGmuj8tTWGtarsuxW0rWsxJOwC6i+0V8= +github.com/fluxcd/pkg/apis/meta v1.6.1 h1:maLhcRJ3P/70ArLCY/LF/YovkxXbX+6sTWZwZQBeNq0= +github.com/fluxcd/pkg/apis/meta v1.6.1/go.mod h1:YndB/gxgGZmKfqpAfFxyCDNFJFP0ikpeJzs66jwq280= github.com/fluxcd/pkg/envsubst v1.1.0 h1:b0a9QsG36btk3MIWf7yM9FhVPhyXh6lLJu8eZk4Fyow= github.com/fluxcd/pkg/envsubst v1.1.0/go.mod h1:4Uca9c2Bhu4+65sa6NbChEA3zZKhqyAjtgDEi+Zq9Y8= github.com/fluxcd/pkg/kustomize v1.13.0 h1:Gfchiw5uNQ7qzHIZwBF2RoE+A2pLkpwbJhZTmgYemxw= github.com/fluxcd/pkg/kustomize v1.13.0/go.mod h1:HtoEWrgeSB1+gzoaGMncwTrEPDQxBgFcLXmE6HhO/VE= -github.com/fluxcd/pkg/runtime v0.49.0 h1:XldsD4C2TsfuIgku3NEQYCXFLZWDau22YqClTGUihVo= -github.com/fluxcd/pkg/runtime v0.49.0/go.mod h1:0JYsoNhrBtBC4mKAuZdfrkfIqsVGAXKM/A234HuNSnk= -github.com/fluxcd/pkg/sourceignore v0.8.0 h1:oHQZ0Fnk88T7EQKfUshgZ4MULVKlt/AbW4C8Chmrrx4= -github.com/fluxcd/pkg/sourceignore v0.8.0/go.mod h1:6dYIHKdlaATjY/e32EDabfyx0m89ObvlYQesJQoPPOc= -github.com/fluxcd/pkg/ssa v0.41.0 h1:UFrnHJ/cT2+6Qoh98o7INipSoj8GjwMEtb9hLus15xQ= -github.com/fluxcd/pkg/ssa v0.41.0/go.mod h1:Lfu6g8AGbJ/MHSq5zSOBWMTJu9pPC5dG1ykmYC1NTPs= -github.com/fluxcd/pkg/tar v0.8.0 h1:YcEW7K40/XM8o+bkU23dceWtxdaKUpsKcsppLSp8QWc= -github.com/fluxcd/pkg/tar v0.8.0/go.mod h1:O0WUC+nUIw7Cnw1h/4V310kLvzW4tvacD/VZTJtGBUM= +github.com/fluxcd/pkg/runtime v0.49.1 h1:Xyruu1VvkaKZaAhm/32tHJnHab9aU3HzZCf+w6Xoq2A= +github.com/fluxcd/pkg/runtime v0.49.1/go.mod h1:ieDaIEcxzVj77Nw64q4Vd3ZGYdLqpnXOr+GX+XwqTS4= +github.com/fluxcd/pkg/sourceignore v0.8.1 h1:XoZLVBO695PAQm+D4HrVQQ/2tQmIHhMdKWENOYG29MI= +github.com/fluxcd/pkg/sourceignore v0.8.1/go.mod h1:UhIBe0VFTMctFg64tyTjVeHZMS9Ij+nC1cz6mPqmAqk= +github.com/fluxcd/pkg/ssa v0.41.1 h1:VW87zsLYAKUvCxJhuEH7VzxVh3SxaU+PyApCT6gKjTk= +github.com/fluxcd/pkg/ssa v0.41.1/go.mod h1:7cbyLHqFd5FpcKvhxbHG3DkMm3cZteW45Mi78B0hg8g= +github.com/fluxcd/pkg/tar v0.8.1 h1:K9RWV+E/+Qbz6Mzcg+S9DkVvZrWwJq4957Kqms183RQ= +github.com/fluxcd/pkg/tar v0.8.1/go.mod h1:vuGrnXQPcdi3M4DoVtwvAyvLnSeFgXRJckTGYuZOy2Q= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -322,24 +322,24 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= -k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= -k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= -k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= -k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= -k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/cli-runtime v0.31.0 h1:V2Q1gj1u3/WfhD475HBQrIYsoryg/LrhhK4RwpN+DhA= -k8s.io/cli-runtime v0.31.0/go.mod h1:vg3H94wsubuvWfSmStDbekvbla5vFGC+zLWqcf+bGDw= -k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= -k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= -k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs= -k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo= +k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= +k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= +k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40= +k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= +k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= +k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/cli-runtime v0.31.1 h1:/ZmKhmZ6hNqDM+yf9s3Y4KEYakNXUn5sod2LWGGwCuk= +k8s.io/cli-runtime v0.31.1/go.mod h1:pKv1cDIaq7ehWGuXQ+A//1OIF+7DI+xudXtExMCbe9U= +k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= +k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= +k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8= +k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3 h1:SbdLaI6mM6ffDSJCadEaD4IkuPzepLDGlkd2xV0t1uA= k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/kubectl v0.31.0 h1:kANwAAPVY02r4U4jARP/C+Q1sssCcN/1p9Nk+7BQKVg= -k8s.io/kubectl v0.31.0/go.mod h1:pB47hhFypGsaHAPjlwrNbvhXgmuAr01ZBvAIIUaI8d4= +k8s.io/kubectl v0.31.1 h1:ih4JQJHxsEggFqDJEHSOdJ69ZxZftgeZvYo7M/cpp24= +k8s.io/kubectl v0.31.1/go.mod h1:aNuQoR43W6MLAtXQ/Bu4GDmoHlbhHKuyD49lmTC8eJM= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q=