Merge pull request #2854 from red-hat-storage/sync_ds--main

Syncing latest changes from main for ocs-operator
red-hat-storage · Oct 18, 2024 · 45a093f · 45a093f
2 parents c07d408 + a1ff3bc
commit 45a093f
Show file tree

Hide file tree

Showing 139 changed files with 33,071 additions and 55 deletions.
diff --git a/api/v1/storagecluster_types.go b/api/v1/storagecluster_types.go
@@ -402,6 +402,10 @@ type StorageDeviceSet struct {
  DataPVCTemplate corev1.PersistentVolumeClaim `json:"dataPVCTemplate"`
  MetadataPVCTemplate *corev1.PersistentVolumeClaim `json:"metadataPVCTemplate,omitempty"`
  WalPVCTemplate *corev1.PersistentVolumeClaim `json:"walPVCTemplate,omitempty"`
+
+ // Whether to encrypt the deviceSet or not
+ // +optional
+ Encrypted *bool `json:"encrypted,omitempty"`
 }
 
 // TODO: Fill in the members when the actual configurable options are defined in rook-ceph

diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/api/v1alpha1/odfinfoconfig.go b/api/v1alpha1/odfinfoconfig.go
@@ -22,13 +22,15 @@ import "k8s.io/apimachinery/pkg/types"
 type ConnectedClient struct {
  Name string `yaml:"name"`
  ClusterID string `yaml:"clusterId"`
+ ClientID string `yaml:"clientId"`
 }
 
 // InfoStorageCluster describes information regarding a storage cluster key
 type InfoStorageCluster struct {
  NamespacedName types.NamespacedName `yaml:"namespacedName"`
  StorageProviderEndpoint string `yaml:"storageProviderEndpoint"`
  CephClusterFSID string `yaml:"cephClusterFSID"`
+ StorageClusterUID string `yaml:"storageClusterUID"`
 }
 
 // OdfInfoData describes odf-info CM's data

diff --git a/api/v1alpha1/storageconsumer_types.go b/api/v1alpha1/storageconsumer_types.go
@@ -93,7 +93,7 @@ type ClientStatus struct {
 
  // StorageQuotaUtilizationRatio is the ratio of utilized quota of connected client
  // +optional
- StorageQuotaUtilizationRatio float32 `json:"storageQuotaUtilizationRatio,omitempty"`
+ StorageQuotaUtilizationRatio float64 `json:"storageQuotaUtilizationRatio,omitempty"`
 }
 
 //+kubebuilder:object:root=true

diff --git a/config/crd/bases/ocs.openshift.io_storageclusters.yaml b/config/crd/bases/ocs.openshift.io_storageclusters.yaml
@@ -4452,6 +4452,9 @@ spec:
  - NVME
  - nvme
  type: string
+ encrypted:
+ description: Whether to encrypt the deviceSet or not
+ type: boolean
  initialWeight:
  description: |-
  InitialWeight is an optional explicit OSD weight value in TiB units.

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -119,6 +119,7 @@ rules:
  - delete
  - get
  - list
+ - patch
  - update
  - watch
 - apiGroups:

diff --git a/controllers/storagecluster/cephcluster.go b/controllers/storagecluster/cephcluster.go
@@ -167,10 +167,11 @@ func (obj *ocsCephCluster) ensureCreated(r *StorageClusterReconciler, sc *ocsv1.
  cephCluster = newExternalCephCluster(sc, monitoringIP, monitoringPort)
  } else {
  // Add KMS details to CephCluster spec, only if
- // cluster-wide encryption is enabled
- // ie, sc.Spec.Encryption.ClusterWide/sc.Spec.Encryption.Enable is True
+ // cluster-wide encryption is enabled or any of the device set is encrypted
+ // ie, sc.Spec.Encryption.ClusterWide/sc.Spec.Encryption.Enable is True or any device is encrypted
  // and KMS ConfigMap is available
- if sc.Spec.Encryption.Enable || sc.Spec.Encryption.ClusterWide {
+
+ if util.IsClusterOrDeviceSetEncrypted(sc) {
  kmsConfigMap, err := getKMSConfigMap(KMSConfigMapName, sc, r.Client)
  if err != nil {
  r.Log.Error(err, "Failed to procure KMS ConfigMap.", "KMSConfigMap", klog.KRef(sc.Namespace, KMSConfigMapName))
@@ -605,13 +606,11 @@ func getNetworkSpec(sc ocsv1.StorageCluster) rookCephv1.NetworkSpec {
  // respect both the old way and the new way for enabling HostNetwork
  networkSpec.HostNetwork = networkSpec.HostNetwork || sc.Spec.HostNetwork
 
- // If it's not an external and not a provider cluster always require msgr2
- if !sc.Spec.AllowRemoteStorageConsumers && !sc.Spec.ExternalStorage.Enable {
- if networkSpec.Connections == nil {
- networkSpec.Connections = &rookCephv1.ConnectionsSpec{}
- }
- networkSpec.Connections.RequireMsgr2 = true
+ if networkSpec.Connections == nil {
+ networkSpec.Connections = &rookCephv1.ConnectionsSpec{}
  }
+ // Always require msgr2 for all modes of ODF
+ networkSpec.Connections.RequireMsgr2 = true
 
  return networkSpec
 }
@@ -882,7 +881,7 @@ func newStorageClassDeviceSets(sc *ocsv1.StorageCluster) []rookCephv1.StorageCla
  Portable: portable,
  TuneSlowDeviceClass: ds.Config.TuneSlowDeviceClass,
  TuneFastDeviceClass: ds.Config.TuneFastDeviceClass,
- Encrypted: sc.Spec.Encryption.Enable || sc.Spec.Encryption.ClusterWide,
+ Encrypted: isDeviceSetToBeEncrypted(sc, ds),
  }
 
  if ds.MetadataPVCTemplate != nil {
@@ -989,6 +988,13 @@ func countAndReplicaOf(ds *ocsv1.StorageDeviceSet) (int, int) {
  return count, replica
 }
 
+func isDeviceSetToBeEncrypted(sc *ocsv1.StorageCluster, ds ocsv1.StorageDeviceSet) bool {
+ if ds.Encrypted != nil {
+ return *ds.Encrypted
+ }
+ return sc.Spec.Encryption.Enable || sc.Spec.Encryption.ClusterWide
+}
+
 func newCephDaemonResources(sc *ocsv1.StorageCluster) map[string]corev1.ResourceRequirements {
  resources := map[string]corev1.ResourceRequirements{
  "mon": defaults.GetProfileDaemonResources("mon", sc),

diff --git a/controllers/storagecluster/cephcluster_test.go b/controllers/storagecluster/cephcluster_test.go
@@ -1301,7 +1301,9 @@ func TestCephClusterNetworkConnectionsSpec(t *testing.T) {
  },
  ccSpec: rookCephv1.ClusterSpec{
  Network: rookCephv1.NetworkSpec{
- Connections: &rookCephv1.ConnectionsSpec{},
+ Connections: &rookCephv1.ConnectionsSpec{
+ RequireMsgr2: true,
+ },
  },
  },
  },
@@ -1319,6 +1321,7 @@ func TestCephClusterNetworkConnectionsSpec(t *testing.T) {
  ccSpec: rookCephv1.ClusterSpec{
  Network: rookCephv1.NetworkSpec{
  Connections: &rookCephv1.ConnectionsSpec{
+ RequireMsgr2: true,
  Encryption: &rookCephv1.EncryptionSpec{
  Enabled: true,
  },
@@ -1340,6 +1343,7 @@ func TestCephClusterNetworkConnectionsSpec(t *testing.T) {
  ccSpec: rookCephv1.ClusterSpec{
  Network: rookCephv1.NetworkSpec{
  Connections: &rookCephv1.ConnectionsSpec{
+ RequireMsgr2: true,
  Compression: &rookCephv1.CompressionSpec{
  Enabled: true,
  },
@@ -1364,6 +1368,7 @@ func TestCephClusterNetworkConnectionsSpec(t *testing.T) {
  ccSpec: rookCephv1.ClusterSpec{
  Network: rookCephv1.NetworkSpec{
  Connections: &rookCephv1.ConnectionsSpec{
+ RequireMsgr2: true,
  Encryption: &rookCephv1.EncryptionSpec{
  Enabled: true,
  },
@@ -1391,6 +1396,7 @@ func TestCephClusterNetworkConnectionsSpec(t *testing.T) {
  ccSpec: rookCephv1.ClusterSpec{
  Network: rookCephv1.NetworkSpec{
  Connections: &rookCephv1.ConnectionsSpec{
+ RequireMsgr2: true,
  Encryption: &rookCephv1.EncryptionSpec{
  Enabled: false,
  },

diff --git a/controllers/storagecluster/cephobjectstores.go b/controllers/storagecluster/cephobjectstores.go
@@ -7,6 +7,7 @@ import (
  ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
  "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults"
  "github.com/red-hat-storage/ocs-operator/v4/controllers/platform"
+ "github.com/red-hat-storage/ocs-operator/v4/controllers/util"
  cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
  corev1 "k8s.io/api/core/v1"
  "k8s.io/apimachinery/pkg/api/errors"
@@ -42,10 +43,10 @@ func (obj *ocsCephObjectStores) ensureCreated(r *StorageClusterReconciler, insta
  }
  var cephObjectStores []*cephv1.CephObjectStore
  // Add KMS details to cephObjectStores spec, only if
- // cluster-wide encryption is enabled
- // ie, sc.Spec.Encryption.ClusterWide/sc.Spec.Encryption.Enable is True
+ // cluster-wide encryption is enabled or any of the device set is encrypted
+ // ie, sc.Spec.Encryption.ClusterWide/sc.Spec.Encryption.Enable is True or any of the deviceSet is encrypted
  // and KMS ConfigMap is available
- if instance.Spec.Encryption.Enable || instance.Spec.Encryption.ClusterWide {
+ if util.IsClusterOrDeviceSetEncrypted(instance) {
  kmsConfigMap, err := getKMSConfigMap(KMSConfigMapName, instance, r.Client)
  if err != nil {
  r.Log.Error(err, "Failed to procure KMS ConfigMap.", "KMSConfigMap", klog.KRef(instance.Namespace, KMSConfigMapName))

diff --git a/controllers/storagecluster/noobaa_system_reconciler.go b/controllers/storagecluster/noobaa_system_reconciler.go
@@ -161,10 +161,15 @@ func (r *StorageClusterReconciler) setNooBaaDesiredState(nb *nbv1.NooBaa, sc *oc
  placement := getPlacement(sc, component)
 
  nb.Spec.Tolerations = placement.Tolerations
- // if we are "noobaa-standalone" and placement is not set - don't set affinity
+
  if !r.IsNoobaaStandalone || ok {
+ // Add affinity if not in noobaa-standalone mode or if placement is specified
  nb.Spec.Affinity = &corev1.Affinity{NodeAffinity: placement.NodeAffinity}
+ } else if nb.Spec.Affinity != nil {
+ // Clear the affinity if it was set previously to handle upgrades
+ nb.Spec.Affinity = nil
  }
+
  nb.Spec.DBVolumeResources = &corev1.VolumeResourceRequirements{
  Limits: dBVolumeResources.Limits,
  Requests: dBVolumeResources.Requests,
@@ -230,10 +235,10 @@ func (r *StorageClusterReconciler) setNooBaaDesiredState(nb *nbv1.NooBaa, sc *oc
 
  // Add KMS details to Noobaa spec, only if
  // KMS is enabled, along with
- // ClusterWide encryption OR in a StandAlone Noobaa cluster mode
+ // ClusterWide encryption/any deviceSet Encryption OR in a StandAlone Noobaa cluster mode
  // PS: sc.Spec.Encryption.Enable field is deprecated and added for backward compatibility
  if sc.Spec.Encryption.KeyManagementService.Enable &&
- (sc.Spec.Encryption.Enable || sc.Spec.Encryption.ClusterWide || r.IsNoobaaStandalone) {
+ (util.IsClusterOrDeviceSetEncrypted(sc) || r.IsNoobaaStandalone) {
  if kmsConfig, err := getKMSConfigMap(KMSConfigMapName, sc, r.Client); err != nil {
  return err
  } else if kmsConfig != nil {

diff --git a/controllers/storagecluster/odfinfoconfig.go b/controllers/storagecluster/odfinfoconfig.go
@@ -151,6 +151,7 @@ func getOdfInfoData(r *StorageClusterReconciler, storageCluster *ocsv1.StorageCl
  NamespacedName: client.ObjectKeyFromObject(storageCluster),
  StorageProviderEndpoint: storageCluster.Status.StorageProviderEndpoint,
  CephClusterFSID: cephFSId,
+ StorageClusterUID: string(storageCluster.UID),
  },
  }
  yamlData, err := yaml.Marshal(data)
@@ -176,6 +177,7 @@ func getConnectedClients(r *StorageClusterReconciler, storageCluster *ocsv1.Stor
  newConnectedClient := ocsv1a1.ConnectedClient{
  Name: name,
  ClusterID: clusterID,
+ ClientID: string(storageConsumer.UID),
  }
  connectedClients = append(connectedClients, newConnectedClient)
  }

diff --git a/controllers/storagecluster/reconcile.go b/controllers/storagecluster/reconcile.go
@@ -114,7 +114,7 @@ var validTopologyLabelKeys = []string{
 // +kubebuilder:rbac:groups=ceph.rook.io,resources=cephclusters;cephblockpools;cephfilesystems;cephnfses;cephobjectstores;cephobjectstoreusers;cephrbdmirrors;cephblockpoolradosnamespaces,verbs=get;list;watch;create;update;delete
 // +kubebuilder:rbac:groups=noobaa.io,resources=noobaas,verbs=get;list;watch;create;update;delete
 // +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=watch;create;delete;get;list
-// +kubebuilder:rbac:groups=core,resources=pods;services;serviceaccounts;endpoints;persistentvolumes;persistentvolumeclaims;events;configmaps;secrets;nodes,verbs=get;list;watch;create;update;delete
+// +kubebuilder:rbac:groups=core,resources=pods;services;serviceaccounts;endpoints;persistentvolumes;persistentvolumeclaims;events;configmaps;secrets;nodes,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=namespaces,verbs=get
 // +kubebuilder:rbac:groups=apps,resources=deployments;daemonsets;replicasets;statefulsets,verbs=get;list;watch;create;update;delete
 // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrules,verbs=get;list;watch;create;update;delete

diff --git a/controllers/util/csi_options.go b/controllers/util/csi_options.go
@@ -13,22 +13,7 @@ func GetCephFSKernelMountOptions(sc *ocsv1.StorageCluster) string {
  return "ms_mode=secure"
  }
 
- // If Encryption is not enabled, but Compression or RequireMsgr2 is enabled, use prefer-crc mode
- if sc.Spec.Network != nil && sc.Spec.Network.Connections != nil &&
- ((sc.Spec.Network.Connections.Compression != nil && sc.Spec.Network.Connections.Compression.Enabled) ||
- sc.Spec.Network.Connections.RequireMsgr2) {
- return "ms_mode=prefer-crc"
- }
-
- // Network spec always has higher precedence even in the External or Provider cluster. so they are checked first above
-
- // None of Encryption, Compression, RequireMsgr2 are enabled on the StorageCluster
- // If it's an External or Provider cluster, We don't require msgr2 by default so no mount options are needed
- if sc.Spec.ExternalStorage.Enable || sc.Spec.AllowRemoteStorageConsumers {
- return "ms_mode=legacy"
- }
- // If none of the above cases apply, We set RequireMsgr2 true by default on the cephcluster
- // so we need to set the mount options to prefer-crc
+ // If encryption is not enabled, use prefer-crc mode
  return "ms_mode=prefer-crc"
 }
 

diff --git a/controllers/util/util.go b/controllers/util/util.go
@@ -5,9 +5,10 @@ import (
  "encoding/hex"
  "encoding/json"
  "fmt"
- ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
  "os"
 
+ ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
+
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -42,8 +43,8 @@ func GetKeyRotationSpec(sc *ocsv1.StorageCluster) (bool, string) {
  }
 
  if sc.Spec.Encryption.KeyRotation.Enable == nil {
- if (sc.Spec.Encryption.Enable || sc.Spec.Encryption.ClusterWide) && !sc.Spec.Encryption.KeyManagementService.Enable {
- // use key-rotation by default if cluster-wide encryption is opted without KMS & "enable" spec is missing
+ if IsClusterOrDeviceSetEncrypted(sc) && !sc.Spec.Encryption.KeyManagementService.Enable {
+ // use key-rotation by default if cluster-wide encryption/any deviceSet encryption is opted without KMS & "enable" spec is missing
  return true, schedule
  }
  return false, schedule
@@ -103,3 +104,19 @@ func AssertEqual[T comparable](actual T, expected T, exitCode int) {
  os.Exit(exitCode)
  }
 }
+
+func IsClusterOrDeviceSetEncrypted(sc *ocsv1.StorageCluster) bool {
+ // If cluster-wide encryption is enabled
+ if sc.Spec.Encryption.Enable || sc.Spec.Encryption.ClusterWide {
+ return true
+ }
+
+ // If any device set is encrypted
+ for _, deviceSet := range sc.Spec.StorageDeviceSets {
+ if deviceSet.Encrypted != nil && *deviceSet.Encrypted {
+ return true
+ }
+ }
+
+ return false
+}
diff --git a/deploy/csv-templates/crds/ocs/ocs.openshift.io_storageclusters.yaml b/deploy/csv-templates/crds/ocs/ocs.openshift.io_storageclusters.yaml
@@ -4452,6 +4452,9 @@ spec:
  - NVME
  - nvme
  type: string
+ encrypted:
+ description: Whether to encrypt the deviceSet or not
+ type: boolean
  initialWeight:
  description: |-
  InitialWeight is an optional explicit OSD weight value in TiB units.

diff --git a/deploy/csv-templates/ocs-operator.csv.yaml.in b/deploy/csv-templates/ocs-operator.csv.yaml.in
@@ -290,6 +290,7 @@ spec:
  - delete
  - get
  - list
+ - patch
  - update
  - watch
  - apiGroups:

diff --git a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml
@@ -299,6 +299,7 @@ spec:
  - delete
  - get
  - list
+ - patch
  - update
  - watch
  - apiGroups:

diff --git a/deploy/ocs-operator/manifests/storagecluster.crd.yaml b/deploy/ocs-operator/manifests/storagecluster.crd.yaml
@@ -4452,6 +4452,9 @@ spec:
  - NVME
  - nvme
  type: string
+ encrypted:
+ description: Whether to encrypt the deviceSet or not
+ type: boolean
  initialWeight:
  description: |-
  InitialWeight is an optional explicit OSD weight value in TiB units.

diff --git a/metrics/go.mod b/metrics/go.mod
@@ -50,6 +50,7 @@ require (
  github.com/containernetworking/cni v1.2.3 // indirect
  github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
  github.com/emicklei/go-restful/v3 v3.12.1 // indirect
+ github.com/evanphx/json-patch/v5 v5.9.0 // indirect
  github.com/fxamacker/cbor/v2 v2.7.0 // indirect
  github.com/go-jose/go-jose/v4 v4.0.4 // indirect
  github.com/go-logr/logr v1.4.2 // indirect
@@ -58,6 +59,7 @@ require (
  github.com/go-openapi/swag v0.23.0 // indirect
  github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
  github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
  github.com/golang/protobuf v1.5.4 // indirect
  github.com/google/gnostic-models v0.6.8 // indirect
  github.com/google/go-cmp v0.6.0 // indirect

diff --git a/metrics/go.sum b/metrics/go.sum
@@ -201,6 +201,8 @@ github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
+github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
@@ -315,6 +317,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -1433,6 +1436,8 @@ k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE=
 k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE=
 k8s.io/apiextensions-apiserver v0.18.3/go.mod h1:TMsNGs7DYpMXd+8MOCX8KzPOCx8fnZMoIGB24m03+JE=
 k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk=
+k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk=
+k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk=
 k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
 k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
 k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=