From d59834c5b6963b03864a85302af0178f2a3f12ac Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 5 Mar 2023 23:47:39 +0800 Subject: [PATCH 1/8] generate crds Signed-off-by: Shi, Crane --- ...etal.dell.com_availablecapacityreservations.yaml | 13 +++++-------- .../crds/csi-baremetal.dell.com_volumes.yaml | 11 +++++------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_availablecapacityreservations.yaml b/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_availablecapacityreservations.yaml index bc721701..d111f3d8 100644 --- a/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_availablecapacityreservations.yaml +++ b/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_availablecapacityreservations.yaml @@ -41,18 +41,13 @@ spec: name: v1 schema: openAPIV3Schema: - description: AvailableCapacityReservation is the Schema for the availablecapacitiereservations - API + description: AvailableCapacityReservation is the Schema for the availablecapacitiereservations API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -86,6 +81,8 @@ spec: type: integer StorageClass: type: string + StorageGroup: + type: string type: object Reservations: description: reservation filled by csi driver controller diff --git a/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_volumes.yaml b/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_volumes.yaml index 23b4aec1..f4298199 100644 --- a/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_volumes.yaml +++ b/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_volumes.yaml @@ -62,14 +62,10 @@ spec: description: Volume is the Schema for the volumes API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -78,6 +74,7 @@ spec: CSIStatus: type: string Ephemeral: + description: inline volumes are not support anymore. need to remove field in the next version type: boolean Health: type: string @@ -102,6 +99,8 @@ spec: type: integer StorageClass: type: string + StorageGroup: + type: string Type: type: string Usage: From 56eef360dab68ee7a3b9635ec23dfd524f3aadff Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sat, 6 May 2023 23:15:35 +0800 Subject: [PATCH 2/8] add storagegroup crd Signed-off-by: Shi, Crane --- .../csi-baremetal.dell.com_storagegroups.yaml | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml diff --git a/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml b/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml new file mode 100644 index 00000000..52915c9a --- /dev/null +++ b/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml @@ -0,0 +1,74 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.0 + creationTimestamp: null + name: storagegroups.csi-baremetal.dell.com +spec: + group: csi-baremetal.dell.com + names: + kind: StorageGroup + listKind: StorageGroupList + plural: storagegroups + shortNames: + - sg + - sgs + singular: storagegroup + scope: Cluster + versions: + - additionalPrinterColumns: + - description: NumberDrivesPerNode of StorageGroup's DriveSelector + jsonPath: .spec.DriveSelector.NumberDrivesPerNode + name: DRIVES PER NODE + type: string + - description: Drive Type of StorageGroup's DriveSelector + jsonPath: .spec.DriveSelector.MatchFields.Type + name: TYPE + type: string + - description: Drive Slot of StorageGroup's DriveSelector + jsonPath: .spec.DriveSelector.MatchFields.Slot + name: SLOT + type: string + - description: Drive Bay of StorageGroup's DriveSelector + jsonPath: .spec.DriveSelector.MatchFields.Bay + name: BAY + type: string + name: v1 + schema: + openAPIV3Schema: + description: StorageGroup is the Schema for the StorageGroups API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + DriveSelector: + properties: + MatchFields: + additionalProperties: + type: string + type: object + NumberDrivesPerNode: + format: int64 + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] From 7bed6a535f05ee9cfcb3f745e06c5709529cecc9 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 7 May 2023 19:16:18 +0800 Subject: [PATCH 3/8] refine StorageGroup CRD Signed-off-by: Shi, Crane --- .../csi-baremetal.dell.com_storagegroups.yaml | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml b/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml index 52915c9a..810f59af 100644 --- a/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml +++ b/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml @@ -20,22 +20,26 @@ spec: scope: Cluster versions: - additionalPrinterColumns: - - description: NumberDrivesPerNode of StorageGroup's DriveSelector - jsonPath: .spec.DriveSelector.NumberDrivesPerNode + - description: numberDrivesPerNode of StorageGroup's DriveSelector + jsonPath: .spec.driveSelector.numberDrivesPerNode name: DRIVES PER NODE type: string - description: Drive Type of StorageGroup's DriveSelector - jsonPath: .spec.DriveSelector.MatchFields.Type + jsonPath: .spec.driveSelector.matchFields.Type name: TYPE type: string - description: Drive Slot of StorageGroup's DriveSelector - jsonPath: .spec.DriveSelector.MatchFields.Slot + jsonPath: .spec.driveSelector.matchFields.Slot name: SLOT type: string - description: Drive Bay of StorageGroup's DriveSelector - jsonPath: .spec.DriveSelector.MatchFields.Bay + jsonPath: .spec.driveSelector.matchFields.Bay name: BAY type: string + - description: Drive Path of StorageGroup's DriveSelector + jsonPath: .spec.driveSelector.matchFields.Path + name: PATH + type: string name: v1 schema: openAPIV3Schema: @@ -51,14 +55,14 @@ spec: type: object spec: properties: - DriveSelector: + driveSelector: properties: - MatchFields: + matchFields: additionalProperties: type: string type: object - NumberDrivesPerNode: - format: int64 + numberDrivesPerNode: + format: int32 type: integer type: object type: object From ee59fcc2c0021301fb434e5f7373b4bd2e3da8f7 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Tue, 30 May 2023 16:51:35 +0800 Subject: [PATCH 4/8] update storagegroup crd Signed-off-by: Shi, Crane --- .../crds/csi-baremetal.dell.com_storagegroups.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml b/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml index 810f59af..a5f5917d 100644 --- a/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml +++ b/charts/csi-baremetal-operator/crds/csi-baremetal.dell.com_storagegroups.yaml @@ -22,7 +22,7 @@ spec: - additionalPrinterColumns: - description: numberDrivesPerNode of StorageGroup's DriveSelector jsonPath: .spec.driveSelector.numberDrivesPerNode - name: DRIVES PER NODE + name: DRIVES_PER_NODE type: string - description: Drive Type of StorageGroup's DriveSelector jsonPath: .spec.driveSelector.matchFields.Type @@ -32,14 +32,14 @@ spec: jsonPath: .spec.driveSelector.matchFields.Slot name: SLOT type: string - - description: Drive Bay of StorageGroup's DriveSelector - jsonPath: .spec.driveSelector.matchFields.Bay - name: BAY - type: string - description: Drive Path of StorageGroup's DriveSelector jsonPath: .spec.driveSelector.matchFields.Path name: PATH type: string + - description: Whether StorageGroup's DriveSelector to Select System Drive + jsonPath: .spec.driveSelector.matchFields.IsSystem + name: SYSTEM + type: string name: v1 schema: openAPIV3Schema: From 4c44c5d9ec437f608fb52bf0ce87ae55a4e6d5be Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 11 Jun 2023 15:51:01 +0800 Subject: [PATCH 5/8] fix crds upgrade error in csi-operator upgrade case Signed-off-by: Shi, Crane --- charts/csi-baremetal-operator/templates/pre-upgrade-crds.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/csi-baremetal-operator/templates/pre-upgrade-crds.yaml b/charts/csi-baremetal-operator/templates/pre-upgrade-crds.yaml index d41731d3..7a7fe1b4 100644 --- a/charts/csi-baremetal-operator/templates/pre-upgrade-crds.yaml +++ b/charts/csi-baremetal-operator/templates/pre-upgrade-crds.yaml @@ -129,4 +129,4 @@ spec: memory: {{ .Values.operator.resources.requests.memory }} command: ["/bin/sh", "-c"] args: - - kubectl replace crd -f /crds; + - kubectl apply -f /crds; From 92434684f22a80996ae0413f7e1984e176a9c1a6 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 25 Jun 2023 12:03:29 +0800 Subject: [PATCH 6/8] include sg del/usage in csi uninstall/usage guide Signed-off-by: Shi, Crane --- docs/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index e0171b9a..ddaf0bb7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -240,6 +240,8 @@ Usage * Provisioned volumes - `kubectl get volumes.csi-baremetal.dell.com` + * Storage groups - `kubectl get sgs` + Upgrade process --------------------- @@ -252,8 +254,10 @@ Uninstallation process * Delete custom resources ``` kubectl delete pvc --all + # wait for a while kubectl delete volumes --all -A kubectl delete lvgs --all + kubectl delete sgs --all kubectl delete csibmnodes --all ``` * Delete helm releases @@ -265,5 +269,6 @@ Uninstallation process ``` kubectl delete crd deployments.csi-baremetal.dell.com availablecapacities.csi-baremetal.dell.com \ availablecapacityreservations.csi-baremetal.dell.com logicalvolumegroups.csi-baremetal.dell.com \ - volumes.csi-baremetal.dell.com drives.csi-baremetal.dell.com nodes.csi-baremetal.dell.com + volumes.csi-baremetal.dell.com drives.csi-baremetal.dell.com nodes.csi-baremetal.dell.com \ + storagegroups.csi-baremetal.dell.com ``` From bf7d62a78053869591ee41997720b56585004e95 Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 25 Jun 2023 15:10:09 +0800 Subject: [PATCH 7/8] refine uninstall process in ReadMe Signed-off-by: Shi, Crane --- docs/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index ddaf0bb7..49679d12 100644 --- a/docs/README.md +++ b/docs/README.md @@ -254,7 +254,8 @@ Uninstallation process * Delete custom resources ``` kubectl delete pvc --all - # wait for a while + + # Please run the following commands after the completion of all Bare-metal-CSI-managed PVCs' deletion kubectl delete volumes --all -A kubectl delete lvgs --all kubectl delete sgs --all From ff4a1f1f7aafd295ae59c09b5a2a791eb468a76a Mon Sep 17 00:00:00 2001 From: "Shi, Crane" Date: Sun, 25 Jun 2023 16:40:15 +0800 Subject: [PATCH 8/8] refine ReadMe doc's uninstall process Signed-off-by: Shi, Crane --- docs/README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/README.md b/docs/README.md index 49679d12..23e231a2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -251,11 +251,20 @@ See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command document Uninstallation process --------------------- +* Delete all Bare-metal-CSI-managed PVCs + ``` + pvcs=($(kubectl get pvc -o wide -A | grep csi-baremetal-sc | awk {'print $1" "$2'})) + for (( i=0; i<${#pvcs[@]} ; i+=2 )) + do + kubectl delete pvc ${pvcs[i+1]} -n ${pvcs[i]} + done + ``` +* Ensure there is no Bare-metal-CSI-managed PVC left + ``` + kubectl get pvc -o wide -A | grep csi-baremetal-sc + ``` * Delete custom resources ``` - kubectl delete pvc --all - - # Please run the following commands after the completion of all Bare-metal-CSI-managed PVCs' deletion kubectl delete volumes --all -A kubectl delete lvgs --all kubectl delete sgs --all