[ISSUE-1018] Custom Storage Group Feature

Makefile

@@ -112,6 +112,7 @@ generate-deepcopy:
 	controller-gen object paths=api/v1/drivecrd/drive_types.go paths=api/v1/drivecrd/groupversion_info.go  output:dir=api/v1/drivecrd
 	controller-gen object paths=api/v1/lvgcrd/logicalvolumegroup_types.go paths=api/v1/lvgcrd/groupversion_info.go  output:dir=api/v1/lvgcrd
 	controller-gen object paths=api/v1/nodecrd/node_types.go paths=api/v1/nodecrd/groupversion_info.go  output:dir=api/v1/nodecrd
+	controller-gen object paths=api/v1/storagegroupcrd/storagegroup_types.go paths=api/v1/storagegroupcrd/groupversion_info.go  output:dir=api/v1/storagegroupcrd
 
 generate-baremetal-crds: install-controller-gen
 	controller-gen $(CRD_OPTIONS) paths=api/v1/availablecapacitycrd/availablecapacity_types.go paths=api/v1/availablecapacitycrd/groupversion_info.go output:crd:dir=$(CSI_CHART_CRDS_PATH)
@@ -120,6 +121,7 @@ generate-baremetal-crds: install-controller-gen
 	controller-gen $(CRD_OPTIONS) paths=api/v1/drivecrd/drive_types.go paths=api/v1/drivecrd/groupversion_info.go output:crd:dir=$(CSI_CHART_CRDS_PATH)
 	controller-gen $(CRD_OPTIONS) paths=api/v1/lvgcrd/logicalvolumegroup_types.go paths=api/v1/lvgcrd/groupversion_info.go output:crd:dir=$(CSI_CHART_CRDS_PATH)
 	controller-gen $(CRD_OPTIONS) paths=api/v1/nodecrd/node_types.go paths=api/v1/nodecrd/groupversion_info.go output:crd:dir=$(CSI_CHART_CRDS_PATH)
+	controller-gen $(CRD_OPTIONS) paths=api/v1/storagegroupcrd/storagegroup_types.go paths=api/v1/storagegroupcrd/groupversion_info.go output:crd:dir=$(CSI_CHART_CRDS_PATH)
 
 generate-api: compile-proto generate-baremetal-crds generate-deepcopy
 

api/v1/constants.go

@@ -138,4 +138,7 @@ const (
 	// CSI Drive taint-like label key and value
 	DriveTaintKey   = "drive.csi-baremetal.dell.com/taint"
 	DriveTaintValue = "NoSchedule"
+
+	// CSI StorageGroup label key
+	StorageGroupLabelKey = "drive.csi-baremetal.dell.com/storage-group"
 )

api/v1/storagegroupcrd/groupversion_info.go

@@ -0,0 +1,38 @@
+/*
+Copyright © 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package sgcrd contains API Schema definitions for the StorageGroup v1 API group
+// +groupName=csi-baremetal.dell.com
+// +versionName=v1
+package sgcrd
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	crScheme "sigs.k8s.io/controller-runtime/pkg/scheme"
+
+	v1 "github.com/dell/csi-baremetal/api/v1"
+)
+
+var (
+	// GroupVersionStorageGroup is group version used to register these objects
+	GroupVersionStorageGroup = schema.GroupVersion{Group: v1.CSICRsGroupVersion, Version: v1.Version}
+
+	// SchemeBuilderStorageGroup is used to add go types to the GroupVersionKind scheme
+	SchemeBuilderStorageGroup = &crScheme.Builder{GroupVersion: GroupVersionStorageGroup}
+
+	// AddToSchemeStorageGroup adds the types in this group-version to the given scheme.
+	AddToSchemeStorageGroup = SchemeBuilderStorageGroup.AddToScheme
+)

api/v1/storagegroupcrd/storagegroup_types.go

@@ -0,0 +1,58 @@
+/*
+Copyright © 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sgcrd
+
+import (
+	api "github.com/dell/csi-baremetal/api/generated/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +kubebuilder:object:root=true
+
+// StorageGroup is the Schema for the StorageGroups API
+// +kubebuilder:resource:scope=Cluster,shortName={sg,sgs}
+// +kubebuilder:printcolumn:name="DRIVES_PER_NODE",type="string",JSONPath=".spec.driveSelector.numberDrivesPerNode",description="numberDrivesPerNode of StorageGroup's DriveSelector"
+// +kubebuilder:printcolumn:name="TYPE",type="string",JSONPath=".spec.driveSelector.matchFields.Type",description="Drive Type of StorageGroup's DriveSelector"
+// +kubebuilder:printcolumn:name="SLOT",type="string",JSONPath=".spec.driveSelector.matchFields.Slot",description="Drive Slot of StorageGroup's DriveSelector"
+// +kubebuilder:printcolumn:name="PATH",type="string",JSONPath=".spec.driveSelector.matchFields.Path",description="Drive Path of StorageGroup's DriveSelector"
+// +kubebuilder:printcolumn:name="SYSTEM",type="string",JSONPath=".spec.driveSelector.matchFields.IsSystem",description="Whether StorageGroup's DriveSelector to Select System Drive"
+type StorageGroup struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              api.StorageGroupSpec `json:"spec,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// StorageGroupList contains a list of StorageGroup
+//+kubebuilder:object:generate=true
+type StorageGroupList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []StorageGroup `json:"items"`
+}
+
+func init() {
+	SchemeBuilderStorageGroup.Register(&StorageGroup{}, &StorageGroupList{})
+}
+
+func (in *StorageGroup) DeepCopyInto(out *StorageGroup) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	out.Spec = in.Spec
+}

api/v1/types.proto

@@ -46,6 +46,7 @@ message Volume {
     string Usage = 13;
     // inline volumes are not support anymore. need to remove field in the next version
     bool Ephemeral = 14;
+    string StorageGroup = 15;
 }
 
 message AvailableCapacity {
@@ -80,6 +81,7 @@ message CapacityRequest {
     string Name = 1;
     string StorageClass = 2;
     int64 Size = 3;
+    string StorageGroup = 4;
 }
 
 message LogicalVolumeGroup {
@@ -97,3 +99,12 @@ message Node {
     // key - address type, value - address, align with NodeAddress struct from k8s.io/api/core/v1
     map<string, string> Addresses = 2;
 }
+
+message StorageGroupSpec {
+    DriveSelector driveSelector = 1;
+}
+
+message DriveSelector {
+    int32 numberDrivesPerNode = 1;
+    map<string, string> matchFields = 2;
+}

cmd/controller/main.go

@@ -40,6 +40,7 @@ import (
 	accrd "github.com/dell/csi-baremetal/api/v1/availablecapacitycrd"
 	"github.com/dell/csi-baremetal/api/v1/drivecrd"
 	"github.com/dell/csi-baremetal/api/v1/lvgcrd"
+	sgcrd "github.com/dell/csi-baremetal/api/v1/storagegroupcrd"
 	"github.com/dell/csi-baremetal/api/v1/volumecrd"
 	"github.com/dell/csi-baremetal/pkg/base"
 	"github.com/dell/csi-baremetal/pkg/base/featureconfig"
@@ -51,6 +52,7 @@ import (
 	"github.com/dell/csi-baremetal/pkg/controller"
 	"github.com/dell/csi-baremetal/pkg/controller/capacitycontroller"
 	"github.com/dell/csi-baremetal/pkg/crcontrollers/reservation"
+	"github.com/dell/csi-baremetal/pkg/crcontrollers/storagegroup"
 	"github.com/dell/csi-baremetal/pkg/metrics"
 )
 
@@ -168,6 +170,10 @@ func createManager(ctx context.Context, client *k8s.KubeClient, log *logrus.Logg
 		return nil, err
 	}
 
+	if err := sgcrd.AddToSchemeStorageGroup(scheme); err != nil {
+		return nil, err
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:    scheme,
 		Namespace: *namespace,
@@ -196,5 +202,11 @@ func createManager(ctx context.Context, client *k8s.KubeClient, log *logrus.Logg
 	if err = capacityController.SetupWithManager(mgr); err != nil {
 		return nil, err
 	}
+
+	storageGroupController := storagegroup.NewController(client, kubeCache, log)
+	if err = storageGroupController.SetupWithManager(mgr); err != nil {
+		return nil, err
+	}
+
 	return mgr, nil
 }

devkit/Dockerfile

@@ -10,7 +10,7 @@ RUN           apt update \
 &&            /kind/kind-build.sh /kind
 
 
-FROM opensuse/leap:latest
+FROM opensuse/leap:15.4
 
 ARG           arg_docker_ver
 ARG           arg_go_ver

pkg/base/capacityplanner/node_capacity.go

@@ -154,7 +154,7 @@ func (nc *nodeCapacity) selectACForVolume(vol *genV1.Volume) *accrd.AvailableCap
 	}
 
 	for _, ac := range nc.acsOrder[vol.StorageClass] {
-		if requiredSize <= nc.acs[ac].Spec.Size {
+		if requiredSize <= nc.acs[ac].Spec.Size && nc.acs[ac].Labels[v1.StorageGroupLabelKey] == vol.StorageGroup {
 			// check if AC is reserved
 			reservation, ok := nc.reservedACs[ac]
 

pkg/base/k8s/kubeclient.go

@@ -39,6 +39,7 @@ import (
 	"github.com/dell/csi-baremetal/api/v1/drivecrd"
 	"github.com/dell/csi-baremetal/api/v1/lvgcrd"
 	"github.com/dell/csi-baremetal/api/v1/nodecrd"
+	sgcrd "github.com/dell/csi-baremetal/api/v1/storagegroupcrd"
 	"github.com/dell/csi-baremetal/api/v1/volumecrd"
 	"github.com/dell/csi-baremetal/pkg/base"
 	checkErr "github.com/dell/csi-baremetal/pkg/base/error"
@@ -409,6 +410,12 @@ func PrepareScheme() (*runtime.Scheme, error) {
 		return nil, err
 	}
 
+	// register csi storagegroup crd
+	err := sgcrd.AddToSchemeStorageGroup(scheme)
+	if err != nil {
+		return nil, err
+	}
+
 	return scheme, nil
 }
 

pkg/common/volume_operations.go

@@ -670,7 +670,7 @@ func (vo *VolumeOperationsImpl) fillCache() {
 	}
 }
 
-// VolumeOperationsImpl returns PVC labels: release, app.kubernetes.io/name and adds short app label
+// VolumeOperationsImpl returns PVC labels: release, app.kubernetes.io/name, storagegroup and adds short app label
 func (vo *VolumeOperationsImpl) getPersistentVolumeClaimLabels(ctx context.Context, pvcName, pvcNamespace string) (
 	map[string]string, error) {
 	ll := vo.log.WithFields(logrus.Fields{
@@ -686,7 +686,7 @@ func (vo *VolumeOperationsImpl) getPersistentVolumeClaimLabels(ctx context.Conte
 		return nil, err
 	}
 
-	// need to get release and app labels only
+	// need to get release, app and storagegroup labels
 	labels := map[string]string{}
 	if value, ok := pvc.GetLabels()[k8s.ReleaseLabelKey]; ok {
 		labels[k8s.ReleaseLabelKey] = value
@@ -695,6 +695,9 @@ func (vo *VolumeOperationsImpl) getPersistentVolumeClaimLabels(ctx context.Conte
 		labels[k8s.AppLabelKey] = value
 		labels[k8s.AppLabelShortKey] = value
 	}
+	if value, ok := pvc.GetLabels()[apiV1.StorageGroupLabelKey]; ok {
+		labels[apiV1.StorageGroupLabelKey] = value
+	}
 
 	return labels, nil
 }

pkg/common/volume_operations_test.go

@@ -74,9 +74,11 @@ func Test_getPersistentVolumeClaimLabels(t *testing.T) {
 	var (
 		appName     = "my-app"
 		releaseName = "my-release"
+		sgName      = "my-group"
 		pvcLabels   = map[string]string{
-			k8s.AppLabelKey:     appName,
-			k8s.ReleaseLabelKey: releaseName,
+			k8s.AppLabelKey:            appName,
+			k8s.ReleaseLabelKey:        releaseName,
+			apiV1.StorageGroupLabelKey: sgName,
 		}
 		pvc = &v1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: pvcName, Namespace: namespace,
 			Labels: pvcLabels}}
@@ -91,6 +93,7 @@ func Test_getPersistentVolumeClaimLabels(t *testing.T) {
 	assert.Equal(t, labels[k8s.AppLabelKey], appName)
 	assert.Equal(t, labels[k8s.AppLabelShortKey], appName)
 	assert.Equal(t, labels[k8s.ReleaseLabelKey], releaseName)
+	assert.Equal(t, labels[apiV1.StorageGroupLabelKey], sgName)
 }
 
 func TestVolumeOperationsImpl_CreateVolume_VolumeExists(t *testing.T) {

pkg/crcontrollers/reservation/reservationcontroller.go

@@ -112,7 +112,7 @@ func (c *Controller) handleReservationUpdate(ctx context.Context, log *logrus.En
 		volumes := make([]*v1api.Volume, len(reservationSpec.ReservationRequests))
 		for i, request := range reservationSpec.ReservationRequests {
 			capacity := request.CapacityRequest
-			volumes[i] = &v1api.Volume{Id: capacity.Name, Size: capacity.Size, StorageClass: capacity.StorageClass}
+			volumes[i] = &v1api.Volume{Id: capacity.Name, Size: capacity.Size, StorageClass: capacity.StorageClass, StorageGroup: capacity.StorageGroup}
 		}
 
 		// TODO: do not read all ACs and ACRs for each request: https://github.com/dell/csi-baremetal/issues/89