[ISSUE-1018] Custom Storage Group Feature

Makefile

@@ -112,6 +112,7 @@ generate-deepcopy:
  controller-gen object paths=api/v1/drivecrd/drive_types.go paths=api/v1/drivecrd/groupversion_info.go output:dir=api/v1/drivecrd
  controller-gen object paths=api/v1/lvgcrd/logicalvolumegroup_types.go paths=api/v1/lvgcrd/groupversion_info.go output:dir=api/v1/lvgcrd
  controller-gen object paths=api/v1/nodecrd/node_types.go paths=api/v1/nodecrd/groupversion_info.go output:dir=api/v1/nodecrd
+ controller-gen object paths=api/v1/storagegroupcrd/storagegroup_types.go paths=api/v1/storagegroupcrd/groupversion_info.go output:dir=api/v1/storagegroupcrd
 
 generate-baremetal-crds: install-controller-gen
  controller-gen $(CRD_OPTIONS) paths=api/v1/availablecapacitycrd/availablecapacity_types.go paths=api/v1/availablecapacitycrd/groupversion_info.go output:crd:dir=$(CSI_CHART_CRDS_PATH)
@@ -120,6 +121,7 @@ generate-baremetal-crds: install-controller-gen
  controller-gen $(CRD_OPTIONS) paths=api/v1/drivecrd/drive_types.go paths=api/v1/drivecrd/groupversion_info.go output:crd:dir=$(CSI_CHART_CRDS_PATH)
  controller-gen $(CRD_OPTIONS) paths=api/v1/lvgcrd/logicalvolumegroup_types.go paths=api/v1/lvgcrd/groupversion_info.go output:crd:dir=$(CSI_CHART_CRDS_PATH)
  controller-gen $(CRD_OPTIONS) paths=api/v1/nodecrd/node_types.go paths=api/v1/nodecrd/groupversion_info.go output:crd:dir=$(CSI_CHART_CRDS_PATH)
+ controller-gen $(CRD_OPTIONS) paths=api/v1/storagegroupcrd/storagegroup_types.go paths=api/v1/storagegroupcrd/groupversion_info.go output:crd:dir=$(CSI_CHART_CRDS_PATH)
 
 generate-api: compile-proto generate-baremetal-crds generate-deepcopy
 

api/v1/constants.go

@@ -138,4 +138,7 @@ const (
  // CSI Drive taint-like label key and value
  DriveTaintKey = "drive.csi-baremetal.dell.com/taint"
  DriveTaintValue = "NoSchedule"
+
+ // CSI StorageGroup label key
+ StorageGroupLabelKey = "drive.csi-baremetal.dell.com/storage-group"
 )

api/v1/storagegroupcrd/groupversion_info.go

@@ -0,0 +1,38 @@
+/*
+Copyright © 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package sgcrd contains API Schema definitions for the StorageGroup v1 API group
+// +groupName=csi-baremetal.dell.com
+// +versionName=v1
+package sgcrd
+
+import (
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ crScheme "sigs.k8s.io/controller-runtime/pkg/scheme"
+
+ v1 "github.com/dell/csi-baremetal/api/v1"
+)
+
+var (
+ // GroupVersionStorageGroup is group version used to register these objects
+ GroupVersionStorageGroup = schema.GroupVersion{Group: v1.CSICRsGroupVersion, Version: v1.Version}
+
+ // SchemeBuilderStorageGroup is used to add go types to the GroupVersionKind scheme
+ SchemeBuilderStorageGroup = &crScheme.Builder{GroupVersion: GroupVersionStorageGroup}
+
+ // AddToSchemeStorageGroup adds the types in this group-version to the given scheme.
+ AddToSchemeStorageGroup = SchemeBuilderStorageGroup.AddToScheme
+)

api/v1/storagegroupcrd/storagegroup_types.go

@@ -0,0 +1,58 @@
+/*
+Copyright © 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sgcrd
+
+import (
+ api "github.com/dell/csi-baremetal/api/generated/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// +kubebuilder:object:root=true
+
+// StorageGroup is the Schema for the StorageGroups API
+// +kubebuilder:resource:scope=Cluster,shortName={sg,sgs}
+// +kubebuilder:printcolumn:name="DRIVES_PER_NODE",type="string",JSONPath=".spec.driveSelector.numberDrivesPerNode",description="numberDrivesPerNode of StorageGroup's DriveSelector"
+// +kubebuilder:printcolumn:name="TYPE",type="string",JSONPath=".spec.driveSelector.matchFields.Type",description="Drive Type of StorageGroup's DriveSelector"
+// +kubebuilder:printcolumn:name="SLOT",type="string",JSONPath=".spec.driveSelector.matchFields.Slot",description="Drive Slot of StorageGroup's DriveSelector"
+// +kubebuilder:printcolumn:name="PATH",type="string",JSONPath=".spec.driveSelector.matchFields.Path",description="Drive Path of StorageGroup's DriveSelector"
+// +kubebuilder:printcolumn:name="SYSTEM",type="string",JSONPath=".spec.driveSelector.matchFields.IsSystem",description="Whether StorageGroup's DriveSelector to Select System Drive"
+type StorageGroup struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ObjectMeta `json:"metadata,omitempty"`
+ Spec api.StorageGroupSpec `json:"spec,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// StorageGroupList contains a list of StorageGroup
+//+kubebuilder:object:generate=true
+type StorageGroupList struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ListMeta `json:"metadata,omitempty"`
+ Items []StorageGroup `json:"items"`
+}
+
+func init() {
+ SchemeBuilderStorageGroup.Register(&StorageGroup{}, &StorageGroupList{})
+}
+
+func (in *StorageGroup) DeepCopyInto(out *StorageGroup) {
+ *out = *in
+ out.TypeMeta = in.TypeMeta
+ in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+ out.Spec = in.Spec
+}

api/v1/types.proto

@@ -46,6 +46,7 @@ message Volume {
  string Usage = 13;
  // inline volumes are not support anymore. need to remove field in the next version
  bool Ephemeral = 14;
+ string StorageGroup = 15;
 }
 
 message AvailableCapacity {
@@ -80,6 +81,7 @@ message CapacityRequest {
  string Name = 1;
  string StorageClass = 2;
  int64 Size = 3;
+ string StorageGroup = 4;
 }
 
 message LogicalVolumeGroup {
@@ -97,3 +99,12 @@ message Node {
  // key - address type, value - address, align with NodeAddress struct from k8s.io/api/core/v1
  map<string, string> Addresses = 2;
 }
+
+message StorageGroupSpec {
+ DriveSelector driveSelector = 1;
+}
+
+message DriveSelector {
+ int32 numberDrivesPerNode = 1;
+ map<string, string> matchFields = 2;
+}

cmd/controller/main.go

@@ -40,6 +40,7 @@ import (
  accrd "github.com/dell/csi-baremetal/api/v1/availablecapacitycrd"
  "github.com/dell/csi-baremetal/api/v1/drivecrd"
  "github.com/dell/csi-baremetal/api/v1/lvgcrd"
+ sgcrd "github.com/dell/csi-baremetal/api/v1/storagegroupcrd"
  "github.com/dell/csi-baremetal/api/v1/volumecrd"
  "github.com/dell/csi-baremetal/pkg/base"
  "github.com/dell/csi-baremetal/pkg/base/featureconfig"
@@ -51,6 +52,7 @@ import (
  "github.com/dell/csi-baremetal/pkg/controller"
  "github.com/dell/csi-baremetal/pkg/controller/capacitycontroller"
  "github.com/dell/csi-baremetal/pkg/crcontrollers/reservation"
+ "github.com/dell/csi-baremetal/pkg/crcontrollers/storagegroup"
  "github.com/dell/csi-baremetal/pkg/metrics"
 )
 
@@ -168,6 +170,10 @@ func createManager(ctx context.Context, client *k8s.KubeClient, log *logrus.Logg
  return nil, err
  }
 
+ if err := sgcrd.AddToSchemeStorageGroup(scheme); err != nil {
+ return nil, err
+ }
+
  mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
  Scheme: scheme,
  Namespace: *namespace,
@@ -196,5 +202,11 @@ func createManager(ctx context.Context, client *k8s.KubeClient, log *logrus.Logg
  if err = capacityController.SetupWithManager(mgr); err != nil {
  return nil, err
  }
+
+ storageGroupController := storagegroup.NewController(client, kubeCache, log)
+ if err = storageGroupController.SetupWithManager(mgr); err != nil {
+ return nil, err
+ }
+
  return mgr, nil
 }

devkit/Dockerfile

@@ -10,7 +10,7 @@ RUN apt update \
 && /kind/kind-build.sh /kind
 
 
-FROM opensuse/leap:latest
+FROM opensuse/leap:15.4
 
 ARG arg_docker_ver
 ARG arg_go_ver

pkg/base/capacityplanner/node_capacity.go

@@ -154,7 +154,7 @@ func (nc *nodeCapacity) selectACForVolume(vol *genV1.Volume) *accrd.AvailableCap
  }
 
  for _, ac := range nc.acsOrder[vol.StorageClass] {
- if requiredSize <= nc.acs[ac].Spec.Size {
+ if requiredSize <= nc.acs[ac].Spec.Size && nc.acs[ac].Labels[v1.StorageGroupLabelKey] == vol.StorageGroup {
  // check if AC is reserved
  reservation, ok := nc.reservedACs[ac]
 

pkg/base/k8s/kubeclient.go

@@ -39,6 +39,7 @@
  "github.com/dell/csi-baremetal/api/v1/drivecrd"
  "github.com/dell/csi-baremetal/api/v1/lvgcrd"
  "github.com/dell/csi-baremetal/api/v1/nodecrd"
+ sgcrd "github.com/dell/csi-baremetal/api/v1/storagegroupcrd"
  "github.com/dell/csi-baremetal/api/v1/volumecrd"
  "github.com/dell/csi-baremetal/pkg/base"
  checkErr "github.com/dell/csi-baremetal/pkg/base/error"
@@ -227,15 +228,15 @@
 // ConstructLVGCR constructs LogicalVolumeGroup custom resource from api.LogicalVolumeGroup struct
 // Receives a name for k8s ObjectMeta and an instance of api.LogicalVolumeGroup struct
 // Returns an instance of LogicalVolumeGroup CR struct
-func (k *KubeClient) ConstructLVGCR(name string, apiLVG api.LogicalVolumeGroup) *lvgcrd.LogicalVolumeGroup {
+func (k *KubeClient) ConstructLVGCR(name, storageGroup string, apiLVG api.LogicalVolumeGroup) *lvgcrd.LogicalVolumeGroup {
  return &lvgcrd.LogicalVolumeGroup{
  TypeMeta: apisV1.TypeMeta{
  Kind: crdV1.LVGKind,
  APIVersion: crdV1.APIV1Version,
  },
  ObjectMeta: apisV1.ObjectMeta{
  Name: name,
- Labels: constructDefaultAppMap(),
+ Labels: constructLVGCRLabels(storageGroup),
  },
  Spec: apiLVG,
  }
@@ -409,6 +410,12 @@
  return nil, err
  }
 
+ // register csi storagegroup crd
+ err := sgcrd.AddToSchemeStorageGroup(scheme)
+ if err != nil {
+ return nil, err
+ }
+
  return scheme, nil
 }
 
@@ -420,3 +427,11 @@
  }
  return
 }
+
+func constructLVGCRLabels(storageGroup string) (labels map[string]string) {
+ labels = constructDefaultAppMap()
+ if storageGroup != "" {
+ labels[crdV1.StorageGroupLabelKey] = storageGroup
+ }
+ return labels
+}

pkg/base/k8s/kubeclient_test.go

@@ -45,6 +45,7 @@ const (
  testID = "someID"
  testNode1Name = "node1"
  testDriveLocation1 = "drive"
+ testStorageGroup = "test-group"
 )
 
 var (
@@ -496,7 +497,7 @@ var _ = Describe("Constructor methods", func() {
  })
  Context("ConstructLVGCR", func() {
  It("Should return right LogicalVolumeGroup CR", func() {
- constructedCR := k8sclient.ConstructLVGCR(testLVGName, testApiLVG)
+ constructedCR := k8sclient.ConstructLVGCR(testLVGName, "", testApiLVG)
  Expect(constructedCR.TypeMeta.Kind).To(Equal(testLVGCR.TypeMeta.Kind))
  Expect(constructedCR.TypeMeta.APIVersion).To(Equal(testLVGCR.TypeMeta.APIVersion))
  Expect(constructedCR.ObjectMeta.Name).To(Equal(testLVGCR.ObjectMeta.Name))
@@ -505,6 +506,18 @@ var _ = Describe("Constructor methods", func() {
  Expect(constructedCR.Labels).To(Equal(constructDefaultAppMap()))
  })
  })
+ Context("ConstructLVGCR with storage group", func() {
+ It("Should return right LogicalVolumeGroup CR with storage group", func() {
+ constructedCR := k8sclient.ConstructLVGCR(testLVGName, testStorageGroup, testApiLVG)
+ Expect(constructedCR.TypeMeta.Kind).To(Equal(testLVGCR.TypeMeta.Kind))
+ Expect(constructedCR.TypeMeta.APIVersion).To(Equal(testLVGCR.TypeMeta.APIVersion))
+ Expect(constructedCR.ObjectMeta.Name).To(Equal(testLVGCR.ObjectMeta.Name))
+ Expect(constructedCR.ObjectMeta.Namespace).To(Equal(testLVGCR.ObjectMeta.Namespace))
+ Expect(constructedCR.Labels[apiV1.StorageGroupLabelKey]).To(Equal(testStorageGroup))
+ Expect(constructedCR.Spec).To(Equal(testLVGCR.Spec))
+ Expect(constructedCR.Labels).To(Equal(constructLVGCRLabels(testStorageGroup)))
+ })
+ })
 })
 
 // remove all crds (volume and ac)

pkg/common/ac_operations.go

@@ -33,7 +33,7 @@
 
 // AvailableCapacityOperations is the interface for interact with AvailableCapacity CRs from Controller
 type AvailableCapacityOperations interface {
- RecreateACToLVGSC(ctx context.Context, sc string, acs ...accrd.AvailableCapacity) *accrd.AvailableCapacity
+ RecreateACToLVGSC(ctx context.Context, sc, sg string, acs ...accrd.AvailableCapacity) *accrd.AvailableCapacity
 }
 
 // ACOperationsImpl is the basic implementation of AvailableCapacityOperations interface
@@ -56,7 +56,7 @@
 // Concerts first AC to LVG SC and set size of remaining to 0
 // Receives newSC as string (e.g. HDDLVG) and AvailableCapacities where LVG should be based
 // Returns created AC or nil
-func (a *ACOperationsImpl) RecreateACToLVGSC(ctx context.Context, newSC string,
+func (a *ACOperationsImpl) RecreateACToLVGSC(ctx context.Context, newSC, storageGroup string,
  acs ...accrd.AvailableCapacity) *accrd.AvailableCapacity {
  ll := a.log.WithFields(logrus.Fields{
  "method": "RecreateACToLVGSC",
@@ -90,7 +90,7 @@
  )
 
  // create LVG CR based on ACs
- lvg := a.k8sClient.ConstructLVGCR(name, apiLVG)
+ lvg := a.k8sClient.ConstructLVGCR(name, storageGroup, apiLVG)
  if err = a.k8sClient.CreateCR(ctx, name, lvg); err != nil {
  ll.Errorf("Unable to create LVG CR: %v", err)
  return nil