From e2c89adfe18610c757a427f31373c4a599fe9411 Mon Sep 17 00:00:00 2001 From: Sunny Date: Sun, 20 Jan 2019 04:29:45 +0530 Subject: [PATCH] hostpath: Add block volume support This change adds block volume support to hostpath driver. When a block volume request is received, a block file is created at provisionRoot with the requested capacity as size. At node publish, a loop device is created associated with the block file. This loop device is then symlinked to the publish target path. For read only volume publish, the loop device is created with read only flag. At node unpublish, the symlinked file is deleted, and the loop device is detached from the block file. At volume delete, the block file is deleted. --- pkg/hostpath/controllerserver.go | 53 ++++++++++-- pkg/hostpath/hostpath.go | 10 ++- pkg/hostpath/nodeserver.go | 144 ++++++++++++++++++++++++------- 3 files changed, 165 insertions(+), 42 deletions(-) diff --git a/pkg/hostpath/controllerserver.go b/pkg/hostpath/controllerserver.go index 576cb1e24..a49887b8c 100644 --- a/pkg/hostpath/controllerserver.go +++ b/pkg/hostpath/controllerserver.go @@ -43,6 +43,13 @@ const ( maxStorageCapacity = tib ) +type accessType int + +const ( + mountAccess accessType = iota + blockAccess +) + type controllerServer struct { *csicommon.DefaultControllerServer } @@ -61,9 +68,16 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol if caps == nil { return nil, status.Error(codes.InvalidArgument, "Volume Capabilities missing in request") } + + // Keep a record of the requested access types. + var accessTypeMount, accessTypeBlock bool + for _, cap := range caps { if cap.GetBlock() != nil { - return nil, status.Error(codes.Unimplemented, "Block Volume not supported") + accessTypeBlock = true + } + if cap.GetMount() != nil { + accessTypeMount = true } } // A real driver would also need to check that the other @@ -72,6 +86,19 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol // volmode)] volumeMode should fail in binding dynamic // provisioned PV to PVC" storage E2E test. + if accessTypeBlock && accessTypeMount { + return nil, status.Error(codes.InvalidArgument, "cannot have both block and mount access type") + } + + var requestedAccessType accessType + + if accessTypeBlock { + requestedAccessType = blockAccess + } else { + // Default to mount. + requestedAccessType = mountAccess + } + // Need to check for already existing volume name, and if found // check for the requested capacity and already allocated capacity if exVol, err := getVolumeByName(req.GetName()); err == nil { @@ -98,11 +125,26 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } volumeID := uuid.NewUUID().String() path := provisionRoot + volumeID - err := os.MkdirAll(path, 0777) - if err != nil { - glog.V(3).Infof("failed to create volume: %v", err) - return nil, err + + switch requestedAccessType { + case blockAccess: + executor := utilexec.New() + of := fmt.Sprintf("%s=%s", "of", path) + count := fmt.Sprintf("%s=%d", "count", capacity/mib) + // Create a block file. + out, err := executor.Command("dd", "if=/dev/zero", of, "bs=1M", count).CombinedOutput() + if err != nil { + glog.V(3).Infof("failed to create block device: %v", string(out)) + return nil, err + } + case mountAccess: + err := os.MkdirAll(path, 0777) + if err != nil { + glog.V(3).Infof("failed to create volume: %v", err) + return nil, err + } } + if req.GetVolumeContentSource() != nil { contentSource := req.GetVolumeContentSource() if contentSource.GetSnapshot() != nil { @@ -129,6 +171,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol hostPathVol.VolID = volumeID hostPathVol.VolSize = capacity hostPathVol.VolPath = path + hostPathVol.VolAccessType = requestedAccessType hostPathVolumes[volumeID] = hostPathVol return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ diff --git a/pkg/hostpath/hostpath.go b/pkg/hostpath/hostpath.go index 68cad52ba..ec4959ab5 100644 --- a/pkg/hostpath/hostpath.go +++ b/pkg/hostpath/hostpath.go @@ -47,10 +47,12 @@ type hostPath struct { } type hostPathVolume struct { - VolName string `json:"volName"` - VolID string `json:"volID"` - VolSize int64 `json:"volSize"` - VolPath string `json:"volPath"` + VolName string `json:"volName"` + VolID string `json:"volID"` + VolSize int64 `json:"volSize"` + VolPath string `json:"volPath"` + VolAccessType accessType `json:"volAccessType"` + VolLoopDevice string `json:"volLoopDevice"` } type hostPathSnapshot struct { diff --git a/pkg/hostpath/nodeserver.go b/pkg/hostpath/nodeserver.go index 5e577d446..29f8b4f14 100644 --- a/pkg/hostpath/nodeserver.go +++ b/pkg/hostpath/nodeserver.go @@ -17,7 +17,9 @@ limitations under the License. package hostpath import ( + "fmt" "os" + "strings" "github.com/golang/glog" "golang.org/x/net/context" @@ -26,6 +28,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/kubernetes/pkg/util/mount" + utilexec "k8s.io/utils/exec" "github.com/kubernetes-csi/drivers/pkg/csi-common" ) @@ -48,45 +51,97 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } targetPath := req.GetTargetPath() - notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath) + + if req.GetVolumeCapability().GetBlock() != nil && + req.GetVolumeCapability().GetMount() != nil { + return nil, status.Error(codes.InvalidArgument, "cannot have both block and mount access type") + } + + vol, err := getVolumeByID(req.GetVolumeId()) if err != nil { - if os.IsNotExist(err) { - if err = os.MkdirAll(targetPath, 0750); err != nil { + return nil, status.Error(codes.NotFound, err.Error()) + } + + if req.GetVolumeCapability().GetBlock() != nil { + if vol.VolAccessType != blockAccess { + return nil, status.Error(codes.InvalidArgument, "cannot publish a non-block volume as block volume") + } + + executor := utilexec.New() + // Get a free loop device. + out, err := executor.Command("losetup", "-f").CombinedOutput() + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("failed to get a free loop device: %v: %s", err, out)) + } + loopDevice := strings.TrimSpace(string(out)) + + losetupArgs := []string{} + if req.GetReadonly() { + losetupArgs = append(losetupArgs, "-r") + } + // Append loop device and file path. + losetupArgs = append(losetupArgs, loopDevice, vol.VolPath) + + // Associate block file with the loop device. + out, err = executor.Command("losetup", losetupArgs...).CombinedOutput() + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("failed to associate loop device with block file: %v: %s", err, out)) + } + + // Create symlink to the target path. + out, err = executor.Command("ln", "-s", loopDevice, targetPath).CombinedOutput() + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("failed to create symlink to target path: %v: %s", err, out)) + } + + // Update the volume info. + vol.VolLoopDevice = loopDevice + hostPathVolumes[vol.VolID] = vol + } else if req.GetVolumeCapability().GetMount() != nil { + if vol.VolAccessType != mountAccess { + return nil, status.Error(codes.InvalidArgument, "cannot publish a non-mount volume as mount volume") + } + + notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath) + if err != nil { + if os.IsNotExist(err) { + if err = os.MkdirAll(targetPath, 0750); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + notMnt = true + } else { return nil, status.Error(codes.Internal, err.Error()) } - notMnt = true - } else { - return nil, status.Error(codes.Internal, err.Error()) } - } - if !notMnt { - return &csi.NodePublishVolumeResponse{}, nil - } + if !notMnt { + return &csi.NodePublishVolumeResponse{}, nil + } - fsType := req.GetVolumeCapability().GetMount().GetFsType() + fsType := req.GetVolumeCapability().GetMount().GetFsType() - deviceId := "" - if req.GetPublishContext() != nil { - deviceId = req.GetPublishContext()[deviceID] - } + deviceId := "" + if req.GetPublishContext() != nil { + deviceId = req.GetPublishContext()[deviceID] + } - readOnly := req.GetReadonly() - volumeId := req.GetVolumeId() - attrib := req.GetVolumeContext() - mountFlags := req.GetVolumeCapability().GetMount().GetMountFlags() + readOnly := req.GetReadonly() + volumeId := req.GetVolumeId() + attrib := req.GetVolumeContext() + mountFlags := req.GetVolumeCapability().GetMount().GetMountFlags() - glog.V(4).Infof("target %v\nfstype %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n", - targetPath, fsType, deviceId, readOnly, volumeId, attrib, mountFlags) + glog.V(4).Infof("target %v\nfstype %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n", + targetPath, fsType, deviceId, readOnly, volumeId, attrib, mountFlags) - options := []string{"bind"} - if readOnly { - options = append(options, "ro") - } - mounter := mount.New("") - path := provisionRoot + volumeId - if err := mounter.Mount(path, targetPath, "", options); err != nil { - return nil, err + options := []string{"bind"} + if readOnly { + options = append(options, "ro") + } + mounter := mount.New("") + path := provisionRoot + volumeId + if err := mounter.Mount(path, targetPath, "", options); err != nil { + return nil, err + } } return &csi.NodePublishVolumeResponse{}, nil @@ -104,12 +159,35 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu targetPath := req.GetTargetPath() volumeID := req.GetVolumeId() - // Unmounting the image - err := mount.New("").Unmount(req.GetTargetPath()) + vol, err := getVolumeByID(volumeID) if err != nil { - return nil, status.Error(codes.Internal, err.Error()) + return nil, status.Error(codes.NotFound, err.Error()) + } + + switch vol.VolAccessType { + case blockAccess: + // Remove the symlink. + os.RemoveAll(targetPath) + + // Disassociate the loop device. + executor := utilexec.New() + out, err := executor.Command("losetup", "-d", vol.VolLoopDevice).CombinedOutput() + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("failed to disassociate loop device: %v: %s", err, out)) + } + + // Update the volume info. + vol.VolLoopDevice = "" + hostPathVolumes[vol.VolID] = vol + glog.V(4).Infof("hostpath: volume %s has been unpublished.", targetPath) + case mountAccess: + // Unmounting the image + err = mount.New("").Unmount(req.GetTargetPath()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + glog.V(4).Infof("hostpath: volume %s/%s has been unmounted.", targetPath, volumeID) } - glog.V(4).Infof("hostpath: volume %s/%s has been unmounted.", targetPath, volumeID) return &csi.NodeUnpublishVolumeResponse{}, nil }