diff --git a/local-volume/provisioner/deployment/kubernetes/provisioner-daemonset.yaml b/local-volume/provisioner/deployment/kubernetes/provisioner-daemonset.yaml index f6e2b8057ec..d10b4aff806 100644 --- a/local-volume/provisioner/deployment/kubernetes/provisioner-daemonset.yaml +++ b/local-volume/provisioner/deployment/kubernetes/provisioner-daemonset.yaml @@ -12,6 +12,8 @@ spec: - name: provisioner image: "gcr.io/msau-k8s-dev/local-volume-provisioner:latest" imagePullPolicy: Always + securityContext: + privileged: true volumeMounts: - name: discovery-vol mountPath: "/local-disks" diff --git a/local-volume/provisioner/pkg/common/common.go b/local-volume/provisioner/pkg/common/common.go index 6867abd5b72..f4e2827d340 100644 --- a/local-volume/provisioner/pkg/common/common.go +++ b/local-volume/provisioner/pkg/common/common.go @@ -65,6 +65,7 @@ type RuntimeConfig struct { type LocalPVConfig struct { Name string HostPath string + Capacity uint64 StorageClass string ProvisionerName string AffinityAnn string @@ -83,8 +84,7 @@ func CreateLocalPVSpec(config *LocalPVConfig) *v1.PersistentVolume { Spec: v1.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimDelete, Capacity: v1.ResourceList{ - // TODO: detect capacity - v1.ResourceName(v1.ResourceStorage): resource.MustParse("10Gi"), + v1.ResourceName(v1.ResourceStorage): *resource.NewQuantity(int64(config.Capacity), resource.BinarySI), }, PersistentVolumeSource: v1.PersistentVolumeSource{ Local: &v1.LocalVolumeSource{ diff --git a/local-volume/provisioner/pkg/discovery/discovery.go b/local-volume/provisioner/pkg/discovery/discovery.go index 7a6a49ee51c..4f9d1fd2ef3 100644 --- a/local-volume/provisioner/pkg/discovery/discovery.go +++ b/local-volume/provisioner/pkg/discovery/discovery.go @@ -105,8 +105,13 @@ func (d *Discoverer) discoverVolumesAtPath(class, relativePath string) { glog.Errorf("Mount path %q validation failed: %v", filePath, err) continue } - // TODO: detect capacity - d.createPV(file, relativePath, class) + + availByte, err := d.VolUtil.GetFsAvailableByte(filePath) + if err != nil { + glog.Errorf("Path %q fs stats error: %v", filePath, err) + continue + } + d.createPV(file, relativePath, class, availByte) } } } @@ -131,14 +136,15 @@ func generatePVName(file, node, class string) string { return fmt.Sprintf("local-pv-%x", h.Sum32()) } -func (d *Discoverer) createPV(file, relativePath, class string) { +func (d *Discoverer) createPV(file, relativePath, class string, availByte uint64) { pvName := generatePVName(file, d.Node.Name, class) outsidePath := filepath.Join(d.HostDir, relativePath, file) - glog.Infof("Found new volume at host path %q, creating Local PV %q", outsidePath, pvName) + glog.Infof("Found new volume at host path %q with capacity %d, creating Local PV %q", outsidePath, availByte, pvName) pvSpec := common.CreateLocalPVSpec(&common.LocalPVConfig{ Name: pvName, HostPath: outsidePath, + Capacity: availByte, StorageClass: class, ProvisionerName: d.Name, AffinityAnn: d.nodeAffinityAnn, diff --git a/local-volume/provisioner/pkg/discovery/discovery_test.go b/local-volume/provisioner/pkg/discovery/discovery_test.go index 795c09f7cf3..50013b3f45c 100644 --- a/local-volume/provisioner/pkg/discovery/discovery_test.go +++ b/local-volume/provisioner/pkg/discovery/discovery_test.go @@ -69,8 +69,8 @@ type testConfig struct { func TestDiscoverVolumes_Basic(t *testing.T) { vols := map[string][]*util.FakeFile{ "dir1": { - {Name: "mount1", Hash: 0xaaaafef5}, - {Name: "mount2", Hash: 0x79412c38}, + {Name: "mount1", Hash: 0xaaaafef5, Capacity: 100 * 1024}, + {Name: "mount2", Hash: 0x79412c38, Capacity: 100 * 1024 * 1024}, }, "dir2": { {Name: "mount1", Hash: 0xa7aafa3c}, @@ -311,13 +311,20 @@ func verifyProvisionerName(t *testing.T, pv *v1.PersistentVolume) { } } +// testPVInfo contains all the fields we are intested in validating. +type testPVInfo struct { + pvName string + path string + capacity uint64 +} + func verifyCreatedPVs(t *testing.T, test *testConfig) { - expectedPVs := map[string]string{} + expectedPVs := map[string]*testPVInfo{} for dir, files := range test.expectedVolumes { for _, file := range files { pvName := fmt.Sprintf("local-pv-%x", file.Hash) path := filepath.Join(testHostDir, dir, file.Name) - expectedPVs[pvName] = path + expectedPVs[pvName] = &testPVInfo{pvName: pvName, path: path, capacity: file.Capacity} } } @@ -328,21 +335,32 @@ func verifyCreatedPVs(t *testing.T, test *testConfig) { t.Errorf("Expected %v created PVs, got %v", expectedLen, actualLen) } - for pvName, pv := range createdPVs { - expectedPath, found := expectedPVs[pvName] + for pvName, createdPV := range createdPVs { + expectedPV, found := expectedPVs[pvName] if !found { t.Errorf("Did not expect created PVs %v", pvName) } - if pv.Spec.PersistentVolumeSource.Local.Path != expectedPath { - t.Errorf("Expected path %q, got %q", expectedPath, expectedPath) + if createdPV.Spec.PersistentVolumeSource.Local.Path != expectedPV.path { + t.Errorf("Expected path %q, got %q", expectedPV.path, createdPV.Spec.PersistentVolumeSource.Local.Path) } _, exists := test.cache.GetPV(pvName) if !exists { t.Errorf("PV %q not in cache", pvName) } + capacity, ok := createdPV.Spec.Capacity[v1.ResourceStorage] + if !ok { + t.Errorf("Unexpected empty resource storage") + } + capacityInt, ok := capacity.AsInt64() + if !ok { + t.Errorf("Unable to convert resource storage into int64") + } + if uint64(capacityInt) != expectedPV.capacity { + t.Errorf("Expected capacity %d, got %d", expectedPV.capacity, capacityInt) + } // TODO: verify storage class - verifyProvisionerName(t, pv) - verifyNodeAffinity(t, pv) + verifyProvisionerName(t, createdPV) + verifyNodeAffinity(t, createdPV) } } diff --git a/local-volume/provisioner/pkg/util/volume_util.go b/local-volume/provisioner/pkg/util/volume_util.go index ecb2e4159df..e85c43d8844 100644 --- a/local-volume/provisioner/pkg/util/volume_util.go +++ b/local-volume/provisioner/pkg/util/volume_util.go @@ -21,6 +21,8 @@ import ( "os" "path/filepath" + "golang.org/x/sys/unix" + "github.com/golang/glog" ) @@ -34,6 +36,9 @@ type VolumeUtil interface { // Delete all the contents under the given path, but not the path itself DeleteContents(fullPath string) error + + // Get available capacity for fs on full path + GetFsAvailableByte(fullPath string) (uint64, error) } var _ VolumeUtil = &volumeUtil{} @@ -99,6 +104,18 @@ func (u *volumeUtil) DeleteContents(fullPath string) error { return nil } +// GetFsAvailableByte returns available capacity in byte about a mounted filesystem. +// fullPath is the pathname of any file within the mounted filesystem. Capacity +// returned here is total capacity available to non-root users, and does not include +// fs reserved space. +func (u *volumeUtil) GetFsAvailableByte(fullPath string) (uint64, error) { + var s unix.Statfs_t + if err := unix.Statfs(fullPath, &s); err != nil { + return 0, err + } + return uint64(s.Frsize) * (s.Blocks - s.Bfree + s.Bavail), nil +} + var _ VolumeUtil = &FakeVolumeUtil{} // FakeVolumeUtil is a stub interface for unit testing @@ -114,7 +131,8 @@ type FakeFile struct { Name string IsNotDir bool // Expected hash value of the PV name - Hash uint32 + Hash uint32 + Capacity uint64 } // NewFakeVolumeUtil returns a VolumeUtil object for use in unit testing @@ -163,6 +181,22 @@ func (u *FakeVolumeUtil) DeleteContents(fullPath string) error { return nil } +func (u *FakeVolumeUtil) GetFsAvailableByte(fullPath string) (uint64, error) { + dir, file := filepath.Split(fullPath) + dir = filepath.Clean(dir) + files, found := u.directoryFiles[dir] + if !found { + return 0, fmt.Errorf("Directory %q not found", dir) + } + + for _, f := range files { + if file == f.Name { + return f.Capacity, nil + } + } + return 0, fmt.Errorf("File %q not found", fullPath) +} + // AddNewFiles adds the given files to the current directory listing // This is only for testing func (u *FakeVolumeUtil) AddNewFiles(mountDir string, dirFiles map[string][]*FakeFile) {