Skip to content
This repository has been archived by the owner on Nov 9, 2020. It is now read-only.

Commit

Permalink
Addressing comments from Mark and Prashant
Browse files Browse the repository at this point in the history
  • Loading branch information
pshahzeb committed May 8, 2017
1 parent be205c1 commit e88907c
Show file tree
Hide file tree
Showing 5 changed files with 97 additions and 79 deletions.
4 changes: 3 additions & 1 deletion misc/scripts/refcnt_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ function check_files {
}

function check_recovery_record {
# restart generates some log.
# log contains refcounting attempts and after success logs summary.
line=`tail -50 /var/log/docker-volume-vsphere.log | $GREP 'Volume name=' | $GREP 'mounted=true'`
expected="count=$count mounted=true"

Expand All @@ -82,6 +82,7 @@ function check_recovery_record {
function test_crash_recovery {
timeout=$1
echo "Checking recovery through docker kill"
# kill docker daemon forcefully
pkill -9 dockerd
until pids=$(pidof dockerd)
do
Expand Down Expand Up @@ -116,6 +117,7 @@ fi
echo "$(docker volume ls)"
for i in `seq 1 $count`
do
# run containers with restart flag so they restart after docker restart
$DOCKER run -d --restart=always -v $vname:/v busybox sh -c "touch /v/file$i; sync ; \
while true; do sleep $timeout; done"
done
Expand Down
50 changes: 25 additions & 25 deletions vmdk_plugin/drivers/photon/photon_driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,16 +55,15 @@ const (
fsTypeTag = "Fs_Type"
)

var fullVolumeNameMap map[string]string // save full volume names to avoid trip to vmdkops service

// VolumeDriver - Photon volume driver struct
type VolumeDriver struct {
client *photon.Client
hostID string
mountRoot string
project string
refCounts *refcount.RefCountsMap
target string
client *photon.Client
hostID string
mountRoot string
project string
refCounts *refcount.RefCountsMap
target string
mountIDtoName map[string]string // map of mountID -> full volume name
}

func (d *VolumeDriver) verifyTarget() error {
Expand All @@ -82,8 +81,6 @@ func (d *VolumeDriver) verifyTarget() error {
// NewVolumeDriver - creates Driver, creates client for given target
func NewVolumeDriver(targetURL string, projectID string, hostID string, mountDir string) *VolumeDriver {

fullVolumeNameMap = make(map[string]string)

d := &VolumeDriver{
target: targetURL,
project: projectID,
Expand All @@ -100,6 +97,7 @@ func NewVolumeDriver(targetURL string, projectID string, hostID string, mountDir
d.mountRoot = mountDir
d.refCounts = refcount.NewRefCountsMap()
d.refCounts.Init(d, mountDir, driverName)
d.mountIDtoName = make(map[string]string)

log.WithFields(log.Fields{
"version": version,
Expand Down Expand Up @@ -368,16 +366,16 @@ func (d *VolumeDriver) MountVolume(name string, fstype string, id string, isRead
// private function that does the job of mounting volume in conjunction with refcounting
func (d *VolumeDriver) processMount(r volume.MountRequest) volume.Response {
// get volume metadata
status, err := d.GetVolume(r.Name)
volumeMeta, err := d.GetVolume(r.Name)

if err != nil {
return volume.Response{Err: err.Error()}
}

if plugin_utils.IsFullVolumeName(r.Name) != true {
datastore := plugin_utils.GetDatastore(r.Name, status)
r.Name = plugin_utils.GetFullVolumeName(r.Name, datastore)
fullVolumeNameMap[r.ID] = r.Name
r.Name, err = plugin_utils.GetFullVolumeName(r.Name, volumeMeta["datastore"].(string), d)
if err != nil {
log.Errorf("Unable to get full name for volume %s. err:%v", r.Name, err)
return volume.Response{Err: err.Error()}
}

// If the volume is already mounted , just increase the refcount.
Expand All @@ -391,19 +389,19 @@ func (d *VolumeDriver) processMount(r volume.MountRequest) volume.Response {
return volume.Response{Mountpoint: d.getMountPoint(r.Name)}
}

if plugin_utils.CheckAlreadyMounted(r.Name, d.mountRoot) {
if plugin_utils.AlreadyMounted(r.Name, d.mountRoot) {
log.WithFields(log.Fields{"name": r.Name}).Info("Already mounted, skipping mount. ")
return volume.Response{Mountpoint: d.getMountPoint(r.Name)}
}

fstype, exists := status[fsTypeTag]
fstype, exists := volumeMeta[fsTypeTag]
if !exists {
fstype = fs.FstypeDefault
}

skipAttach := false
// If the volume is already attached to the VM, skip the attach.
if state, stateExists := status["State"]; stateExists {
if state, stateExists := volumeMeta["State"]; stateExists {
if strings.Compare(state.(string), "DETACHED") != 0 {
skipAttach = true
}
Expand All @@ -413,7 +411,7 @@ func (d *VolumeDriver) processMount(r volume.MountRequest) volume.Response {
}

// Mount the volume and for now its always read-write.
mountpoint, err := d.MountVolume(r.Name, fstype.(string), status["ID"].(string), false, skipAttach)
mountpoint, err := d.MountVolume(r.Name, fstype.(string), volumeMeta["ID"].(string), false, skipAttach)
if err != nil {
log.WithFields(
log.Fields{"name": r.Name, "error": err.Error()},
Expand Down Expand Up @@ -633,12 +631,14 @@ func (d *VolumeDriver) Unmount(r volume.UnmountRequest) volume.Response {
return volume.Response{Err: ""}
}

if plugin_utils.IsFullVolumeName(r.Name) != true {
fullVolName := fullVolumeNameMap[r.ID]
if fullVolName == "" { // if ID not present in local map, do a get trip
status, _ := d.GetVolume(r.Name)
datastore := plugin_utils.GetDatastore(r.Name, status)
fullVolName = plugin_utils.GetFullVolumeName(r.Name, datastore)
if fullVolName, exist := d.mountIDtoName[r.ID]; exist {
r.Name = fullVolName
delete(d.mountIDtoName, r.ID) //cleanup the map
} else {
fullVolName, err := plugin_utils.GetFullVolumeName(r.Name, "", d)
if err != nil {
log.Errorf("Unable to get full name for volume %s. err:%v", r.Name, err)
return volume.Response{Err: err.Error()}
}
r.Name = fullVolName
}
Expand Down
44 changes: 22 additions & 22 deletions vmdk_plugin/drivers/vmdk/vmdk_driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,23 +47,20 @@ const (

// VolumeDriver - VMDK driver struct
type VolumeDriver struct {
useMockEsx bool
ops vmdkops.VmdkOps
refCounts *refcount.RefCountsMap
useMockEsx bool
ops vmdkops.VmdkOps
refCounts *refcount.RefCountsMap
mountIDtoName map[string]string // map of mountID -> full volume name
}

var (
mountRoot string
fullVolumeNameMap map[string]string // save full volume names to avoid trip to vmdkops service
)
var mountRoot string

// NewVolumeDriver creates Driver which to real ESX (useMockEsx=False) or a mock
func NewVolumeDriver(port int, useMockEsx bool, mountDir string, driverName string) *VolumeDriver {
var d *VolumeDriver

vmdkops.EsxPort = port
mountRoot = mountDir
fullVolumeNameMap = make(map[string]string)

if useMockEsx {
d = &VolumeDriver{
Expand All @@ -83,6 +80,7 @@ func NewVolumeDriver(port int, useMockEsx bool, mountDir string, driverName stri
}
}

d.mountIDtoName = make(map[string]string)
d.refCounts.Init(d, mountDir, driverName)

log.WithFields(log.Fields{
Expand Down Expand Up @@ -218,16 +216,16 @@ func (d *VolumeDriver) UnmountVolume(name string) error {
// private function that does the job of mounting volume in conjunction with refcounting
func (d *VolumeDriver) processMount(r volume.MountRequest) volume.Response {
// get volume metadata
status, err := d.ops.Get(r.Name)
volumeMeta, err := d.ops.Get(r.Name)

if err != nil {
return volume.Response{Err: err.Error()}
}

if plugin_utils.IsFullVolumeName(r.Name) != true {
datastore := plugin_utils.GetDatastore(r.Name, status)
r.Name = plugin_utils.GetFullVolumeName(r.Name, datastore)
fullVolumeNameMap[r.ID] = r.Name
r.Name, err = plugin_utils.GetFullVolumeName(r.Name, volumeMeta["datastore"].(string), d)
if err != nil {
log.Errorf("Unable to get full name for volume %s. err:%v", r.Name, err)
return volume.Response{Err: err.Error()}
}

// If the volume is already mounted , just increase the refcount.
Expand All @@ -241,7 +239,7 @@ func (d *VolumeDriver) processMount(r volume.MountRequest) volume.Response {
return volume.Response{Mountpoint: getMountPoint(r.Name)}
}

if plugin_utils.CheckAlreadyMounted(r.Name, mountRoot) {
if plugin_utils.AlreadyMounted(r.Name, mountRoot) {
log.WithFields(log.Fields{"name": r.Name}).Info("Already mounted, skipping mount. ")
return volume.Response{Mountpoint: getMountPoint(r.Name)}
}
Expand All @@ -253,7 +251,7 @@ func (d *VolumeDriver) processMount(r volume.MountRequest) volume.Response {
return volume.Response{Err: err.Error()}
}
// Check access type.
value, exists := status["access"].(string)
value, exists := volumeMeta["access"].(string)
if !exists {
msg := fmt.Sprintf("Invalid access type for %s, assuming read-write access.", r.Name)
log.WithFields(log.Fields{"name": r.Name, "error": msg}).Error("")
Expand All @@ -263,7 +261,7 @@ func (d *VolumeDriver) processMount(r volume.MountRequest) volume.Response {
}

// Check file system type.
value, exists = status["fstype"].(string)
value, exists = volumeMeta["fstype"].(string)
if !exists {
msg := fmt.Sprintf("Invalid filesystem type for %s, assuming type as %s.",
r.Name, fstype)
Expand Down Expand Up @@ -477,12 +475,14 @@ func (d *VolumeDriver) Unmount(r volume.UnmountRequest) volume.Response {
return volume.Response{Err: ""}
}

if plugin_utils.IsFullVolumeName(r.Name) != true {
fullVolName := fullVolumeNameMap[r.ID]
if fullVolName == "" { // if ID not present in local map, do a get trip
status, _ := d.ops.Get(r.Name)
datastore := plugin_utils.GetDatastore(r.Name, status)
fullVolName = plugin_utils.GetFullVolumeName(r.Name, datastore)
if fullVolName, exist := d.mountIDtoName[r.ID]; exist {
r.Name = fullVolName
delete(d.mountIDtoName, r.ID) //cleanup the map
} else {
fullVolName, err := plugin_utils.GetFullVolumeName(r.Name, "", d)
if err != nil {
log.Errorf("Unable to get full name for volume %s. err:%v", r.Name, err)
return volume.Response{Err: err.Error()}
}
r.Name = fullVolName
}
Expand Down
43 changes: 27 additions & 16 deletions vmdk_plugin/utils/plugin_utils/plugin_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,22 @@ import (
"strings"

log "github.com/Sirupsen/logrus"
"github.com/vmware/docker-volume-vsphere/vmdk_plugin/drivers"
)

const (
// consts for finding and parsing linux mount information
linuxMountsFile = "/proc/mounts"
)

// GetMountInfo - return a map of mounted volumes and devices
func GetMountInfo(mountRoot string) (map[string]string, error) {
volumeMap := make(map[string]string)
volumeMountMap := make(map[string]string) //map [volume mount path] -> device
data, err := ioutil.ReadFile(linuxMountsFile)

if err != nil {
log.Errorf("Can't get info from %s (%v)", linuxMountsFile, err)
return volumeMap, err
return volumeMountMap, err
}

for _, line := range strings.Split(string(data), "\n") {
Expand All @@ -47,13 +49,13 @@ func GetMountInfo(mountRoot string) (map[string]string, error) {
if filepath.Dir(field[1]) != mountRoot {
continue
}
volumeMap[filepath.Base(field[1])] = field[0]
volumeMountMap[filepath.Base(field[1])] = field[0]
}
return volumeMap, nil
return volumeMountMap, nil
}

// CheckAlreadyMounted - check if volume is already mounted on the mountRoot
func CheckAlreadyMounted(name string, mountRoot string) bool {
// AlreadyMounted - check if volume is already mounted on the mountRoot
func AlreadyMounted(name string, mountRoot string) bool {
volumeMap, err := GetMountInfo(mountRoot)

if err != nil {
Expand All @@ -68,18 +70,27 @@ func CheckAlreadyMounted(name string, mountRoot string) bool {

// GetDatastore - get datastore from volume metadata
// Note "datastore" key is defined in vmdkops service
func GetDatastore(name string, volumeMeta map[string]interface{}) string {
datastore, _ := volumeMeta["datastore"].(string)
return datastore
func GetDatastore(name string, d drivers.VolumeDriver) (string, error) {
volumeMeta, err := d.GetVolume(name)
if err != nil {
log.Errorf("Unable to get volume metadata %s (err: %v)", name, err)
return "", err
}
return volumeMeta["datastore"].(string), nil
}

// GetFullVolumeName - append datastore to the volume name
func GetFullVolumeName(name string, datastore string) string {
s := []string{name, datastore}
return strings.Join(s, "@")
}
func GetFullVolumeName(name string, datastoreName string, d drivers.VolumeDriver) (string, error) {
if strings.ContainsAny(name, "@") {
return name, nil
}
if datastoreName != "" {
return strings.Join([]string{name, datastoreName}, "@"), nil
}

// IsFullVolumeName - return if name is full volume name i.e. volume@datastore
func IsFullVolumeName(name string) bool {
return strings.ContainsAny(name, "@")
datastoreName, err := GetDatastore(name, d)
if err != nil {
return "", err
}
return strings.Join([]string{name, datastoreName}, "@"), nil
}
35 changes: 20 additions & 15 deletions vmdk_plugin/utils/refcount/refcnt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,7 @@ const (
refCountDelayStartSec = 2
refCountRetryAttempts = 20

// consts for finding and parsing linux mount information
linuxMountsFile = "/proc/mounts"
photonDriver = "photon"
photonDriver = "photon"
)

// info about individual volume ref counts and mount
Expand Down Expand Up @@ -315,8 +313,8 @@ func (r *RefCountsMap) Decr(vol string) (uint, error) {
return rc.count, nil
}

// check if volume with source as mount_source belongs to vsphere plugin
func matchNameforVMDK(mount_source string) bool {
// check if volume with source as mount_source belongs to vmdk plugin
func isVMDKMount(mount_source string) bool {
managedPluginMountStart := "/var/lib/docker/plugins/"

// if plugin is used as a service
Expand Down Expand Up @@ -366,7 +364,7 @@ func (r *RefCountsMap) discoverAndSync(c *client.Client, d drivers.VolumeDriver)
}

// use same datastore for all volumes with short names
datastore := ""
datastoreName := ""

log.Infof("Found %d running or paused containers", len(containers))
for _, ct := range containers {
Expand All @@ -387,20 +385,27 @@ func (r *RefCountsMap) discoverAndSync(c *client.Client, d drivers.VolumeDriver)
log.Debugf(" Mounts for %v", ct.Names)
for _, mount := range containerJSONInfo.Mounts {
// check if the mount location belongs to vmdk plugin
if matchNameforVMDK(mount.Source) != true {
if isVMDKMount(mount.Source) != true {
continue
}
volname := mount.Name
if plugin_utils.IsFullVolumeName(volname) != true {
if datastore == "" {
volumeMeta, _ := d.GetVolume(volname)
datastore = plugin_utils.GetDatastore(volname, volumeMeta)
// gets hit once to retrieve the default datastore. datastoreName is reused henceforth
if datastoreName == "" {
datastoreName, err = plugin_utils.GetDatastore(volname, d)
if err != nil {
log.Errorf("Unable to get datastore for volume %s. err:%v", volname, err)
return err
}
volname = plugin_utils.GetFullVolumeName(volname, datastore)
r.Incr(volname)
log.Debugf("name=%v (driver=%s source=%s) (%v)",
mount.Name, mount.Driver, mount.Source, mount)
}

volname, err = plugin_utils.GetFullVolumeName(volname, datastoreName, d)
if err != nil {
log.Errorf("Unable to get full name for volume %s. err:%v", volname, err)
return err
}
r.Incr(volname)
log.Debugf("name=%v (driver=%s source=%s) (%v)",
mount.Name, mount.Driver, mount.Source, mount)
}
}

Expand Down

0 comments on commit e88907c

Please sign in to comment.