-
Notifications
You must be signed in to change notification settings - Fork 95
Avoiding redundant mounts by using /proc/mount info #1224
Changes from 2 commits
be205c1
e88907c
62782d9
51468d6
f7e3360
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -67,10 +67,11 @@ function check_files { | |
} | ||
|
||
function check_recovery_record { | ||
line=`tail -10 /var/log/docker-volume-vsphere.log | $GREP 'Volume name=' | $GREP 'mounted=true'` | ||
expected="name=$vname count=$count mounted=true" | ||
# log contains refcounting attempts and after success logs summary. | ||
line=`tail -50 /var/log/docker-volume-vsphere.log | $GREP 'Volume name=' | $GREP 'mounted=true'` | ||
expected="count=$count mounted=true" | ||
|
||
echo $line | $GREP -q "$expected" ; if [ $? -ne 0 ] ; then | ||
echo $line | $GREP "$vname" | $GREP -q "$expected" ; if [ $? -ne 0 ] ; then | ||
echo Found: \"$line\" | ||
echo Expected pattern: \"$expected\" | ||
return 1 | ||
|
@@ -80,20 +81,21 @@ function check_recovery_record { | |
|
||
function test_crash_recovery { | ||
timeout=$1 | ||
echo "Checking recovery for VMDK plugin kill -9" | ||
kill -9 `pidof docker-volume-vsphere` | ||
until pids=$(pidof docker-volume-vsphere) | ||
echo "Checking recovery through docker kill" | ||
# kill docker daemon forcefully | ||
pkill -9 dockerd | ||
until pids=$(pidof dockerd) | ||
do | ||
echo "Waiting for docker-volume-vsphere to restart" | ||
echo "Waiting for docker to restart" | ||
sleep 1 | ||
done | ||
|
||
echo "Waiting for plugin init" | ||
sleep 3 | ||
sleep 5 | ||
sync # give log the time to flush | ||
wait_for check_recovery_record $timeout | ||
if [ "$?" -ne 0 ] ; then | ||
echo PLUGIN RESTART TEST FAILED. Did not find proper recovery record | ||
echo DOCKER RESTART TEST FAILED. Did not find proper recovery record | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not sure this change is needed. The test is to see if the plugin restarts then the refcounts are restored. If the plugin restart isn't possible then this test should be made manual to execute with a standalone binary and deprecated from automated test run. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test will in turn test plugin restart (docker restart). Plugin restart through docker restart tests the docker api consumption, timeout and handling of parallel mounts/unmounts till refcounting is successful. Not sure why not have this as automated test? |
||
exit 1 | ||
fi | ||
} | ||
|
@@ -115,8 +117,9 @@ fi | |
echo "$(docker volume ls)" | ||
for i in `seq 1 $count` | ||
do | ||
$DOCKER run -d -v $vname:/v busybox sh -c "touch /v/file$i; sync ; \ | ||
sleep $timeout" | ||
# run containers with restart flag so they restart after docker restart | ||
$DOCKER run -d --restart=always -v $vname:/v busybox sh -c "touch /v/file$i; sync ; \ | ||
while true; do sleep $timeout; done" | ||
done | ||
|
||
echo "Checking the last refcount and mount record" | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -38,6 +38,7 @@ import ( | |
log "github.com/Sirupsen/logrus" | ||
"github.com/docker/go-plugins-helpers/volume" | ||
"github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils/fs" | ||
"github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils/plugin_utils" | ||
"github.com/vmware/docker-volume-vsphere/vmdk_plugin/utils/refcount" | ||
"github.com/vmware/photon-controller-go-sdk/photon" | ||
) | ||
|
@@ -56,12 +57,13 @@ const ( | |
|
||
// VolumeDriver - Photon volume driver struct | ||
type VolumeDriver struct { | ||
client *photon.Client | ||
hostID string | ||
mountRoot string | ||
project string | ||
refCounts *refcount.RefCountsMap | ||
target string | ||
client *photon.Client | ||
hostID string | ||
mountRoot string | ||
project string | ||
refCounts *refcount.RefCountsMap | ||
target string | ||
mountIDtoName map[string]string // map of mountID -> full volume name | ||
} | ||
|
||
func (d *VolumeDriver) verifyTarget() error { | ||
|
@@ -95,6 +97,7 @@ func NewVolumeDriver(targetURL string, projectID string, hostID string, mountDir | |
d.mountRoot = mountDir | ||
d.refCounts = refcount.NewRefCountsMap() | ||
d.refCounts.Init(d, mountDir, driverName) | ||
d.mountIDtoName = make(map[string]string) | ||
|
||
log.WithFields(log.Fields{ | ||
"version": version, | ||
|
@@ -362,6 +365,19 @@ func (d *VolumeDriver) MountVolume(name string, fstype string, id string, isRead | |
|
||
// private function that does the job of mounting volume in conjunction with refcounting | ||
func (d *VolumeDriver) processMount(r volume.MountRequest) volume.Response { | ||
// get volume metadata | ||
volumeMeta, err := d.GetVolume(r.Name) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. saving VolumeMeta as part of retrieving full volume name. If found, avoiding next get trip. |
||
|
||
if err != nil { | ||
return volume.Response{Err: err.Error()} | ||
} | ||
|
||
r.Name, err = plugin_utils.GetFullVolumeName(r.Name, volumeMeta["datastore"].(string), d) | ||
if err != nil { | ||
log.Errorf("Unable to get full name for volume %s. err:%v", r.Name, err) | ||
return volume.Response{Err: err.Error()} | ||
} | ||
|
||
// If the volume is already mounted , just increase the refcount. | ||
// Note: for new keys, GO maps return zero value, so no need for if_exists. | ||
refcnt := d.incrRefCount(r.Name) // save map traversal | ||
|
@@ -373,22 +389,19 @@ func (d *VolumeDriver) processMount(r volume.MountRequest) volume.Response { | |
return volume.Response{Mountpoint: d.getMountPoint(r.Name)} | ||
} | ||
|
||
// There can be redundant mounts till refcounts are properly initialized | ||
// TODO: #1220 | ||
status, err := d.GetVolume(r.Name) | ||
if err != nil { | ||
d.decrRefCount(r.Name) | ||
return volume.Response{Err: err.Error()} | ||
if plugin_utils.AlreadyMounted(r.Name, d.mountRoot) { | ||
log.WithFields(log.Fields{"name": r.Name}).Info("Already mounted, skipping mount. ") | ||
return volume.Response{Mountpoint: d.getMountPoint(r.Name)} | ||
} | ||
|
||
fstype, exists := status[fsTypeTag] | ||
fstype, exists := volumeMeta[fsTypeTag] | ||
if !exists { | ||
fstype = fs.FstypeDefault | ||
} | ||
|
||
skipAttach := false | ||
// If the volume is already attached to the VM, skip the attach. | ||
if state, stateExists := status["State"]; stateExists { | ||
if state, stateExists := volumeMeta["State"]; stateExists { | ||
if strings.Compare(state.(string), "DETACHED") != 0 { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is not your change but please file an issue and let's revisit. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
skipAttach = true | ||
} | ||
|
@@ -398,7 +411,7 @@ func (d *VolumeDriver) processMount(r volume.MountRequest) volume.Response { | |
} | ||
|
||
// Mount the volume and for now its always read-write. | ||
mountpoint, err := d.MountVolume(r.Name, fstype.(string), status["ID"].(string), false, skipAttach) | ||
mountpoint, err := d.MountVolume(r.Name, fstype.(string), volumeMeta["ID"].(string), false, skipAttach) | ||
if err != nil { | ||
log.WithFields( | ||
log.Fields{"name": r.Name, "error": err.Error()}, | ||
|
@@ -618,6 +631,18 @@ func (d *VolumeDriver) Unmount(r volume.UnmountRequest) volume.Response { | |
return volume.Response{Err: ""} | ||
} | ||
|
||
if fullVolName, exist := d.mountIDtoName[r.ID]; exist { | ||
r.Name = fullVolName | ||
delete(d.mountIDtoName, r.ID) //cleanup the map | ||
} else { | ||
fullVolName, err := plugin_utils.GetFullVolumeName(r.Name, "", d) | ||
if err != nil { | ||
log.Errorf("Unable to get full name for volume %s. err:%v", r.Name, err) | ||
return volume.Response{Err: err.Error()} | ||
} | ||
r.Name = fullVolName | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I suggest to wrap up L639->644 (with slight modification) in |
||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Dont you have to delete mountID => volName entry from map? |
||
// if refcount has been succcessful, Normal flow. | ||
// if the volume is still used by other containers, just return OK | ||
refcnt, err := d.decrRefCount(r.Name) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,96 @@ | ||
// Copyright 2016 VMware, Inc. All Rights Reserved. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package plugin_utils | ||
|
||
// This file holds utility/helper methods required in plugin module | ||
|
||
import ( | ||
"io/ioutil" | ||
"path/filepath" | ||
"strings" | ||
|
||
log "github.com/Sirupsen/logrus" | ||
"github.com/vmware/docker-volume-vsphere/vmdk_plugin/drivers" | ||
) | ||
|
||
const ( | ||
// consts for finding and parsing linux mount information | ||
linuxMountsFile = "/proc/mounts" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this was already defined somewhere. Did you fully move the definition here ? If not, please do |
||
) | ||
|
||
// GetMountInfo - return a map of mounted volumes and devices | ||
func GetMountInfo(mountRoot string) (map[string]string, error) { | ||
volumeMountMap := make(map[string]string) //map [volume mount path] -> device | ||
data, err := ioutil.ReadFile(linuxMountsFile) | ||
|
||
if err != nil { | ||
log.Errorf("Can't get info from %s (%v)", linuxMountsFile, err) | ||
return volumeMountMap, err | ||
} | ||
|
||
for _, line := range strings.Split(string(data), "\n") { | ||
field := strings.Fields(line) | ||
if len(field) < 2 { | ||
continue // skip empty line and lines too short to have our mount | ||
} | ||
// fields format: [/dev/sdb /mnt/vmdk/vol1 ext2 rw,relatime 0 0] | ||
if filepath.Dir(field[1]) != mountRoot { | ||
continue | ||
} | ||
volumeMountMap[filepath.Base(field[1])] = field[0] | ||
} | ||
return volumeMountMap, nil | ||
} | ||
|
||
// AlreadyMounted - check if volume is already mounted on the mountRoot | ||
func AlreadyMounted(name string, mountRoot string) bool { | ||
volumeMap, err := GetMountInfo(mountRoot) | ||
|
||
if err != nil { | ||
return false | ||
} | ||
|
||
if _, ok := volumeMap[name]; ok { | ||
return true | ||
} | ||
return false | ||
} | ||
|
||
// GetDatastore - get datastore from volume metadata | ||
// Note "datastore" key is defined in vmdkops service | ||
func GetDatastore(name string, d drivers.VolumeDriver) (string, error) { | ||
volumeMeta, err := d.GetVolume(name) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Function is returning values that are not seemingly related to the name of the function - GetDatastore(). Preferably remove this and let the calling function just call GetVolume() and pick the datastore entry from the return meta-data. If this function is a must-have then let it return just the datastore name. I mean we are returning the datastore name and the meta-data which also has the same datastore name in it. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. refactored. This function is no more needed. did the getvolume in the calling function. |
||
if err != nil { | ||
log.Errorf("Unable to get volume metadata %s (err: %v)", name, err) | ||
return "", err | ||
} | ||
return volumeMeta["datastore"].(string), nil | ||
} | ||
|
||
// GetFullVolumeName - append datastore to the volume name | ||
func GetFullVolumeName(name string, datastoreName string, d drivers.VolumeDriver) (string, error) { | ||
if strings.ContainsAny(name, "@") { | ||
return name, nil | ||
} | ||
if datastoreName != "" { | ||
return strings.Join([]string{name, datastoreName}, "@"), nil | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Unlike what the name suggests "GetFullNameAndMeta()" most of the returns here return NULL for the meta. Is that intended? If only one condition is going to return the metadata for a volume then suggest that this function just return the full name or even merge it with the calling function. Should never mix multiple unrelated actions inside a single function. - doesn't need to return the meta-data for a volume GetVolume() should be used for that. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes renaming it. The idea was to return additional information along with full name so as to avoid trips to esx |
||
|
||
datastoreName, err := GetDatastore(name, d) | ||
if err != nil { | ||
return "", err | ||
} | ||
return strings.Join([]string{name, datastoreName}, "@"), nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
How does this impact the refcount module. Unless the plugin is restarted, the refcount module hasn't lost it's refcounts to do a recovery.