Add lockfile at snapshot activation to avoid race condition

This creates an ignite lock file at /tmp/ignite-snapshot.lock
when an overlay snapshot is created. The locking is handled via
pid file using github.com/nightlyone/lockfile package. This
helps avoid the race condition when multiple ignite processes try
to create loop device and use the device mapper for overlay
snapshot at the same time. When a process obtains a lock, other
processes retry to obtain a lock, until a lock is obtained. Once
the snapshot is activated, the lock is released.

cmd/ignite/run/create.go

@@ -4,6 +4,7 @@ import (
  "fmt"
  "path"
  "strings"
+ "time"
 
  api "github.com/weaveworks/ignite/pkg/apis/ignite"
  "github.com/weaveworks/ignite/pkg/apis/ignite/scheme"
@@ -15,6 +16,10 @@ import (
  "github.com/weaveworks/ignite/pkg/providers"
 )
 
+// lockRetryTime is the time for which ignite create waits, if it fails to
+// obtain a lock, before retrying.
+const lockRetryTime = 3 * time.Second
+
 func NewCreateFlags() *CreateFlags {
  return &CreateFlags{
  VM: providers.Client.VMs().New(),

pkg/dmlegacy/snapshot.go

@@ -4,6 +4,9 @@ import (
  "fmt"
  "os"
  "path"
+ "path/filepath"
+
+ "github.com/nightlyone/lockfile"
 
  api "github.com/weaveworks/ignite/pkg/apis/ignite"
  "github.com/weaveworks/ignite/pkg/constants"
@@ -12,6 +15,8 @@ import (
  "github.com/weaveworks/ignite/pkg/util"
 )
 
+const snapshotLockFileName = "ignite-snapshot.lock"
+
 // ActivateSnapshot sets up the snapshot with devicemapper so that it is active and can be used
 func ActivateSnapshot(vm *api.VM) error {
  device := util.NewPrefixer().Prefix(vm.GetUID())
@@ -28,6 +33,28 @@ func ActivateSnapshot(vm *api.VM) error {
  return err
  }
 
+ // NOTE: Multiple ignite processes trying to create loop devices at the
+ // same time results in race condition. When multiple processes request for
+ // a free loop device at the same time, they may get the same device ID and
+ // try to create the same device multiple times.
+ // Serialize this operation by creating a global lock file when creating a
+ // loop device and release the lock after setting up device mapper using the
+ // loop device.
+
+ // Global lock path.
+ glpath := filepath.Join(os.TempDir(), snapshotLockFileName)
+
+ // Create a lockfile and obtain a lock.
+ lock, err := lockfile.New(glpath)
+ if err != nil {
+ return fmt.Errorf("failed to create lock: %v", err)
+ }
+ if err := obtainLock(lock); err != nil {
+ return err
+ }
+ // Release the lock at the end.
+ defer lock.Unlock()
+
  // Setup loop device for the image
  imageLoop, err := newLoopDev(path.Join(constants.IMAGE_DIR, imageUID.String(), constants.IMAGE_FS), true)
  if err != nil {
@@ -101,3 +128,33 @@ func ActivateSnapshot(vm *api.VM) error {
 
  return overlayLoop.Detach()
 }
+
+// obtainLock tries to obtain a lock and retries if the lock is owned by
+// another process, until a lock is obtained.
+func obtainLock(lock lockfile.Lockfile) error {
+ // Check if the lock has any owner.
+ process, err := lock.GetOwner()
+ if err == nil {
+ // A lock already exists. Check if the lock owner is the current process
+ // itself.
+ if process.Pid == os.Getpid() {
+ return fmt.Errorf("lockfile %q already locked by this process", lock)
+ }
+
+ // A lock already exists, but it's owned by some other process. Continue
+ // to obtain lock, in case the lock owner no longer exists.
+ }
+
+ // Obtain a lock. Retry if the lock can't be obtained.
+ err = lock.TryLock()
+ for err != nil {
+ // Check if it's a lock temporary error that can be mitigated with a
+ // retry. Fail if any other error.
+ if _, ok := err.(interface{ Temporary() bool }); !ok {
+ return fmt.Errorf("unable to lock %q: %v", lock, err)
+ }
+ err = lock.TryLock()
+ }
+
+ return nil
+}