Add retries when removing device mapper target

Additionally ignore the error from device mapper if target removal
still fails. The corresponding layer path is already unmounted
at that point and this avoids having an inconsistent state.

Signed-off-by: Maksim An <maksiman@microsoft.com>

internal/guest/storage/devicemapper/devicemapper.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"time"
 	"unsafe"
 
 	"golang.org/x/sys/unix"
@@ -270,13 +271,30 @@ func CreateDevice(name string, flags CreateFlags, targets []Target) (_ string, e
 
 // RemoveDevice removes a device-mapper device and its associated device node.
 func RemoveDevice(name string) error {
-	f, err := openMapper()
-	if err != nil {
-		return err
+	rm := func() error {
+		f, err := openMapper()
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		os.Remove(path.Join("/dev/mapper", name))
+		return removeDevice(f, name)
+	}
+
+	// This is workaround for "device or resource busy" error, which occasionally happens after the device mapper
+	// target has been unmounted.
+	for {
+		if err := rm(); err != nil {
+			select {
+			case <-time.After(time.Second):
+				return fmt.Errorf("timeout removing device %s", name)
+			default:
+				time.Sleep(10 * time.Millisecond)
+				continue
+			}
+		}
+		return nil
 	}
-	defer f.Close()
-	os.Remove(path.Join("/dev/mapper", name))
-	return removeDevice(f, name)
 }
 
 func removeDevice(f *os.File, name string) error {

internal/guest/storage/pmem/pmem.go

@@ -139,14 +139,16 @@ func Unmount(ctx context.Context, devNumber uint32, target string, mappingInfo *
 	if verityInfo != nil {
 		dmVerityName := fmt.Sprintf(verityDeviceFmt, devNumber, verityInfo.RootDigest)
 		if err := dm.RemoveDevice(dmVerityName); err != nil {
-			return errors.Wrapf(err, "failed to remove dm verity target: %s", dmVerityName)
+			// The target is already unmounted at this point, ignore potential errors
+			log.G(ctx).WithError(err).Debugf("failed to remove dm verity target: %s", dmVerityName)
 		}
 	}
 
 	if mappingInfo != nil {
 		dmLinearName := fmt.Sprintf(linearDeviceFmt, devNumber, mappingInfo.DeviceOffsetInBytes, mappingInfo.DeviceSizeInBytes)
 		if err := dm.RemoveDevice(dmLinearName); err != nil {
-			return errors.Wrapf(err, "failed to remove dm linear target: %s", dmLinearName)
+			// The target is already unmounted at this point, ignore potential errors
+			log.G(ctx).WithError(err).Debugf("failed to remove dm linear target: %s", dmLinearName)
 		}
 	}