devicemapper: add EBUSY to the retriable errors

Testing revealed that creating device mapper targets sometimes
yields `device or resource busy` error (`EBUSY`). Add it to
the list of retriable errors and consolidate them into a single
slice.

Add unit tests for `CreateDeviceWithRetryError`.

Signed-off-by: Maksim An <maksiman@microsoft.com>

internal/guest/storage/devicemapper/devicemapper.go

@@ -30,6 +30,7 @@ const (
 var (
  removeDeviceWrapper = removeDevice
  openMapperWrapper = openMapper
+ _createDevice = CreateDevice
 )
 
 //nolint:stylecheck // ST1003: ALL_CAPS
@@ -229,7 +230,7 @@ func CreateDeviceWithRetryErrors(
 ) (string, error) {
 retry:
  for {
- dmPath, err := CreateDevice(name, flags, targets)
+ dmPath, err := _createDevice(name, flags, targets)
  if err == nil {
  return dmPath, nil
  }
@@ -244,7 +245,7 @@ retry:
  if errors.Is(dmErr.Err, e) {
  select {
  case <-ctx.Done():
- log.G(ctx).WithError(err).Warning("CreateDeviceWithRetryErrors failed, context timeout")
+ log.G(ctx).WithError(err).Error("CreateDeviceWithRetryErrors failed, context timeout")
  return "", err
  default:
  time.Sleep(100 * time.Millisecond)

internal/guest/storage/devicemapper/devicemapper_test.go

@@ -4,8 +4,10 @@
 package devicemapper
 
 import (
+ "context"
  "errors"
  "flag"
+ "fmt"
  "os"
  "syscall"
  "testing"
@@ -19,6 +21,7 @@ var (
 )
 
 func clearTestDependencies() {
+ _createDevice = CreateDevice
  removeDeviceWrapper = removeDevice
  openMapperWrapper = openMapper
 }
@@ -73,7 +76,7 @@ func (d *device) Close() (err error) {
 }
 
 func createDevice(name string, flags CreateFlags, targets []Target) (*device, error) {
- p, err := CreateDevice(name, flags, targets)
+ p, err := _createDevice(name, flags, targets)
  if err != nil {
  return nil, err
  }
@@ -188,6 +191,40 @@ func TestRemoveDeviceRetriesOnSyscallEBUSY(t *testing.T) {
  }
 }
 
+func TestCreateDeviceWithRetryError(t *testing.T) {
+ for _, rerr := range retryErrors {
+ t.Run(fmt.Sprintf("error-%s", rerr), func(t *testing.T) {
+ clearTestDependencies()
+ createDeviceCalled := false
+ retryDone := false
+ createDeviceStub := func(name string, flags CreateFlags, targets []Target) (string, error) {
+ if !createDeviceCalled {
+ createDeviceCalled = true
+ return "", &dmError{
+ Op: 0,
+ Err: rerr,
+ }
+ }
+ retryDone = true
+ return fmt.Sprintf("/dev/mapper/%s", name), nil
+ }
+ _createDevice = createDeviceStub
+
+ ctx := context.Background()
+ _, err := CreateDeviceWithRetryErrors(ctx, "test-device", 0, []Target{}, retryErrors...)
+ if err != nil {
+ t.Fatalf("unexpected error: %s\n", err)
+ }
+ if !createDeviceCalled {
+ t.Fatal("CreateDevice was not called")
+ }
+ if !retryDone {
+ t.Fatalf("error %s was not retried\n", rerr)
+ }
+ })
+ }
+}
+
 func TestRemoveDeviceFailsOnNonSyscallEBUSY(t *testing.T) {
  clearTestDependencies()
 

internal/guest/storage/devicemapper/targets.go

@@ -15,6 +15,17 @@ import (
  "github.com/Microsoft/hcsshim/internal/protocol/guestresource"
 )
 
+// retryErrors are treated as transient errors and device-mapper operations will be tried again.
+//
+// These errors are occasionally returned by the kernel when a device-mapper operation is attempted. These errors
+// are transient and can occur when the kernel is busy or the device is not ready yet.
+var retryErrors = []error{
+ unix.EBUSY,
+ unix.ENOENT,
+ unix.ENXIO,
+ unix.ENODEV,
+}
+
 // CreateZeroSectorLinearTarget creates dm-linear target for a device at `devPath` and `mappingInfo`, returns
 // virtual block device path.
 func CreateZeroSectorLinearTarget(ctx context.Context, devPath, devName string, mappingInfo *guestresource.LCOWVPMemMappingInfo) (_ string, err error) {
@@ -37,9 +48,7 @@ func CreateZeroSectorLinearTarget(ctx context.Context, devPath, devName string,
  devName,
  CreateReadOnly,
  []Target{linearTarget},
- unix.ENOENT,
- unix.ENXIO,
- unix.ENODEV,
+ retryErrors...,
  )
  if err != nil {
  return "", fmt.Errorf("failed to create dm-linear target, device=%s, offset=%d: %w", devPath,
@@ -94,9 +103,7 @@ func CreateVerityTarget(ctx context.Context, devPath, devName string, verityInfo
  devName,
  CreateReadOnly,
  []Target{verityTarget},
- unix.ENOENT,
- unix.ENXIO,
- unix.ENODEV,
+ retryErrors...,
  )
  if err != nil {
  return "", fmt.Errorf("frailed to create dm-verity target for device=%s: %w", devPath, err)