DataMovementTestBase pull out shared functionality (#37493)

* move job part plan file utility out of testbase

* file util

* remove empty setup method

* pull out and rename enum

* remove dmlibtestbase from rehydrate tests

* remove recordings

sdk/storage/Azure.Storage.Common/tests/Shared/DataProvider.cs

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+
+namespace Azure.Storage.Test
+{
+    public static class DataProvider
+    {
+        public static Dictionary<string, string> BuildTags()
+            => new()
+            {
+                { "tagKey0", "tagValue0" },
+                { "tagKey1", "tagValue1" }
+            };
+
+        public static Dictionary<string, string> BuildMetadata()
+            => new(StringComparer.OrdinalIgnoreCase)
+                {
+                    { "foo", "bar" },
+                    { "meta", "data" },
+                    { "Capital", "letter" },
+                    { "UPPER", "case" }
+                };
+
+        public static string GetNewString(int length = 20, Random random = null)
+        {
+            random ??= new Random();
+            var buffer = new char[length];
+            for (var i = 0; i < length; i++)
+            {
+                buffer[i] = (char)('a' + random.Next(0, 25));
+            }
+            return new string(buffer);
+        }
+    }
+}

sdk/storage/Azure.Storage.Common/tests/Shared/FileUtil.cs

@@ -0,0 +1,62 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System.Collections.Generic;
+using System.IO;
+
+namespace Azure.Storage.Test.Shared
+{
+    internal static class FileUtil
+    {
+        public static List<string> ListFileNamesRecursive(string directory)
+        {
+            List<string> files = new List<string>();
+
+            // Create a queue of folders to enumerate files from, starting with provided path
+            Queue<string> folders = new();
+            folders.Enqueue(directory);
+
+            while (folders.Count > 0)
+            {
+                // Grab a folder from the queue
+                string dir = folders.Dequeue();
+
+                // Try to enumerate and queue all subdirectories of the current folder
+                try
+                {
+                    foreach (string subdir in Directory.EnumerateDirectories(dir))
+                    {
+                        folders.Enqueue(subdir);
+                    }
+                }
+                catch
+                {
+                    // If we lack permissions to enumerate, throw if the caller specifies
+                    // that we shouldn't continue on error.
+                    //
+                    // TODO: Logging for missing permissions to enumerate folder
+                    //
+                    // Afterthought: once logging is implemented, we can just log any problems
+                    // (whether with given dir or subdir), and skip if told to/throw if not. No need for
+                    // the `dir == _basePath` check (which right now is just a filler signal
+                    // for something going wrong, as opposed to an "success" with an empty list).
+                    if (dir == directory)
+                    {
+                        throw;
+                    }
+
+                    // Otherwise, just log the failed subdirectory and continue to list as many
+                    // files as accessible.
+                    continue;
+                }
+
+                // Send all files in the directory to the scan results
+                foreach (string file in Directory.EnumerateFiles(dir))
+                {
+                    files.Add(file);
+                }
+            }
+            return files;
+        }
+    }
+}

sdk/storage/Azure.Storage.Common/tests/Shared/StorageTestBase.cs

@@ -107,26 +107,11 @@ public virtual void ClearCaches() =>
         public byte[] GetRandomBuffer(long size)
             => TestHelper.GetRandomBuffer(size, Recording.Random);
 
-        public string GetNewString(int length = 20)
-        {
-            var buffer = new char[length];
-            for (var i = 0; i < length; i++)
-            {
-                buffer[i] = (char)('a' + Recording.Random.Next(0, 25));
-            }
-            return new string(buffer);
-        }
+        public string GetNewString(int length = 20) => DataProvider.GetNewString(length, Recording.Random);
 
         public string GetNewMetadataName() => $"test_metadata_{Recording.Random.NewGuid().ToString().Replace("-", "_")}";
 
-        public IDictionary<string, string> BuildMetadata()
-            => new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
-                {
-                    { "foo", "bar" },
-                    { "meta", "data" },
-                    { "Capital", "letter" },
-                    { "UPPER", "case" }
-                };
+        public IDictionary<string, string> BuildMetadata() => DataProvider.BuildMetadata();
 
         public IPAddress GetIPAddress()
         {

sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj

@@ -10,8 +10,11 @@
     <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.DataMovement\src\Azure.Storage.DataMovement.csproj" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="$(AzureStorageDataMovementTestSharedSources)CheckpointerTesting.cs" LinkBase="Shared\DataMovement" />
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)DataMovementTestBase.cs" LinkBase="Shared\DataMovement" />
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)DisposingLocalDirectory.cs" LinkBase="Shared\DataMovement" />
+    <Compile Include="$(AzureStorageDataMovementTestSharedSources)TransferDirection.cs" LinkBase="Shared\DataMovement" />
+    <Compile Include="$(AzureStorageDataMovementTestSharedSources)TransferUtility.cs" LinkBase="Shared\DataMovement" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="$(AzureStorageDataMovementSharedSources)..\Models\JobPlan\*.cs" LinkBase="Shared\DataMovement\JobPlanModels" />

sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs

@@ -10,12 +10,14 @@
 using Azure.Storage.DataMovement.Blobs;
 using Azure.Storage.DataMovement.Models;
 using Azure.Storage.DataMovement.Models.JobPlan;
+using Azure.Storage.Test;
 using Moq;
 using NUnit.Framework;
+using static Azure.Storage.DataMovement.Tests.TransferUtility;
 
 namespace Azure.Storage.DataMovement.Tests
 {
-    public class RehydrateBlobResourceTests : DataMovementTestBase
+    public class RehydrateBlobResourceTests
     {
         public enum RehydrateApi
         {
@@ -36,8 +38,7 @@ public enum RehydrateApi
         }
         public static IEnumerable<RehydrateApi> GetRehydrateApis() => Enum.GetValues(typeof(RehydrateApi)).Cast<RehydrateApi>();
 
-        public RehydrateBlobResourceTests(bool async)
-            : base(async, null /* RecordedTestMode.Record /* to re-record */)
+        public RehydrateBlobResourceTests()
         { }
 
         private enum StorageResourceType
@@ -132,7 +133,7 @@ private async Task AddJobPartToCheckpointer(
 
             for (int currentPart = 0; currentPart < partCount; currentPart++)
             {
-                header ??= CreateDefaultJobPartHeader(
+                header ??= CheckpointerTesting.CreateDefaultJobPartHeader(
                     transferId: transferId,
                     partNumber: currentPart,
                     sourcePath: sourcePaths[currentPart],
@@ -236,10 +237,10 @@ public async Task RehydrateBlockBlob_Options(
                 ToResourceId(destinationType),
                 isContainer: false).Object;
 
-            IDictionary<string, string> metadata = BuildMetadata();
-            IDictionary<string, string> blobTags = BuildTags();
+            IDictionary<string, string> metadata = DataProvider.BuildMetadata();
+            IDictionary<string, string> blobTags = DataProvider.BuildTags();
 
-            JobPartPlanHeader header = CreateDefaultJobPartHeader(
+            JobPartPlanHeader header = CheckpointerTesting.CreateDefaultJobPartHeader(
                     transferId: transferId,
                     partNumber: 0,
                     sourcePath: sourcePath,
@@ -343,10 +344,10 @@ public async Task RehydratePageBlob_Options(
                 ToResourceId(destinationType),
                 isContainer: false).Object;
 
-            IDictionary<string, string> metadata = BuildMetadata();
-            IDictionary<string, string> blobTags = BuildTags();
+            IDictionary<string, string> metadata = DataProvider.BuildMetadata();
+            IDictionary<string, string> blobTags = DataProvider.BuildTags();
 
-            JobPartPlanHeader header = CreateDefaultJobPartHeader(
+            JobPartPlanHeader header = CheckpointerTesting.CreateDefaultJobPartHeader(
                     transferId: transferId,
                     partNumber: 0,
                     sourcePath: sourcePath,
@@ -450,10 +451,10 @@ public async Task RehydrateAppendBlob_Options(
                 ToResourceId(destinationType),
                 isContainer: false).Object;
 
-            IDictionary<string, string> metadata = BuildMetadata();
-            IDictionary<string, string> blobTags = BuildTags();
+            IDictionary<string, string> metadata = DataProvider.BuildMetadata();
+            IDictionary<string, string> blobTags = DataProvider.BuildTags();
 
-            JobPartPlanHeader header = CreateDefaultJobPartHeader(
+            JobPartPlanHeader header = CheckpointerTesting.CreateDefaultJobPartHeader(
                     transferId: transferId,
                     partNumber: 0,
                     sourcePath: sourcePath,
@@ -502,7 +503,7 @@ public async Task RehydrateBlobContainer(
             int jobPartCount = 10;
             for (int i = 0; i < jobPartCount; i++)
             {
-                string childPath = GetNewString(5);
+                string childPath = DataProvider.GetNewString(5);
                 sourcePaths.Add(string.Join("/", sourceParentPath, childPath));
                 destinationPaths.Add(string.Join("/", destinationParentPath, childPath));
             }