[Storage][DataMovement] Add perf tests for DMLib Track1

eng/Packages.Data.props

@@ -313,6 +313,7 @@
     <PackageReference Update="Microsoft.Azure.ResourceManager" Version="[1.1.0-preview]" />
     <PackageReference Update="Microsoft.Azure.Services.AppAuthentication" Version="[1.0.3, 2.0.0)" />
     <PackageReference Update="Microsoft.Azure.Storage.Blob" Version="11.1.7" />
+    <PackageReference Update="Microsoft.Azure.Storage.DataMovement" Version="2.0.5" />
     <PackageReference Update="Microsoft.Azure.Storage.File" Version="11.2.2" />
     <PackageReference Update="Microsoft.Azure.Storage.Queue" Version="11.1.7" />
     <PackageReference Update="Microsoft.Azure.Test.HttpRecorder" Version="[1.13.3, 2.0.0)" />

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Azure.Storage.DataMovement.Blobs.Perf.csproj

@@ -8,8 +8,4 @@
     <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj" />
     <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj" />
   </ItemGroup>
-
-  <ItemGroup>
-    <Compile Include="$(AzureStorageSharedTestSources)\RepeatingStream.cs" LinkBase="Shared" />
-  </ItemGroup>
 </Project>

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/DirectoryTransferTest.cs

@@ -7,26 +7,33 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Azure.Storage.Blobs;
-using Azure.Storage.Tests.Shared;
 using Azure.Test.Perf;
 
 namespace Azure.Storage.DataMovement.Blobs.Perf
 {
     public abstract class DirectoryTransferTest<TOptions> : PerfTest<TOptions> where TOptions : DirectoryTransferOptions
     {
-        protected Random Random { get; }
         protected BlobServiceClient BlobServiceClient { get; }
         protected LocalFilesStorageResourceProvider LocalFileResourceProvider { get; }
         protected BlobsStorageResourceProvider BlobResourceProvider { get; }
+
+        private TransferManager _transferManager;
         private TimeSpan _transferTimeout;
 
         public DirectoryTransferTest(TOptions options) : base(options)
         {
-            Random = new Random();
-            BlobServiceClient = new BlobServiceClient(PerfTestEnvironment.Instance.BlobStorageEndpoint, PerfTestEnvironment.Instance.Credential);
+            BlobServiceClient = new BlobServiceClient(PerfTestEnvironment.Instance.StorageEndpoint, PerfTestEnvironment.Instance.Credential);
             LocalFileResourceProvider = new LocalFilesStorageResourceProvider();
             BlobResourceProvider = new BlobsStorageResourceProvider(PerfTestEnvironment.Instance.Credential);
-            _transferTimeout = TimeSpan.FromSeconds(5 + (Options.Count * Options.Size) / (1 * 1024 * 1024));
+            _transferTimeout = TimeSpan.FromSeconds(10 + (Options.Count * Options.Size) / (1 * 1024 * 1024));
+
+            TransferManagerOptions managerOptions = new()
+            {
+                ErrorHandling = DataTransferErrorMode.StopOnAnyFailure,
+                CheckpointerOptions = Options.DisableCheckpointer ? TransferCheckpointStoreOptions.Disabled() : default,
+                MaximumConcurrency = Options.Concurrency
+            };
+            _transferManager = new TransferManager(managerOptions);
         }
 
         protected string CreateLocalDirectory(bool populate = false)
@@ -39,7 +46,7 @@ protected string CreateLocalDirectory(bool populate = false)
                 foreach (int i in Enumerable.Range(0, Options.Count))
                 {
                     string filePath = Path.Combine(directory, $"file{i}");
-                    using (RepeatingStream stream = new(1024 * 1024, Options.Size, true))
+                    using (Stream stream = RandomStream.Create(Options.Size))
                     using (FileStream file = File.Open(filePath, FileMode.Create))
                     {
                         stream.CopyTo(file);
@@ -61,7 +68,7 @@ protected async Task<BlobContainerClient> CreateBlobContainerAsync(bool populate
                 foreach (int i in Enumerable.Range(0, Options.Count))
                 {
                     BlobClient blob = container.GetBlobClient($"blob{i}");
-                    using (RepeatingStream stream = new(1024 * 1024, Options.Size, true))
+                    using (Stream stream = RandomStream.Create(Options.Size))
                     {
                         await blob.UploadAsync(stream);
                     }
@@ -76,21 +83,14 @@ protected async Task RunAndVerifyTransferAsync(
             StorageResource destination,
             CancellationToken cancellationToken)
         {
-            TransferManagerOptions managerOptions = new()
-            {
-                ErrorHandling = DataTransferErrorMode.StopOnAnyFailure,
-                CheckpointerOptions = Options.DisableCheckpointer ? TransferCheckpointStoreOptions.Disabled() : default
-            };
-            TransferManager transferManager = new(managerOptions);
-
             DataTransferOptions options = new()
             {
                 CreationPreference = StorageResourceCreationPreference.OverwriteIfExists,
                 InitialTransferSize = Options.InitialTransferSize,
                 MaximumTransferChunkSize = Options.ChunkSize,
             };
             options.ItemTransferFailed += HandleFailure;
-            DataTransfer transfer = await transferManager.StartTransferAsync(
+            DataTransfer transfer = await _transferManager.StartTransferAsync(
                 source, destination, options, cancellationToken);
 
             // The test runs for a specified duration and then cancels the token.

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Infrastructure/PerfTestEnvironment.cs

@@ -25,19 +25,19 @@ internal sealed class PerfTestEnvironment : TestEnvironment
         /// The name of the Blob storage account to test against.
         /// </summary>
         /// <value>The Blob storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value>
-        public string BlobStorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
+        public string StorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
 
         /// <summary>
         /// The Blob storage endpoint.
         /// </summary>
-        public Uri BlobStorageEndpoint { get; }
+        public Uri StorageEndpoint { get; }
 
         /// <summary>
         /// Initializes a new instance of the <see cref="PerfTestEnvironment"/> class.
         /// </summary>
         public PerfTestEnvironment()
         {
-            BlobStorageEndpoint = new Uri($"https://{BlobStorageAccountName}.blob.{StorageEndpointSuffix}");
+            StorageEndpoint = new Uri($"https://{StorageAccountName}.blob.{StorageEndpointSuffix}");
         }
     }
 }

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Azure.Storage.DataMovement.Blobs.Perf/Options/DirectoryTransferOptions.cs

@@ -20,6 +20,9 @@ public class DirectoryTransferOptions : PerfOptions
         [Option("chunk-size", HelpText = "The chunk/block size to use during transfers (in bytes)")]
         public long? ChunkSize { get; set; }
 
+        [Option("concurrency", HelpText = "The max concurrency to use during each transfer.")]
+        public int? Concurrency { get; set; }
+
         [Option("disable-checkpointer", HelpText = "Set to disable checkpointing.")]
         public bool DisableCheckpointer { get; set; }
 

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Infrastructure/DirectoryTransferTest.cs

@@ -0,0 +1,96 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Azure.Test.Perf;
+using Microsoft.Azure.Storage.Auth;
+using Microsoft.Azure.Storage.Blob;
+
+namespace Microsoft.Azure.Storage.DataMovement.Perf
+{
+    public abstract class DirectoryTransferTest<TOptions> : PerfTest<TOptions> where TOptions : DirectoryTransferOptions
+    {
+        protected CloudBlobClient BlobClient;
+
+        protected static DirectoryTransferContext DefaultTransferContext => new()
+        {
+            ShouldOverwriteCallbackAsync = DirectoryTransferContext.ForceOverwrite
+        };
+
+        public DirectoryTransferTest(TOptions options) : base(options)
+        {
+            StorageCredentials credentials = new(
+                PerfTestEnvironment.Instance.StorageAccountName,
+                PerfTestEnvironment.Instance.StorageAccountKey);
+
+            CloudStorageAccount account = new(credentials, PerfTestEnvironment.Instance.StorageEndpointSuffix, useHttps: true);
+            BlobClient = account.CreateCloudBlobClient();
+
+            if (Options.ChunkSize.HasValue)
+            {
+                TransferManager.Configurations.BlockSize = (int)Options.ChunkSize.Value;
+            }
+            if (Options.Concurrency.HasValue)
+            {
+                TransferManager.Configurations.ParallelOperations = Options.Concurrency.Value;
+            }
+        }
+
+        protected string CreateLocalDirectory(bool populate = false)
+        {
+            string directory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+            Directory.CreateDirectory(directory);
+
+            if (populate)
+            {
+                foreach (int i in Enumerable.Range(0, Options.Count))
+                {
+                    string filePath = Path.Combine(directory, $"file{i}");
+                    using (Stream stream = RandomStream.Create(Options.Size))
+                    using (FileStream file = System.IO.File.Open(filePath, FileMode.Create))
+                    {
+                        stream.CopyTo(file);
+                    }
+                }
+            }
+
+            return directory;
+        }
+
+        protected async Task<CloudBlobContainer> CreateBlobContainerAsync(bool populate = false)
+        {
+            string containerName = $"test-{Guid.NewGuid()}".ToLowerInvariant();
+            CloudBlobContainer container = BlobClient.GetContainerReference(containerName);
+            await container.CreateIfNotExistsAsync();
+
+            if (populate)
+            {
+                foreach (int i in Enumerable.Range(0, Options.Count))
+                {
+                    CloudBlockBlob blob = container.GetBlockBlobReference($"blob{i}");
+                    using (Stream stream = RandomStream.Create(Options.Size))
+                    {
+                        await blob.UploadFromStreamAsync(stream);
+                    }
+                }
+            }
+
+            return container;
+        }
+
+        protected void AssertTransferStatus(TransferStatus status)
+        {
+            if (status.NumberOfFilesSkipped > 0)
+            {
+                throw new Exception($"Transfer contained {status.NumberOfFilesSkipped} skipped files.");
+            }
+            if (status.NumberOfFilesFailed > 0)
+            {
+                throw new Exception($"Transfer contaiend {status.NumberOfFilesFailed} failed files.");
+            }
+        }
+    }
+}

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Infrastructure/PerfTestEnvironment.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Core.TestFramework;
+
+namespace Microsoft.Azure.Storage.DataMovement.Perf
+{
+    /// <summary>
+    /// Represents the ambient environment in which the test suite is being run, offering access to information such as environment variables.
+    /// </summary>
+    internal sealed class PerfTestEnvironment : TestEnvironment
+    {
+        /// <summary>
+        /// The shared instance of the <see cref="PerfTestEnvironment"/> to be used during test runs.
+        /// </summary>
+        public static PerfTestEnvironment Instance { get; } = new PerfTestEnvironment();
+
+        /// <summary>
+        /// The storage account endpoint suffix to use for testing.
+        /// </summary>
+        public new string StorageEndpointSuffix => base.StorageEndpointSuffix ?? "core.windows.net";
+
+        /// <summary>
+        /// The name of the Blob storage account to test against.
+        /// </summary>
+        /// <value>The Blob storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value>
+        public string StorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME");
+
+        /// <summary>
+        /// The shared access key of the Blob storage account to test against.
+        /// </summary>
+        /// <value>The Blob storage account key, read from the "AZURE_STORAGE_ACCOUNT_KEY" environment variable.</value>
+        public string StorageAccountKey => GetVariable("AZURE_STORAGE_ACCOUNT_KEY");
+    }
+}

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Microsoft.Azure.Storage.DataMovement.Perf.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Azure.Storage.DataMovement" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj" />
+    <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj" />
+  </ItemGroup>
+</Project>

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Options/DirectoryTransferOptions.cs

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Test.Perf;
+using CommandLine;
+
+namespace Microsoft.Azure.Storage.DataMovement.Perf
+{
+    public class DirectoryTransferOptions : PerfOptions
+    {
+        [Option('c', "count", Default = 10, HelpText = "Number of items in each transfer.")]
+        public int Count { get; set; }
+
+        [Option('s', "size", Default = 1024, HelpText = "Size of each file (in bytes)")]
+        public long Size { get; set; }
+
+        [Option("chunk-size", HelpText = "The chunk/block size to use during transfers (in bytes)")]
+        public long? ChunkSize { get; set; }
+
+        [Option("concurrency", HelpText = "The max concurrency to use during each transfer.")]
+        public int? Concurrency { get; set; }
+
+        // Override warmup to set default to 0
+        [Option('w', "warmup", Default = 0, HelpText = "Duration of warmup in seconds")]
+        public new int Warmup { get; set; }
+    }
+}

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Program.cs

@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System.Reflection;
+using Azure.Test.Perf;
+
+await PerfProgram.Main(Assembly.GetEntryAssembly(), args);

sdk/storage/Azure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Scenarios/CopyDirectory.cs

@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Azure.Storage.Blob;
+
+namespace Microsoft.Azure.Storage.DataMovement.Perf
+{
+    public class CopyDirectory : DirectoryTransferTest<DirectoryTransferOptions>
+    {
+        private CloudBlobContainer _sourceContainer;
+        private CloudBlobContainer _destinationContainer;
+
+        public CopyDirectory(DirectoryTransferOptions options) : base(options)
+        {
+        }
+
+        public override async Task GlobalSetupAsync()
+        {
+            await base.GlobalSetupAsync();
+            _sourceContainer = await CreateBlobContainerAsync(populate: true);
+            _destinationContainer = await CreateBlobContainerAsync();
+        }
+
+        public override async Task GlobalCleanupAsync()
+        {
+            await _sourceContainer.DeleteIfExistsAsync();
+            await _destinationContainer.DeleteIfExistsAsync();
+            await base.GlobalCleanupAsync();
+        }
+
+        public override void Run(CancellationToken cancellationToken)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override async Task RunAsync(CancellationToken cancellationToken)
+        {
+            TransferStatus transfer = await TransferManager.CopyDirectoryAsync(
+                _sourceContainer.GetDirectoryReference(string.Empty),
+                _destinationContainer.GetDirectoryReference(string.Empty),
+                CopyMethod.ServiceSideSyncCopy,
+                options: null,
+                DefaultTransferContext,
+                CancellationToken.None);  // Don't pass cancellation token to let ransfer finish gracefully
+            AssertTransferStatus(transfer);
+        }
+    }
+}