-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Storage][DataMovement] Add perf tests for DMLib Track1 (#46768)
- Loading branch information
1 parent
3db918d
commit fd0b3e7
Showing
14 changed files
with
357 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
96 changes: 96 additions & 0 deletions
96
...bs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Infrastructure/DirectoryTransferTest.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System; | ||
| using System.IO; | ||
| using System.Linq; | ||
| using System.Threading.Tasks; | ||
| using Azure.Test.Perf; | ||
| using Microsoft.Azure.Storage.Auth; | ||
| using Microsoft.Azure.Storage.Blob; | ||
|
|
||
| namespace Microsoft.Azure.Storage.DataMovement.Perf | ||
| { | ||
| public abstract class DirectoryTransferTest<TOptions> : PerfTest<TOptions> where TOptions : DirectoryTransferOptions | ||
| { | ||
| protected CloudBlobClient BlobClient; | ||
|
|
||
| protected static DirectoryTransferContext DefaultTransferContext => new() | ||
| { | ||
| ShouldOverwriteCallbackAsync = DirectoryTransferContext.ForceOverwrite | ||
| }; | ||
|
|
||
| public DirectoryTransferTest(TOptions options) : base(options) | ||
| { | ||
| StorageCredentials credentials = new( | ||
| PerfTestEnvironment.Instance.StorageAccountName, | ||
| PerfTestEnvironment.Instance.StorageAccountKey); | ||
|
|
||
| CloudStorageAccount account = new(credentials, PerfTestEnvironment.Instance.StorageEndpointSuffix, useHttps: true); | ||
| BlobClient = account.CreateCloudBlobClient(); | ||
|
|
||
| if (Options.ChunkSize.HasValue) | ||
| { | ||
| TransferManager.Configurations.BlockSize = (int)Options.ChunkSize.Value; | ||
| } | ||
| if (Options.Concurrency.HasValue) | ||
| { | ||
| TransferManager.Configurations.ParallelOperations = Options.Concurrency.Value; | ||
| } | ||
| } | ||
|
|
||
| protected string CreateLocalDirectory(bool populate = false) | ||
| { | ||
| string directory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); | ||
| Directory.CreateDirectory(directory); | ||
|
|
||
| if (populate) | ||
| { | ||
| foreach (int i in Enumerable.Range(0, Options.Count)) | ||
| { | ||
| string filePath = Path.Combine(directory, $"file{i}"); | ||
| using (Stream stream = RandomStream.Create(Options.Size)) | ||
| using (FileStream file = System.IO.File.Open(filePath, FileMode.Create)) | ||
| { | ||
| stream.CopyTo(file); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return directory; | ||
| } | ||
|
|
||
| protected async Task<CloudBlobContainer> CreateBlobContainerAsync(bool populate = false) | ||
| { | ||
| string containerName = $"test-{Guid.NewGuid()}".ToLowerInvariant(); | ||
| CloudBlobContainer container = BlobClient.GetContainerReference(containerName); | ||
| await container.CreateIfNotExistsAsync(); | ||
|
|
||
| if (populate) | ||
| { | ||
| foreach (int i in Enumerable.Range(0, Options.Count)) | ||
| { | ||
| CloudBlockBlob blob = container.GetBlockBlobReference($"blob{i}"); | ||
| using (Stream stream = RandomStream.Create(Options.Size)) | ||
| { | ||
| await blob.UploadFromStreamAsync(stream); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return container; | ||
| } | ||
|
|
||
| protected void AssertTransferStatus(TransferStatus status) | ||
| { | ||
| if (status.NumberOfFilesSkipped > 0) | ||
| { | ||
| throw new Exception($"Transfer contained {status.NumberOfFilesSkipped} skipped files."); | ||
| } | ||
| if (status.NumberOfFilesFailed > 0) | ||
| { | ||
| throw new Exception($"Transfer contaiend {status.NumberOfFilesFailed} failed files."); | ||
| } | ||
| } | ||
| } | ||
| } |
35 changes: 35 additions & 0 deletions
35
...lobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Infrastructure/PerfTestEnvironment.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using Azure.Core.TestFramework; | ||
|
|
||
| namespace Microsoft.Azure.Storage.DataMovement.Perf | ||
| { | ||
| /// <summary> | ||
| /// Represents the ambient environment in which the test suite is being run, offering access to information such as environment variables. | ||
| /// </summary> | ||
| internal sealed class PerfTestEnvironment : TestEnvironment | ||
| { | ||
| /// <summary> | ||
| /// The shared instance of the <see cref="PerfTestEnvironment"/> to be used during test runs. | ||
| /// </summary> | ||
| public static PerfTestEnvironment Instance { get; } = new PerfTestEnvironment(); | ||
|
|
||
| /// <summary> | ||
| /// The storage account endpoint suffix to use for testing. | ||
| /// </summary> | ||
| public new string StorageEndpointSuffix => base.StorageEndpointSuffix ?? "core.windows.net"; | ||
|
|
||
| /// <summary> | ||
| /// The name of the Blob storage account to test against. | ||
| /// </summary> | ||
| /// <value>The Blob storage account name, read from the "AZURE_STORAGE_ACCOUNT_NAME" environment variable.</value> | ||
| public string StorageAccountName => GetVariable("AZURE_STORAGE_ACCOUNT_NAME"); | ||
|
|
||
| /// <summary> | ||
| /// The shared access key of the Blob storage account to test against. | ||
| /// </summary> | ||
| /// <value>The Blob storage account key, read from the "AZURE_STORAGE_ACCOUNT_KEY" environment variable.</value> | ||
| public string StorageAccountKey => GetVariable("AZURE_STORAGE_ACCOUNT_KEY"); | ||
| } | ||
| } |
13 changes: 13 additions & 0 deletions
13
...icrosoft.Azure.Storage.DataMovement.Perf/Microsoft.Azure.Storage.DataMovement.Perf.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| </PropertyGroup> | ||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.Azure.Storage.DataMovement" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj" /> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj" /> | ||
| </ItemGroup> | ||
| </Project> |
27 changes: 27 additions & 0 deletions
27
....Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Options/DirectoryTransferOptions.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using Azure.Test.Perf; | ||
| using CommandLine; | ||
|
|
||
| namespace Microsoft.Azure.Storage.DataMovement.Perf | ||
| { | ||
| public class DirectoryTransferOptions : PerfOptions | ||
| { | ||
| [Option('c', "count", Default = 10, HelpText = "Number of items in each transfer.")] | ||
| public int Count { get; set; } | ||
|
|
||
| [Option('s', "size", Default = 1024, HelpText = "Size of each file (in bytes)")] | ||
| public long Size { get; set; } | ||
|
|
||
| [Option("chunk-size", HelpText = "The chunk/block size to use during transfers (in bytes)")] | ||
| public long? ChunkSize { get; set; } | ||
|
|
||
| [Option("concurrency", HelpText = "The max concurrency to use during each transfer.")] | ||
| public int? Concurrency { get; set; } | ||
|
|
||
| // Override warmup to set default to 0 | ||
| [Option('w', "warmup", Default = 0, HelpText = "Duration of warmup in seconds")] | ||
| public new int Warmup { get; set; } | ||
| } | ||
| } |
7 changes: 7 additions & 0 deletions
7
...zure.Storage.DataMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Program.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System.Reflection; | ||
| using Azure.Test.Perf; | ||
|
|
||
| await PerfProgram.Main(Assembly.GetEntryAssembly(), args); |
51 changes: 51 additions & 0 deletions
51
...aMovement.Blobs/perf/Microsoft.Azure.Storage.DataMovement.Perf/Scenarios/CopyDirectory.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Azure.Storage.Blob; | ||
|
|
||
| namespace Microsoft.Azure.Storage.DataMovement.Perf | ||
| { | ||
| public class CopyDirectory : DirectoryTransferTest<DirectoryTransferOptions> | ||
| { | ||
| private CloudBlobContainer _sourceContainer; | ||
| private CloudBlobContainer _destinationContainer; | ||
|
|
||
| public CopyDirectory(DirectoryTransferOptions options) : base(options) | ||
| { | ||
| } | ||
|
|
||
| public override async Task GlobalSetupAsync() | ||
| { | ||
| await base.GlobalSetupAsync(); | ||
| _sourceContainer = await CreateBlobContainerAsync(populate: true); | ||
| _destinationContainer = await CreateBlobContainerAsync(); | ||
| } | ||
|
|
||
| public override async Task GlobalCleanupAsync() | ||
| { | ||
| await _sourceContainer.DeleteIfExistsAsync(); | ||
| await _destinationContainer.DeleteIfExistsAsync(); | ||
| await base.GlobalCleanupAsync(); | ||
| } | ||
|
|
||
| public override void Run(CancellationToken cancellationToken) | ||
| { | ||
| throw new NotImplementedException(); | ||
| } | ||
|
|
||
| public override async Task RunAsync(CancellationToken cancellationToken) | ||
| { | ||
| TransferStatus transfer = await TransferManager.CopyDirectoryAsync( | ||
| _sourceContainer.GetDirectoryReference(string.Empty), | ||
| _destinationContainer.GetDirectoryReference(string.Empty), | ||
| CopyMethod.ServiceSideSyncCopy, | ||
| options: null, | ||
| DefaultTransferContext, | ||
| CancellationToken.None); // Don't pass cancellation token to let ransfer finish gracefully | ||
| AssertTransferStatus(transfer); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.