forked from Azure/azure-sdk-for-net
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Data Lake Performance] Test Scenarios for T2 (Azure#17307)
The focus of these changes is to create the initial set of performance test scenarios for the v12.x (track two) Data Lake client library.
- Loading branch information
1 parent
cc6042c
commit 7e94547
Showing
11 changed files
with
697 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
16 changes: 16 additions & 0 deletions
16
....DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <PropertyGroup> | ||
| <Version>1.0.0</Version> | ||
| <OutputType>Exe</OutputType> | ||
| <LangVersion>latest</LangVersion> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Azure.Storage.Files.DataLake" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj" /> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj" /> | ||
| </ItemGroup> | ||
| </Project> |
43 changes: 43 additions & 0 deletions
43
...les.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.sln
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
|
|
||
| Microsoft Visual Studio Solution File, Format Version 12.00 | ||
| # Visual Studio Version 16 | ||
| VisualStudioVersion = 16.0.30711.63 | ||
| MinimumVisualStudioVersion = 10.0.40219.1 | ||
| Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Files.DataLake.Perf", "Azure.Storage.Files.DataLake.Perf.csproj", "{700A2B09-B9BE-4727-8CD8-9513EAA185C1}" | ||
| EndProject | ||
| Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "External", "External", "{D19AC4D9-9D47-4CBB-9D39-BBCCB73E0093}" | ||
| EndProject | ||
| Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Perf", "..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj", "{19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}" | ||
| EndProject | ||
| Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Core.TestFramework", "..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj", "{59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}" | ||
| EndProject | ||
| Global | ||
| GlobalSection(SolutionConfigurationPlatforms) = preSolution | ||
| Debug|Any CPU = Debug|Any CPU | ||
| Release|Any CPU = Release|Any CPU | ||
| EndGlobalSection | ||
| GlobalSection(ProjectConfigurationPlatforms) = postSolution | ||
| {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU | ||
| {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Debug|Any CPU.Build.0 = Debug|Any CPU | ||
| {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Release|Any CPU.ActiveCfg = Release|Any CPU | ||
| {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Release|Any CPU.Build.0 = Release|Any CPU | ||
| {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU | ||
| {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Debug|Any CPU.Build.0 = Debug|Any CPU | ||
| {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Release|Any CPU.ActiveCfg = Release|Any CPU | ||
| {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Release|Any CPU.Build.0 = Release|Any CPU | ||
| {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU | ||
| {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Debug|Any CPU.Build.0 = Debug|Any CPU | ||
| {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Release|Any CPU.ActiveCfg = Release|Any CPU | ||
| {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Release|Any CPU.Build.0 = Release|Any CPU | ||
| EndGlobalSection | ||
| GlobalSection(SolutionProperties) = preSolution | ||
| HideSolutionNode = FALSE | ||
| EndGlobalSection | ||
| GlobalSection(NestedProjects) = preSolution | ||
| {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D} = {D19AC4D9-9D47-4CBB-9D39-BBCCB73E0093} | ||
| {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6} = {D19AC4D9-9D47-4CBB-9D39-BBCCB73E0093} | ||
| EndGlobalSection | ||
| GlobalSection(ExtensibilityGlobals) = postSolution | ||
| SolutionGuid = {8E6A0374-AFF4-4DF0-8663-856605C48B62} | ||
| EndGlobalSection | ||
| EndGlobal |
69 changes: 69 additions & 0 deletions
69
...les.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Infrastructure/PerfTestEnvironment.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System; | ||
| using Azure.Core.TestFramework; | ||
|
|
||
| namespace Azure.Storage.Files.DataLake.Perf | ||
| { | ||
| /// <summary> | ||
| /// Represents the ambient environment in which the test suite is | ||
| /// being run, offering access to information such as environment | ||
| /// variables. | ||
| /// </summary> | ||
| /// | ||
| public sealed class PerfTestEnvironment : TestEnvironment | ||
| { | ||
| /// <summary> | ||
| /// The shared instance of the <see cref="PerfTestEnvironment"/> to be used during test runs. | ||
| /// </summary> | ||
| /// | ||
| public static PerfTestEnvironment Instance { get; } = new PerfTestEnvironment(); | ||
|
|
||
| /// <summary> | ||
| /// The storage account endpoint suffix of the cloud to use for testing. | ||
| /// </summary> | ||
| /// | ||
| public new string StorageEndpointSuffix => base.StorageEndpointSuffix ?? "core.windows.net"; | ||
|
|
||
| /// <summary> | ||
| /// The name of the Data Lake storage account to test against. | ||
| /// </summary> | ||
| /// | ||
| /// <value>The Data Lake storage account name, read from the "DATALAKE_STORAGE_ACCOUNT_NAME" environment variable.</value> | ||
| /// | ||
| public string DataLakeAccountName => GetVariable("DATALAKE_STORAGE_ACCOUNT_NAME"); | ||
|
|
||
| /// <summary> | ||
| /// The shared access key of the Data Lake storage account to test against. | ||
| /// </summary> | ||
| /// | ||
| /// <value>The Data Lake storage account key, read from the "DATALAKE_STORAGE_ACCOUNT_KEY" environment variable.</value> | ||
| /// | ||
| public string DataLakeAccountKey => GetVariable("DATALAKE_STORAGE_ACCOUNT_KEY"); | ||
|
|
||
| /// <summary> | ||
| /// The fully-qualified URI for the Data Lake storage account to test against. | ||
| /// </summary> | ||
| /// | ||
| public Uri DataLakeServiceUri { get; } | ||
|
|
||
| /// <summary> | ||
| /// The credential for accessing the Data Lake storage account used for testing. | ||
| /// </summary> | ||
| /// | ||
| /// <value>This credential is based on the configured Data Lake shared key.</value> | ||
| /// | ||
| public StorageSharedKeyCredential DataLakeCredential { get; } | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="PerfTestEnvironment"/> class. | ||
| /// </summary> | ||
| /// | ||
| public PerfTestEnvironment() : base("storage") | ||
| { | ||
| DataLakeServiceUri = new Uri($"{ Uri.UriSchemeHttps }{ Uri.SchemeDelimiter }{ DataLakeAccountName }.dfs.{ StorageEndpointSuffix }"); | ||
| DataLakeCredential = new StorageSharedKeyCredential(DataLakeAccountName, DataLakeAccountKey); | ||
| } | ||
| } | ||
| } |
7 changes: 7 additions & 0 deletions
7
sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Program.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System.Reflection; | ||
| using Azure.Test.Perf; | ||
|
|
||
| await PerfProgram.Main(Assembly.GetEntryAssembly(), args); |
144 changes: 144 additions & 0 deletions
144
...e/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Append.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| //Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System; | ||
| using System.IO; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Azure.Test.Perf; | ||
|
|
||
| namespace Azure.Storage.Files.DataLake.Perf.Scenarios | ||
| { | ||
| /// <summary> | ||
| /// The performance test scenario focused on appending to an existing | ||
| /// file in Data Lake storage. | ||
| /// </summary> | ||
| /// | ||
| /// <seealso cref="Azure.Test.Perf.PerfTest{SizeOptions}" /> | ||
| /// | ||
| public sealed class Append : PerfTest<SizeOptions> | ||
| { | ||
| /// <summary> | ||
| /// The ambient test environment associated with the current execution. | ||
| /// </summary> | ||
| /// | ||
| private static PerfTestEnvironment TestEnvironment { get; } = PerfTestEnvironment.Instance; | ||
|
|
||
| /// <summary> | ||
| /// The name of the file system to use across parallel executions of the scenario. | ||
| /// </summary> | ||
| /// | ||
| private static string FileSystemName { get; } = Guid.NewGuid().ToString(); | ||
|
|
||
| /// <summary> | ||
| /// The client for interaction with the Data Lake file system. | ||
| /// </summary> | ||
| /// | ||
| private DataLakeFileSystemClient FileSystemClient { get; } | ||
|
|
||
| /// <summary> | ||
| /// The client to use for interactions with the test file. | ||
| /// </summary> | ||
| /// | ||
| private DataLakeFileClient FileClient { get; } | ||
|
|
||
| /// <summary> | ||
| /// The payload to use with a file being uploaded. | ||
| /// </summary> | ||
| /// | ||
| private Stream Payload { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="Append"/> class. | ||
| /// </summary> | ||
| /// | ||
| /// <param name="options">The set of options to consider for configuring the scenario.</param> | ||
| /// | ||
| public Append(SizeOptions options) : base(options) | ||
| { | ||
| var serviceClient = new DataLakeServiceClient(TestEnvironment.DataLakeServiceUri, TestEnvironment.DataLakeCredential); | ||
|
|
||
| FileSystemClient = serviceClient.GetFileSystemClient(FileSystemName); | ||
| FileClient = FileSystemClient.GetFileClient(Path.GetRandomFileName()); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Performs the tasks needed to initialize and set up the environment for the test scenario. | ||
| /// When multiple instances are run in parallel, the setup will take place once, prior to the | ||
| /// execution of the first test instance. | ||
| /// </summary> | ||
| /// | ||
| public async override Task GlobalSetupAsync() | ||
| { | ||
| await base.GlobalSetupAsync(); | ||
| await FileSystemClient.CreateAsync(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Performs the tasks needed to clean up the environment for the test scenario. | ||
| /// When multiple instances are run in parallel, the cleanup will take place once, | ||
| /// after the execution of all test instances. | ||
| /// </summary> | ||
| /// | ||
| public async override Task GlobalCleanupAsync() | ||
| { | ||
| await base.GlobalCleanupAsync(); | ||
| await FileSystemClient.DeleteAsync(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Performs the tasks needed to initialize and set up the environment for an instance | ||
| /// of the test scenario. When multiple instances are run in parallel, setup will be | ||
| /// run once for each prior to its execution. | ||
| /// </summary> | ||
| /// | ||
| public async override Task SetupAsync() | ||
| { | ||
| await base.SetupAsync(); | ||
| Payload = RandomStream.Create(Options.Size); | ||
|
|
||
| // Create the test file that will be used as the basis for uploading. | ||
|
|
||
| using var randomStream = RandomStream.Create(1024); | ||
|
|
||
| await FileClient.CreateAsync(); | ||
| await FileClient.UploadAsync(randomStream, true); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Performs the tasks needed to clean up the environment for an instance | ||
| /// of the test scenario. When multiple instances are run in parallel, cleanup | ||
| /// will be run once for each after execution has completed. | ||
| /// </summary> | ||
| /// | ||
| public async override Task CleanupAsync() | ||
| { | ||
| await base.CleanupAsync(); | ||
| Payload.Dispose(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Executes the performance test scenario synchronously. | ||
| /// </summary> | ||
| /// | ||
| /// <param name="cancellationToken">The token used to signal when cancellation is requested.</param> | ||
| /// | ||
| public override void Run(CancellationToken cancellationToken) | ||
| { | ||
| Payload.Position = 0; | ||
| FileClient.Append(Payload, 0, cancellationToken:cancellationToken); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Executes the performance test scenario asynchronously. | ||
| /// </summary> | ||
| /// | ||
| /// <param name="cancellationToken">The token used to signal when cancellation is requested.</param> | ||
| /// | ||
| public async override Task RunAsync(CancellationToken cancellationToken) | ||
| { | ||
| Payload.Position = 0; | ||
| await FileClient.AppendAsync(Payload, 0, cancellationToken: cancellationToken); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.