From 9152bff751297955d7acbf0cb411c924f2cef683 Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Thu, 13 Jul 2023 12:12:15 -0400 Subject: [PATCH 1/9] starting point readme --- .../README.md | 278 +++++++++--------- .../Azure.Storage.DataMovement/README.md | 200 ++++++++----- 2 files changed, 274 insertions(+), 204 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md index f5bf06a5de1e..2bb3cee23222 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md @@ -45,189 +45,195 @@ The Azure.Storage.DataMovement.Blobs library uses clients from the Azure.Storage ## Key concepts -The Azure Storage Common client library contains shared infrastructure like -[authentication credentials][auth_credentials] and [RequestFailedException][RequestFailedException]. +***TODO*** -### Thread safety -We guarantee that all client instance methods are thread-safe and independent of each other ([guideline](https://azure.github.io/azure-sdk/dotnet_introduction.html#dotnet-service-methods-thread-safety)). This ensures that the recommendation of reusing client instances is always safe, even across threads. +## Sample Usage -### Additional concepts - -[Client options](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/README.md#configuring-service-clients-using-clientoptions) | -[Accessing the response](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/README.md#accessing-http-response-details-using-responset) | -[Long-running operations](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/README.md#consuming-long-running-operations-using-operationt) | -[Handling failures](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/README.md#reporting-errors-requestfailedexception) | -[Diagnostics](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/samples/Diagnostics.md) | -[Mocking](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/README.md#mocking) | -[Client lifetime](https://devblogs.microsoft.com/azure-sdk/lifetime-management-and-thread-safety-guarantees-of-azure-sdk-net-clients/) - +This section demonstrates usage of Data Movement for interacting with blob storage. -## Examples +### Initializing Blob Storage `StorageResource` -### Examples using BlobContainerClient extension methods to upload and download directories. +Azure.Storage.DataMovement.Blobs exposes a `StorageResource` for each type of blob (block, page, append) as well as a blob container. Storage resources are initialized with the appropriate client object from Azure.Storage.Blobs. -Instantiate the BlobContainerClient -```C# Snippet:ExtensionMethodCreateContainerClient -BlobServiceClient service = new BlobServiceClient(serviceUri, credential); +```csharp +BlobContainerClient containerClient; +BlockBlobClient blockBlobClient; +PageBlobClient pageBlobClient; +AppendBlobClient appendBlobClient; -BlobContainerClient container = service.GetBlobContainerClient(containerName); +StorageResource containerResource = new BlobStorageResourceContainer(containerClient); +StorageResource blockBlobResource = new BlockBlobStorageResource(blockBlobClient); +StorageResource pageBlobResource = new PageBlobStorageResource(pageBlobClient); +StorageResource appendBlobResource = new AppendBlobStorageResource(appendBlobClient); ``` -Upload a local directory to the root of the container -```C# Snippet:ExtensionMethodSimpleUploadToRoot -DataTransfer transfer = await container.StartUploadDirectoryAsync(localPath); +Blob Storage `StorageResouce` objects can be constructed with optional "options" arguments specific to the type of resource. -await transfer.AwaitCompletion(); -``` - -Upload a local directory to a virtual directory in the container by specifying a directory prefix -```C# Snippet:ExtensionMethodSimpleUploadToDirectoryPrefix -DataTransfer transfer = await container.StartUploadDirectoryAsync(localPath, blobDirectoryPrefix); +```csharp +BlobContainerClient containerClient; +BlobStorageResourceContainerOptions options = new() +{ + DirectoryPrefix = "blob/directory/prefix" +} -await transfer.AwaitCompletion(); +StorageResource virtualDirectoryResource = new BlobStorageResourceContainer( + containerClient, + options); ``` -Upload a local directory to a virtual directory in the container specifying more advanced options -```C# Snippet:ExtensionMethodSimpleUploadWithOptions -BlobContainerClientTransferOptions options = new BlobContainerClientTransferOptions +```csharp +BlockBlobClient blockBlobClient; +string leaseId; +BlockBlobStorageResourceOptions options = new() { - BlobContainerOptions = new BlobStorageResourceContainerOptions - { - DirectoryPrefix = blobDirectoryPrefix - }, - TransferOptions = new TransferOptions() + SourceConditions = new() { - CreateMode = StorageResourceCreateMode.Overwrite, + LeaseId = leaseId } -}; +} -DataTransfer transfer = await container.StartUploadDirectoryAsync(localPath, options); - -await transfer.AwaitCompletion(); +StorageResource virtualDirectoryResource = new BlockBlobStorageResource( + blockBlobClient, + options); ``` -Download the entire container to a local directory -```C# Snippet:ExtensionMethodSimpleDownloadContainer -DataTransfer transfer = await container.StartDownloadToDirectoryAsync(localDirectoryPath); - -await transfer.AwaitCompletion(); -``` +When resuming a transfer, a credential to Azure Storage is likely needed. Credentials are not persisted by the transfer manager. When using `BlobStorageResources.TryGetResourceProviders()` to recreate a `StorageResource` for resume, the returned provider can create the resource with a credential specified by the calling code. This allows for workflows like scoping generation of a Shared Access Signature to the given resource path. Your application should provide its own mechanism for getting the appropriate credential, represented by `GenerateMySasCredential()` in the sample below. -Download a directory in the container by specifying a directory prefix -```C# Snippet:ExtensionMethodSimpleDownloadContainerDirectory -DataTransfer tranfer = await container.StartDownloadToDirectoryAsync(localDirectoryPath2, blobDirectoryPrefix); +```csharp +AzureSasCredential GenerateMySasCredential(string blobUri); -await tranfer.AwaitCompletion(); +if (BlobStorageResources.TryGetResourceProviders( + info, + out BlobStorageResourceProvider blobSrcProvider, + out BlobStorageResourceProvider blobDstProvider)) +{ + sourceResource ??= await blobSrcProvider?.MakeResourceAsync( + GenerateMySasCredential(info.SourcePath)); + destinationResource ??= await blobSrcProvider?.MakeResourceAsync( + GenerateMySasCredential(info.DestinationPath)); +} ``` -Download from the container specifying more advanced options -```C# Snippet:ExtensionMethodSimpleDownloadContainerDirectoryWithOptions -BlobContainerClientTransferOptions options = new BlobContainerClientTransferOptions -{ - BlobContainerOptions = new BlobStorageResourceContainerOptions - { - DirectoryPrefix = blobDirectoryPrefix - }, - TransferOptions = new TransferOptions() - { - CreateMode = StorageResourceCreateMode.Overwrite, - } -}; +### Upload -DataTransfer tranfer = await container.StartDownloadToDirectoryAsync(localDirectoryPath2, options); +An upload takes place between a local file `StorageResource` as source and blob `StorageResource` as destination. -await tranfer.AwaitCompletion(); -``` +Upload a block blob. -### Examples using BlobContainerClient extension methods to upload and download directories. +```csharp +string sourceLocalPath; +BlockBlobClient destinationBlob; -Create Instance of TransferManager with Options -```C# Snippet:CreateTransferManagerWithOptions -// Create BlobTransferManager with event handler in Options bag -TransferManagerOptions transferManagerOptions = new TransferManagerOptions(); -TransferOptions options = new TransferOptions() -{ - MaximumTransferChunkSize = 4 * Constants.MB, - CreateMode = StorageResourceCreateMode.Overwrite, -}; -TransferManager transferManager = new TransferManager(transferManagerOptions); -``` - -Start Upload from Local File to Block Blob -```C# Snippet:SimpleBlobUpload DataTransfer dataTransfer = await transferManager.StartTransferAsync( sourceResource: new LocalFileStorageResource(sourceLocalPath), destinationResource: new BlockBlobStorageResource(destinationBlob)); await dataTransfer.AwaitCompletion(); ``` -Apply Options to Block Blob Download -```C# Snippet:BlockBlobDownloadOptions -await transferManager.StartTransferAsync( - sourceResource: new BlockBlobStorageResource(sourceBlob, new BlockBlobStorageResourceOptions() - { - DestinationConditions = new BlobRequestConditions(){ LeaseId = "xyz" } - }), - destinationResource: new LocalFileStorageResource(downloadPath2)); -``` -Start Directory Upload -```C# Snippet:SimpleDirectoryUpload -// Create simple transfer directory upload job which uploads the directory and the contents of that directory +Upload a directory as a specific blob type. + +```csharp +string sourceLocalPath; +BlobContainerClient destinationContainer; +string optionalDestinationPrefix; + DataTransfer dataTransfer = await transferManager.StartTransferAsync( - sourceResource: new LocalDirectoryStorageResourceContainer(sourcePath), + sourceResource: new LocalDirectoryStorageResourceContainer(sourceLocalPath), destinationResource: new BlobStorageResourceContainer( - container, - new BlobStorageResourceContainerOptions() { DirectoryPrefix = "sample-directory2" }), - transferOptions: options); + destinationContainer, + new BlobStorageResourceContainerOptions() + { + // Block blobs are the default if not specified + BlobType = BlobType.Block, + DirectoryPrefix = optionalDestinationPrefix + })); +await dataTransfer.AwaitCompletion(); +``` + +### Download + +A download takes place between a blob `StorageResource` as source and local file `StorageResource` as destination. + +Download a block blob. + +```csharp +BlockBlobClient sourceBlob; +string destinationLocalPath; + +DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource: new BlockBlobStorageResource(sourceBlob), + destinationResource: new LocalFileStorageResource(destinationLocalPath)); +await dataTransfer.AwaitCompletion(); ``` -Start Directory Download -```C# Snippet:SimpleDirectoryDownload -DataTransfer downloadDirectoryJobId2 = await transferManager.StartTransferAsync( - sourceDirectory2, - destinationDirectory2); +Download a container which may contain a mix of blob types. + +```csharp +BlobContainerClient sourceContainer; +string optionalSourcePrefix; +string destinationLocalPath; + +DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource: new BlobStorageResourceContainer( + sourceContainer, + new BlobStorageResourceContainerOptions() + { + DirectoryPrefix = optionalSourcePrefix + }), + destinationResource: new LocalDirectoryStorageResourceContainer(destinationLocalPath)); +await dataTransfer.AwaitCompletion(); ``` -Simple Logger Sample for Transfer Manager Options -```C# Snippet:SimpleLoggingSample -// Create BlobTransferManager with event handler in Options bag -TransferManagerOptions options = new TransferManagerOptions(); -TransferOptions transferOptions = new TransferOptions(); -transferOptions.SingleTransferCompleted += (SingleTransferCompletedEventArgs args) => -{ - using (StreamWriter logStream = File.AppendText(logFile)) - { - logStream.WriteLine($"File Completed Transfer: {args.SourceResource.Path}"); - } - return Task.CompletedTask; -}; +### Blob Copy + +A copy takes place between two blob `StorageResource` instances. Copying between to Azure blobs uses PUT from URL REST APIs, which do not pass data through the calling machine. + +Copy a single blob. Note the change in blob type on this copy from block to append. + +```csharp +BlockBlobClient sourceBlob; +AppendBlobClient destinationBlob; + +DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource: new BlockBlobStorageResource(sourceBlob), + destinationResource: new AppendBlobStorageResource(destinationBlob)); +await dataTransfer.AwaitCompletion(); ``` -Simple Failed Event Delegation for Container Transfer Options -```C# Snippet:FailedEventDelegation -transferOptions.TransferFailed += (TransferFailedEventArgs args) => -{ - using (StreamWriter logStream = File.AppendText(logFile)) - { - // Specifying specific resources that failed, since its a directory transfer - // maybe only one file failed out of many - logStream.WriteLine($"Exception occured with TransferId: {args.TransferId}," + - $"Source Resource: {args.SourceResource.Path}, +" + - $"Destination Resource: {args.DestinationResource.Path}," + - $"Exception Message: {args.Exception.Message}"); - } - return Task.CompletedTask; -}; +Copy a blob container. + +```csharp +BlobContainerClient sourceContainer; +string optionalSourcePrefix; +BlobContainerClient destinationContainer; +string optionalDestinationPrefix; + +DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource: sourceResource: new BlobStorageResourceContainer( + sourceContainer, + new BlobStorageResourceContainerOptions() + { + DirectoryPrefix = optionalSourcePrefix + }), + destinationResource: new BlobStorageResourceContainer( + destinationContainer, + new BlobStorageResourceContainerOptions() + { + // all source blobs will be copied as a single type of destination blob + // defaults to block blobs if unspecified + BlobType = BlobType.Block, + DirectoryPrefix = optionalDestinationPrefix + })); +await dataTransfer.AwaitCompletion(); ``` ## Troubleshooting -All Azure Storage services will throw a [RequestFailedException][RequestFailedException] -with helpful [`ErrorCode`s][error_codes]. +***TODO*** ## Next steps -Get started with our [Blob DataMovement samples][samples]. +***TODO*** ## Contributing diff --git a/sdk/storage/Azure.Storage.DataMovement/README.md b/sdk/storage/Azure.Storage.DataMovement/README.md index 26cdaca28bec..9353597fc4ed 100644 --- a/sdk/storage/Azure.Storage.DataMovement/README.md +++ b/sdk/storage/Azure.Storage.DataMovement/README.md @@ -36,115 +36,179 @@ Here's an example using the Azure CLI: az storage account create --name MyStorageAccount --resource-group MyResourceGroup --location westus --sku Standard_LRS ``` -### Authenticate the client -In order to interact with the Data Movement library you have to create an instance with the TransferManager class. +## Key concepts + +***TODO*** + +## Sample Usage + +This section demonstrates usage of Data Movement regardless of extension package. Package-specific information and usage samples can be found in that package's documentation. These examples will use local disk and Azure Blob Storage when specific resources are needed for demonstration purposes, but the topics here will apply to other packages. -### Create Instance of TransferManager -```C# Snippet:CreateTransferManagerSimple -TransferManager transferManager = new TransferManager(new TransferManagerOptions()); +### Setup the `TransferManager` + +Singleton usage of `TransferManager` is recommended. Providing `TransferManagerOptions` is optional. + +```csharp +TransferManagerOptions options = new(); +TransferManger transferManager = new TransferManager(options); ``` -### Create Instance of TransferManager with Options -```C# Snippet:CreateTransferManagerWithOptions -// Create BlobTransferManager with event handler in Options bag -TransferManagerOptions transferManagerOptions = new TransferManagerOptions(); -TransferOptions options = new TransferOptions() -{ - MaximumTransferChunkSize = 4 * Constants.MB, - CreateMode = StorageResourceCreateMode.Overwrite, -}; -TransferManager transferManager = new TransferManager(transferManagerOptions); +### Starting New Transfers + +Transfers are defined by a source and destination `StorageResource`. There are two kinds of `StorageResource`: `StorageResourceSingle` and `StorageResourceContainer`. Source and destination of a given transfer must be of the same kind. + +Configurations for accessing data are configured on the `StorageResource`. See further documentation for setting up and configuring your `StorageResource` objects. + +```csharp +// provide these resources +StorageResource sourceResource, destinationResource; +// optionally configure options for this individual transfer +TransferOptions transferOptions = new(); +// optional cancellation token for transfer startup +// not for cancelling the transfer once started! +CancellationToken cancellationToken; + +DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + transferOptions, + cancellationToken); ``` -## Key concepts +### Monitoring Transfers -The Azure Storage Common client library contains shared infrastructure like -[authentication credentials][auth_credentials] and [RequestFailedException][RequestFailedException]. +Transfers can be observed through several mechanisms, depending on your needs. -### Thread safety -We guarantee that all client instance methods are thread-safe and independent of each other ([guideline](https://azure.github.io/azure-sdk/dotnet_introduction.html#dotnet-service-methods-thread-safety)). This ensures that the recommendation of reusing client instances is always safe, even across threads. +#### With `DataTransfer` -### Additional concepts - -[Client options](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/README.md#configuring-service-clients-using-clientoptions) | -[Accessing the response](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/README.md#accessing-http-response-details-using-responset) | -[Long-running operations](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/README.md#consuming-long-running-operations-using-operationt) | -[Handling failures](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/README.md#reporting-errors-requestfailedexception) | -[Diagnostics](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/samples/Diagnostics.md) | -[Mocking](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/README.md#mocking) | -[Client lifetime](https://devblogs.microsoft.com/azure-sdk/lifetime-management-and-thread-safety-guarantees-of-azure-sdk-net-clients/) - +Simple observation can be done through a `DataTransfer` instance representing an individual transfer. This is obtained on transfer start. You can also enumerate through all transfers on a `TransferManager`. -## Examples +```csharp +TransferManager transferManager; -Please see the examples for [Blobs DataMovement][blobs_examples]. +await foreach (DataTransfer transfer in transferManager.GetTransfersAsync()) { + // do something with transfer +} +``` -Pause a transfer using the TransferManager using the respective DataTransfer object -```C# Snippet:TransferManagerTryPause_Async -DataTransfer dataTransfer = await transferManager.StartTransferAsync( - sourceResource: sourceResource, - destinationResource: destinationResource); +`DataTransfer` contains property `TransferStatus`. You can read this to determine the state of the transfer. States include queued for transfer, in progress, paused, completed, and more. -// Pause from the Transfer Manager using the DataTransfer object -await transferManager.PauseTransferIfRunningAsync(dataTransfer); +`DataTransfer` also exposes a task for transfer completion that can be awaited. + +```csharp +DataTransfer dataTransfer; + +await dataTransfer.AwaitCompletion(cancellationToken); ``` -Pause a transfer using the TransferManager using the respective transfer ID -```C# Snippet:TransferManagerTryPauseId_Async -DataTransfer dataTransfer = await transferManager.StartTransferAsync( - sourceResource: sourceResource, - destinationResource: destinationResource); -string transferId = dataTransfer.Id; +#### With Events via `TransferOptions` + +When starting a transfer, `TransferOptions` contains multiple events that can be listened to for observation. Below demonstrates listening to the event for a change in transfer status. -// Pause from the Transfer Manager using the Transfer Id -await transferManager.PauseTransferIfRunningAsync(transferId); +```csharp +TransferOptions transferOption; + +transferOptions.TransferStatus += (TransferStatusEventArgs args) => +{ + // handle status change on the whole transfer +} ``` -Pause a transfer using the respective DataTransfer -```C# Snippet:DataTransferTryPause_Async -DataTransfer dataTransfer = await transferManager.StartTransferAsync( - sourceResource: sourceResource, - destinationResource: destinationResource); +#### With IProgress via `TransferOptions` + +When starting a transfer, `TransferOptions` allows setting a progress handler that contains the progress information for the overall transfer. Granular progress updates will be communicated to the provided `IProgress` instance. + +```csharp +TransferOptions transferOptions = new() +{ + ProgressHandler = new Progress(storageTransferProgress => + { + // handle progress update + }), + // optionally include the below if progress updates on bytes transferred are desired + ProgressHandlerOptions = new() + { + TrackBytesTransferred = true + } +} +``` + +### Pausing transfers + +Transfers can be paused either by a given `DataTransfer` or through the `TransferManager` handling the transfer. + +```csharp +DataTransfer dataTransfer; -// Pause from the DataTransfer object -await dataTransfer.PauseIfRunningAsync(); +await dataTransfer.PauseIfRunningAsync(cancellationToken); ``` -Resume a transfer -```C# Snippet:TransferManagerResume_Async +```csharp +TransferManager transferManager; +string transferId; + +await transferManager.PauseTransferIfRunningAsync(transferId, cancellationToken); +``` + +### Resuming transfers + +Transfer progress is persisted such that it can resume from where it left off. No persisted knowledge is required from your code. The below sample queries a `TransferManager` for information on all resumable transfers and recreates the properly configured resources for these transfers using a helper method we'll define next. It then resumes each of those transfers with the given ID and puts the resulting `DataTransfer` objects into a list. + +```csharp +TransferManager transferManager; + +List resumedTransfers = new(); +await foreach (DataTransferProperties transferProperties in transferManager.GetResumableTransfersAsync()) +{ + (StorageResource resumeSource, StorageResource resumeDestination) = await MakeResourcesAsync(transferProperties); + resumedTransfers.Add(await transferManager.ResumeTransferAsync(transferProperties.TransferId, resumeSource, resumeDestination)); +} +``` + +Note that the transfer manager can only check for resumable transfers based on the `TransferCheckpointerOptions` configured in the `TransferManagerOptions` (default checkpointer options are used if none are provided). + +The above sample's `MakeResourcesAsync` method is defined below. Different `DataMovement` packages provide their own helper functions to recreate the correctly configured `StorageResource` for resuming a transfer. The following example of such a method uses `Azure.Storage.DataMovement`'s built-in local filesystem helper and `Azure.Storage.DataMovement.Blobs`'s helper. You will need to add in other helpers for each package you use (e.g. `Azure.Storage.DataMovement.Files.Shares`). + +Note these resources return a "provider" rather than the resource itself. The provider can make the resource using a credential argument based on resource information (or some other value that was not persisted), rather than create an unauthenticated `StorageResource`. More information on this can be found in applicable packages. + +```csharp async Task<(StorageResource Source, StorageResource Destination)> MakeResourcesAsync(DataTransferProperties info) { StorageResource sourceResource = null, destinationResource = null; + // ask DataMovement.Blobs if it can recreate source or destination resources to Blob Storage if (BlobStorageResources.TryGetResourceProviders( info, out BlobStorageResourceProvider blobSrcProvider, out BlobStorageResourceProvider blobDstProvider)) { - sourceResource ??= await blobSrcProvider.MakeResourceAsync(GetMyCredential(info.SourcePath)); - destinationResource ??= await blobSrcProvider.MakeResourceAsync(GetMyCredential(info.DestinationPath)); + sourceResource ??= await blobSrcProvider?.MakeResourceAsync(); + destinationResource ??= await blobSrcProvider?.MakeResourceAsync(); } + // ask DataMovement if it can recreate source or destination resources to local storage if (LocalStorageResources.TryGetResourceProviders( info, out LocalStorageResourceProvider localSrcProvider, out LocalStorageResourceProvider localDstProvider)) { - sourceResource ??= localSrcProvider.MakeResource(); - destinationResource ??= localDstProvider.MakeResource(); + sourceResource ??= localSrcProvider?.MakeResource(); + destinationResource ??= localDstProvider?.MakeResource(); } return (sourceResource, destinationResource); } -List resumedTransfers = new(); -await foreach (DataTransferProperties transferProperties in transferManager.GetResumableTransfersAsync()) -{ - (StorageResource resumeSource, StorageResource resumeDestination) = await MakeResourcesAsync(transferProperties); - resumedTransfers.Add(await transferManager.ResumeTransferAsync(transferProperties.TransferId, resumeSource, resumeDestination)); -} +``` + +### Initializing Local File `StorageResource` + +When transferring to or from local storage, construct a `LocalFileStorageResource` for single-file transfers or `LocalDirectoryStorageResourceContainer` for directory transfers. Use one of these as the source resource for upload and as the destination for download. Local to local copies are not supported. + +```csharp +StorageResource fileResource = new LocalFileStorageResource("C:/path/to/file.txt"); +StorageResource directoryResource = new LocalDirectoryStorageResourceContainer("C:/path/to/dir"); ``` ## Troubleshooting -All Azure Storage services will throw a [RequestFailedException][RequestFailedException] -with helpful [`ErrorCode`s][error_codes]. +***TODO*** ## Next steps From 0ea1095f2b43b6127b4cfc136e3b1240981f3135 Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Thu, 13 Jul 2023 13:53:48 -0400 Subject: [PATCH 2/9] re-added missed extensions samples --- .../README.md | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md index 2bb3cee23222..abdda46608f2 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md @@ -51,6 +51,83 @@ The Azure.Storage.DataMovement.Blobs library uses clients from the Azure.Storage This section demonstrates usage of Data Movement for interacting with blob storage. +### Extensions on `BlobContainerClient` + +For applicatons with preexisting code using Azure.Storage.Blobs, this package provides extension methods for `BlobContainerClient` to get some of the benefits of the `TransferManager` with minimal extra code. + +Instantiate the BlobContainerClient +```C# Snippet:ExtensionMethodCreateContainerClient +BlobServiceClient service = new BlobServiceClient(serviceUri, credential); + +BlobContainerClient container = service.GetBlobContainerClient(containerName); +``` + +Upload a local directory to the root of the container +```C# Snippet:ExtensionMethodSimpleUploadToRoot +DataTransfer transfer = await container.StartUploadDirectoryAsync(localPath); + +await transfer.AwaitCompletion(); +``` + +Upload a local directory to a virtual directory in the container by specifying a directory prefix +```C# Snippet:ExtensionMethodSimpleUploadToDirectoryPrefix +DataTransfer transfer = await container.StartUploadDirectoryAsync(localPath, blobDirectoryPrefix); + +await transfer.AwaitCompletion(); +``` + +Upload a local directory to a virtual directory in the container specifying more advanced options +```C# Snippet:ExtensionMethodSimpleUploadWithOptions +BlobContainerClientTransferOptions options = new BlobContainerClientTransferOptions +{ + BlobContainerOptions = new BlobStorageResourceContainerOptions + { + DirectoryPrefix = blobDirectoryPrefix + }, + TransferOptions = new TransferOptions() + { + CreateMode = StorageResourceCreateMode.Overwrite, + } +}; + +DataTransfer transfer = await container.StartUploadDirectoryAsync(localPath, options); + +await transfer.AwaitCompletion(); +``` + +Download the entire container to a local directory +```C# Snippet:ExtensionMethodSimpleDownloadContainer +DataTransfer transfer = await container.StartDownloadToDirectoryAsync(localDirectoryPath); + +await transfer.AwaitCompletion(); +``` + +Download a directory in the container by specifying a directory prefix +```C# Snippet:ExtensionMethodSimpleDownloadContainerDirectory +DataTransfer tranfer = await container.StartDownloadToDirectoryAsync(localDirectoryPath2, blobDirectoryPrefix); + +await tranfer.AwaitCompletion(); +``` + +Download from the container specifying more advanced options +```C# Snippet:ExtensionMethodSimpleDownloadContainerDirectoryWithOptions +BlobContainerClientTransferOptions options = new BlobContainerClientTransferOptions +{ + BlobContainerOptions = new BlobStorageResourceContainerOptions + { + DirectoryPrefix = blobDirectoryPrefix + }, + TransferOptions = new TransferOptions() + { + CreateMode = StorageResourceCreateMode.Overwrite, + } +}; + +DataTransfer tranfer = await container.StartDownloadToDirectoryAsync(localDirectoryPath2, options); + +await tranfer.AwaitCompletion(); +``` + ### Initializing Blob Storage `StorageResource` Azure.Storage.DataMovement.Blobs exposes a `StorageResource` for each type of blob (block, page, append) as well as a blob container. Storage resources are initialized with the appropriate client object from Azure.Storage.Blobs. From 42be550ccbf1f65da74d976514b375da135a3311 Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Thu, 13 Jul 2023 14:51:34 -0400 Subject: [PATCH 3/9] handling failures --- .../Azure.Storage.DataMovement/README.md | 57 +++++++++++++++++-- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement/README.md b/sdk/storage/Azure.Storage.DataMovement/README.md index 9353597fc4ed..da080ded16a5 100644 --- a/sdk/storage/Azure.Storage.DataMovement/README.md +++ b/sdk/storage/Azure.Storage.DataMovement/README.md @@ -103,15 +103,27 @@ await dataTransfer.AwaitCompletion(cancellationToken); #### With Events via `TransferOptions` -When starting a transfer, `TransferOptions` contains multiple events that can be listened to for observation. Below demonstrates listening to the event for a change in transfer status. +When starting a transfer, `TransferOptions` contains multiple events that can be listened to for observation. Below demonstrates listening to the event for individual file completion and logging the result. ```csharp -TransferOptions transferOption; +TransferManager transferManager; +StorageResource sourceResource; +StorageResource destinationResource; -transferOptions.TransferStatus += (TransferStatusEventArgs args) => +TransferOptions transferOptions = new(); +transferOptions.SingleTransferCompleted += (SingleTransferCompletedEventArgs args) => { - // handle status change on the whole transfer + using (StreamWriter logStream = File.AppendText(logFile)) + { + logStream.WriteLine($"File Completed Transfer: {args.SourceResource.Path}"); + } + return Task.CompletedTask; } +DataTransfer transfer = transferManager.StartTransferAsync( + sourceResource, + destinationResource, + transferOptions, + cancellationToken); ``` #### With IProgress via `TransferOptions` @@ -197,6 +209,43 @@ async Task<(StorageResource Source, StorageResource Destination)> MakeResourcesA } ``` +### Handling Failed Transfers + +Transfer failure can be observed by checking the `DataTransfer` status upon completion, or by listening to failure events on the transfer. While checking the `DataTransfer` may be sufficient for handling single-file transfer failures, event listening is recommended for container transfers. + +Below logs failure for a single transnfer. + +```csharp +DataTransfer transfer; + +await transfer.AwaitCompletion(); +if (transfer.TransferStatus == StorageTransferStatus.CompletedWithFailedTransfers) +{ + using (StreamWriter logStream = File.AppendText(logFile)) + { + logStream.WriteLine($"Failure for TransferId: {args.TransferId}"); + } +} +``` + +Below logs individual failures in a container transfer. + +```csharp +transferOptions.TransferFailed += (TransferFailedEventArgs args) => +{ + using (StreamWriter logStream = File.AppendText(logFile)) + { + // Specifying specific resources that failed, since its a directory transfer + // maybe only one file failed out of many + logStream.WriteLine($"Exception occured with TransferId: {args.TransferId}," + + $"Source Resource: {args.SourceResource.Path}, +" + + $"Destination Resource: {args.DestinationResource.Path}," + + $"Exception Message: {args.Exception.Message}"); + } + return Task.CompletedTask; +}; +``` + ### Initializing Local File `StorageResource` When transferring to or from local storage, construct a `LocalFileStorageResource` for single-file transfers or `LocalDirectoryStorageResourceContainer` for directory transfers. Use one of these as the source resource for upload and as the destination for download. Local to local copies are not supported. From 3bb5b6ac5a8b3f3181ce1ffbc6d2d9314debff14 Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Mon, 17 Jul 2023 13:18:03 -0400 Subject: [PATCH 4/9] fix readme verification --- sdk/storage/Azure.Storage.DataMovement.Blobs/README.md | 2 +- sdk/storage/Azure.Storage.DataMovement/README.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md index abdda46608f2..0aa7eae88704 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md @@ -47,7 +47,7 @@ The Azure.Storage.DataMovement.Blobs library uses clients from the Azure.Storage ***TODO*** -## Sample Usage +## Examples This section demonstrates usage of Data Movement for interacting with blob storage. diff --git a/sdk/storage/Azure.Storage.DataMovement/README.md b/sdk/storage/Azure.Storage.DataMovement/README.md index da080ded16a5..87f73f8155f0 100644 --- a/sdk/storage/Azure.Storage.DataMovement/README.md +++ b/sdk/storage/Azure.Storage.DataMovement/README.md @@ -36,6 +36,10 @@ Here's an example using the Azure CLI: az storage account create --name MyStorageAccount --resource-group MyResourceGroup --location westus --sku Standard_LRS ``` +### Authenticate the client + +Authentication is specific to the targeted storage service. Please see documentation for the individual services + ## Key concepts ***TODO*** From e8c012a77bb4023fd53e1ab3bcbca8869dce9ece Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Mon, 17 Jul 2023 14:03:18 -0400 Subject: [PATCH 5/9] fixes --- sdk/storage/Azure.Storage.DataMovement/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.DataMovement/README.md b/sdk/storage/Azure.Storage.DataMovement/README.md index 87f73f8155f0..2df7f8696373 100644 --- a/sdk/storage/Azure.Storage.DataMovement/README.md +++ b/sdk/storage/Azure.Storage.DataMovement/README.md @@ -44,7 +44,7 @@ Authentication is specific to the targeted storage service. Please see documenta ***TODO*** -## Sample Usage +## Examples This section demonstrates usage of Data Movement regardless of extension package. Package-specific information and usage samples can be found in that package's documentation. These examples will use local disk and Azure Blob Storage when specific resources are needed for demonstration purposes, but the topics here will apply to other packages. From 28372e0d09b407350577481ca60ed29dc4b9a93e Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Tue, 18 Jul 2023 10:07:22 -0400 Subject: [PATCH 6/9] spelling --- sdk/storage/Azure.Storage.DataMovement.Blobs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md index 0aa7eae88704..39e29de092f4 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md @@ -144,7 +144,7 @@ StorageResource pageBlobResource = new PageBlobStorageResource(pageBlobClient); StorageResource appendBlobResource = new AppendBlobStorageResource(appendBlobClient); ``` -Blob Storage `StorageResouce` objects can be constructed with optional "options" arguments specific to the type of resource. +Blob `StorageResource` objects can be constructed with optional "options" arguments specific to the type of resource. ```csharp BlobContainerClient containerClient; From 7aa13b6013697ebea7ab1fc36c773f7b232de4c2 Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Mon, 24 Jul 2023 16:58:53 -0400 Subject: [PATCH 7/9] add samples to validation --- .../Azure.Storage.DataMovement/README.md | 142 +++---- ....Storage.DataMovement.Samples.Tests.csproj | 40 ++ .../samples/Sample01b_HelloWorldAsync.cs | 391 ++++++++++++++++++ 3 files changed, 499 insertions(+), 74 deletions(-) create mode 100644 sdk/storage/Azure.Storage.DataMovement/samples/Azure.Storage.DataMovement.Samples.Tests.csproj create mode 100644 sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_HelloWorldAsync.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/README.md b/sdk/storage/Azure.Storage.DataMovement/README.md index 6ff76f0eeced..76c1a429810c 100644 --- a/sdk/storage/Azure.Storage.DataMovement/README.md +++ b/sdk/storage/Azure.Storage.DataMovement/README.md @@ -67,9 +67,8 @@ This section demonstrates usage of Data Movement regardless of extension package Singleton usage of `TransferManager` is recommended. Providing `TransferManagerOptions` is optional. -```csharp -TransferManagerOptions options = new(); -TransferManger transferManager = new TransferManager(options); +```C# Snippet:CreateTransferManagerSimple +TransferManager transferManager = new TransferManager(new TransferManagerOptions()); ``` ### Starting New Transfers @@ -78,20 +77,19 @@ Transfers are defined by a source and destination `StorageResource`. There are t Configurations for accessing data are configured on the `StorageResource`. See further documentation for setting up and configuring your `StorageResource` objects. -```csharp -// provide these resources -StorageResource sourceResource, destinationResource; -// optionally configure options for this individual transfer -TransferOptions transferOptions = new(); -// optional cancellation token for transfer startup -// not for cancelling the transfer once started! -CancellationToken cancellationToken; - -DataTransfer dataTransfer = await transferManager.StartTransferAsync( - sourceResource, - destinationResource, - transferOptions, - cancellationToken); +A function that starts a transfer and then awaits it's completion: + +```C# Snippet:SimpleBlobUpload +async Task TransferAsync(StorageResource source, StorageResource destination, + TransferOptions transferOptions = default, CancellationToken cancellationToken = default) +{ + DataTransfer dataTransfer = await transferManager.StartTransferAsync( + source, + destination, + transferOptions, + cancellationToken); + await dataTransfer.WaitForCompletionAsync(cancellationToken); +} ``` ### Monitoring Transfers @@ -102,82 +100,82 @@ Transfers can be observed through several mechanisms, depending on your needs. Simple observation can be done through a `DataTransfer` instance representing an individual transfer. This is obtained on transfer start. You can also enumerate through all transfers on a `TransferManager`. -```csharp -TransferManager transferManager; +A function that writes the status of each transfer to console: -await foreach (DataTransfer transfer in transferManager.GetTransfersAsync()) { - // do something with transfer +```C# Snippet:EnumerateTransfers +async Task CheckTransfersAsync(TransferManager transferManager) +{ + await foreach (DataTransfer transfer in transferManager.GetTransfersAsync()) + { + using StreamWriter logStream = File.AppendText(logFile); + logStream.WriteLine(Enum.GetName(typeof(StorageTransferStatus), transfer.TransferStatus)); + } } ``` `DataTransfer` contains property `TransferStatus`. You can read this to determine the state of the transfer. States include queued for transfer, in progress, paused, completed, and more. -`DataTransfer` also exposes a task for transfer completion that can be awaited. - -```csharp -DataTransfer dataTransfer; - -await dataTransfer.AwaitCompletion(cancellationToken); -``` +`DataTransfer` also exposes a task for transfer completion, shown in [Starting New Transfers](#starting-new-transfers). #### With Events via `TransferOptions` When starting a transfer, `TransferOptions` contains multiple events that can be listened to for observation. Below demonstrates listening to the event for individual file completion and logging the result. -```csharp -TransferManager transferManager; -StorageResource sourceResource; -StorageResource destinationResource; +A function that listens to status events for a given transfer: -TransferOptions transferOptions = new(); -transferOptions.SingleTransferCompleted += (SingleTransferCompletedEventArgs args) => +```C# Snippet:ListenToTransferEvents +async Task ListenToTransfersAsync(TransferManager transferManager, + StorageResource source, StorageResource destination) { - using (StreamWriter logStream = File.AppendText(logFile)) + TransferOptions transferOptions = new(); + transferOptions.SingleTransferCompleted += (SingleTransferCompletedEventArgs args) => { + using StreamWriter logStream = File.AppendText(logFile); logStream.WriteLine($"File Completed Transfer: {args.SourceResource.Path}"); - } - return Task.CompletedTask; + return Task.CompletedTask; + }; + DataTransfer transfer = await transferManager.StartTransferAsync( + source, + destination, + transferOptions); } -DataTransfer transfer = transferManager.StartTransferAsync( - sourceResource, - destinationResource, - transferOptions, - cancellationToken); ``` #### With IProgress via `TransferOptions` When starting a transfer, `TransferOptions` allows setting a progress handler that contains the progress information for the overall transfer. Granular progress updates will be communicated to the provided `IProgress` instance. -```csharp -TransferOptions transferOptions = new() +A function that listens to progress updates for a given transfer with a supplied `IProgress`: + +```C# Snippet:TODO +async Task ListenToProgressAsync(TransferManager transferManager, IProgress progress, + StorageResource source, StorageResource destination) { - ProgressHandler = new Progress(storageTransferProgress => - { - // handle progress update - }), - // optionally include the below if progress updates on bytes transferred are desired - ProgressHandlerOptions = new() + TransferOptions transferOptions = new() { - TrackBytesTransferred = true - } + ProgressHandler = progress, + // optionally include the below if progress updates on total bytes transferred are desired + ProgressHandlerOptions = new() + { + TrackBytesTransferred = true + } + }; + DataTransfer transfer = await transferManager.StartTransferAsync( + source, + destination, + transferOptions); } ``` ### Pausing transfers -Transfers can be paused either by a given `DataTransfer` or through the `TransferManager` handling the transfer. - -```csharp -DataTransfer dataTransfer; +Transfers can be paused either by a given `DataTransfer` or through the `TransferManager` handling the transfer by referencing the transfer ID. The ID can be found on the `DataTransfer` object you recieved upon transfer start. +```C# Snippet:PauseFromTransfer await dataTransfer.PauseIfRunningAsync(cancellationToken); ``` -```csharp -TransferManager transferManager; -string transferId; - +```C# Snippet:PauseFromManager await transferManager.PauseTransferIfRunningAsync(transferId, cancellationToken); ``` @@ -185,9 +183,7 @@ await transferManager.PauseTransferIfRunningAsync(transferId, cancellationToken) Transfer progress is persisted such that it can resume from where it left off. No persisted knowledge is required from your code. The below sample queries a `TransferManager` for information on all resumable transfers and recreates the properly configured resources for these transfers using a helper method we'll define next. It then resumes each of those transfers with the given ID and puts the resulting `DataTransfer` objects into a list. -```csharp -TransferManager transferManager; - +```C# Snippet:ResumeAllTransfers List resumedTransfers = new(); await foreach (DataTransferProperties transferProperties in transferManager.GetResumableTransfersAsync()) { @@ -202,7 +198,7 @@ The above sample's `MakeResourcesAsync` method is defined below. Different `Data Note these resources return a "provider" rather than the resource itself. The provider can make the resource using a credential argument based on resource information (or some other value that was not persisted), rather than create an unauthenticated `StorageResource`. More information on this can be found in applicable packages. -```csharp +```C# Snippet:RehydrateResources async Task<(StorageResource Source, StorageResource Destination)> MakeResourcesAsync(DataTransferProperties info) { StorageResource sourceResource = null, destinationResource = null; @@ -212,8 +208,8 @@ async Task<(StorageResource Source, StorageResource Destination)> MakeResourcesA out BlobStorageResourceProvider blobSrcProvider, out BlobStorageResourceProvider blobDstProvider)) { - sourceResource ??= await blobSrcProvider?.MakeResourceAsync(); - destinationResource ??= await blobSrcProvider?.MakeResourceAsync(); + sourceResource ??= await blobSrcProvider?.MakeResourceAsync(credential); + destinationResource ??= await blobSrcProvider?.MakeResourceAsync(credential); } // ask DataMovement if it can recreate source or destination resources to local storage if (LocalStorageResources.TryGetResourceProviders( @@ -232,13 +228,11 @@ async Task<(StorageResource Source, StorageResource Destination)> MakeResourcesA Transfer failure can be observed by checking the `DataTransfer` status upon completion, or by listening to failure events on the transfer. While checking the `DataTransfer` may be sufficient for handling single-file transfer failures, event listening is recommended for container transfers. -Below logs failure for a single transnfer. +Below logs failure for a single transfer by checking its status after completion. -```csharp -DataTransfer transfer; - -await transfer.AwaitCompletion(); -if (transfer.TransferStatus == StorageTransferStatus.CompletedWithFailedTransfers) +```C# Snippet:LogTotalTransferFailure +await dataTransfer.WaitForCompletionAsync(); +if (dataTransfer.TransferStatus == StorageTransferStatus.CompletedWithFailedTransfers) { using (StreamWriter logStream = File.AppendText(logFile)) { @@ -247,9 +241,9 @@ if (transfer.TransferStatus == StorageTransferStatus.CompletedWithFailedTransfer } ``` -Below logs individual failures in a container transfer. +Below logs individual failures in a container transfer via `TransferOptions` events. -```csharp +```C# Snippet:LogIndividualTransferFailures transferOptions.TransferFailed += (TransferFailedEventArgs args) => { using (StreamWriter logStream = File.AppendText(logFile)) diff --git a/sdk/storage/Azure.Storage.DataMovement/samples/Azure.Storage.DataMovement.Samples.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement/samples/Azure.Storage.DataMovement.Samples.Tests.csproj new file mode 100644 index 000000000000..1470c83f1602 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/samples/Azure.Storage.DataMovement.Samples.Tests.csproj @@ -0,0 +1,40 @@ + + + $(RequiredTargetFrameworks) + Microsoft Azure.Storage.DataMovement client library samples + false + + + + + + + + + + + + PreserveNewest + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_HelloWorldAsync.cs b/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_HelloWorldAsync.cs new file mode 100644 index 000000000000..34636e021a3a --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_HelloWorldAsync.cs @@ -0,0 +1,391 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading.Tasks; +using Azure.Storage.Blobs.Specialized; +using Azure.Storage.Sas; +using NUnit.Framework; +using Azure.Core; +using Azure.Identity; +using System.Threading; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs; +using System.Collections.Generic; +using System.Linq; + +namespace Azure.Storage.DataMovement.Blobs.Samples +{ + /// + /// Basic Azure Blob Storage samples. + /// + public class Sample01b_HelloWorldAsync : SampleTest + { + public Random _rand = new Random(); + + /// + /// Use a connection string to connect to a Storage account and upload two single blobs. + /// + [Test] + public async Task UploadSingle_ConnectionStringAsync() + { + // Create a temporary Lorem Ipsum file on disk that we can upload + string sourceLocalPath = CreateTempFile(SampleFileContent); + + // Get a connection string to our Azure Storage account. You can + // obtain your connection string from the Azure Portal (click + // Access Keys under Settings in the Portal Storage account blade) + // or using the Azure CLI with: + // + // az storage account show-connection-string --name --resource-group + // + // And you can provide the connection string to your application + // using an environment variable. + + string connectionString = ConnectionString; + string containerName = Randomize("sample-container"); + + // Create a client that can authenticate with a connection string + BlobContainerClient container = new BlobContainerClient(connectionString, containerName); + await container.CreateIfNotExistsAsync(); + try + { + // Get a reference to a source local file + StorageResource sourceResource = new LocalFileStorageResource(sourceLocalPath); + + // Get a reference to a destination blobs + BlockBlobClient destinationBlob = container.GetBlockBlobClient("sample-blob"); + StorageResource destinationResource = new BlockBlobStorageResource(destinationBlob); + + // Create the transfer manager + #region Snippet:CreateTransferManagerSimple + TransferManager transferManager = new TransferManager(new TransferManagerOptions()); + #endregion + + // Create simple transfer single blob upload job + #region Snippet:SimpleBlobUpload + async Task TransferAsync(StorageResource source, StorageResource destination, + TransferOptions transferOptions = default, CancellationToken cancellationToken = default) + { + DataTransfer dataTransfer = await transferManager.StartTransferAsync( + source, + destination, + transferOptions, + cancellationToken); + await dataTransfer.WaitForCompletionAsync(cancellationToken); + } + #endregion + + await TransferAsync( + new LocalFileStorageResource(sourceLocalPath), + new BlockBlobStorageResource(destinationBlob)); + } + finally + { + await container.DeleteIfExistsAsync(); + } + } + + [Test] + public async Task MonitorUploadAsync() + { + string sourceLocalPath = CreateTempFile(SampleFileContent); + BlobContainerClient container = new BlobContainerClient(ConnectionString, Randomize("sample-container")); + await container.CreateIfNotExistsAsync(); + + try + { + // Get a reference to a source local file + StorageResource sourceResource = new LocalFileStorageResource(sourceLocalPath); + + // Get a reference to a destination blob + TransferManager transferManager = new TransferManager(); + + string logFile = CreateTempPath(); + + #region Snippet:EnumerateTransfers + async Task CheckTransfersAsync(TransferManager transferManager) + { + await foreach (DataTransfer transfer in transferManager.GetTransfersAsync()) + { + using StreamWriter logStream = File.AppendText(logFile); + logStream.WriteLine(Enum.GetName(typeof(StorageTransferStatus), transfer.TransferStatus)); + } + } + #endregion + + #region Snippet:ListenToTransferEvents + async Task ListenToTransfersAsync(TransferManager transferManager, + StorageResource source, StorageResource destination) + { + TransferOptions transferOptions = new(); + transferOptions.SingleTransferCompleted += (SingleTransferCompletedEventArgs args) => + { + using StreamWriter logStream = File.AppendText(logFile); + logStream.WriteLine($"File Completed Transfer: {args.SourceResource.Path}"); + return Task.CompletedTask; + }; + return await transferManager.StartTransferAsync( + source, + destination, + transferOptions); + } + #endregion + + #region Snippet:ListenToProgress + async Task ListenToProgressAsync(TransferManager transferManager, IProgress progress, + StorageResource source, StorageResource destination) + { + TransferOptions transferOptions = new() + { + ProgressHandler = progress, + // optionally include the below if progress updates on bytes transferred are desired + ProgressHandlerOptions = new() + { + TrackBytesTransferred = true + } + }; + return await transferManager.StartTransferAsync( + source, + destination, + transferOptions); + } + #endregion + + StorageResource destinationResource1 = new BlockBlobStorageResource(container.GetBlockBlobClient("sample-blob-1")); + StorageResource destinationResource2 = new BlockBlobStorageResource(container.GetBlockBlobClient("sample-blob-2")); + DataTransfer t1 = await ListenToTransfersAsync(transferManager, sourceResource, destinationResource1); + DataTransfer t2 = await ListenToProgressAsync(transferManager, new Progress(p => {}), sourceResource, destinationResource2); + await CheckTransfersAsync(transferManager); + await t1.WaitForCompletionAsync(); + await t2.WaitForCompletionAsync(); + } + finally + { + await container.DeleteIfExistsAsync(); + } + } + + [Test] + public async Task PauseTransferAsync() + { + string sourceLocalPath = CreateTempFile(SampleFileContent); + BlobContainerClient container = new BlobContainerClient(ConnectionString, Randomize("sample-container")); + await container.CreateIfNotExistsAsync(); + + try + { + StorageResource sourceResource = new LocalFileStorageResource(sourceLocalPath); + StorageResource destinationResource = new BlockBlobStorageResource(container.GetBlockBlobClient("sample-blob")); + + // Get a reference to a destination blob + TransferManager transferManager = new TransferManager(); + DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource, destinationResource); + + CancellationToken cancellationToken = CancellationToken.None; + #region Snippet:PauseFromTransfer + await dataTransfer.PauseIfRunningAsync(cancellationToken); + #endregion + + string transferId = dataTransfer.Id; + #region Snippet:PauseFromManager + await transferManager.PauseTransferIfRunningAsync(transferId, cancellationToken); + #endregion + } + finally + { + await container.DeleteIfExistsAsync(); + } + } + + [Test] + public async Task ResumeTransferAsync() + { + string sourceLocalPath = CreateTempFile(SampleFileContent); + BlobContainerClient container = new BlobContainerClient(ConnectionString, Randomize("sample-container")); + await container.CreateIfNotExistsAsync(); + + try + { + StorageResource sourceResource = new LocalFileStorageResource(sourceLocalPath); + StorageResource destinationResource = new BlockBlobStorageResource(container.GetBlockBlobClient("sample-blob")); + + StorageSharedKeyCredential credential = new(StorageAccountName, StorageAccountKey); + #region Snippet:RehydrateResources + async Task<(StorageResource Source, StorageResource Destination)> MakeResourcesAsync(DataTransferProperties info) + { + StorageResource sourceResource = null, destinationResource = null; + // ask DataMovement.Blobs if it can recreate source or destination resources to Blob Storage + if (BlobStorageResources.TryGetResourceProviders( + info, + out BlobStorageResourceProvider blobSrcProvider, + out BlobStorageResourceProvider blobDstProvider)) + { + sourceResource ??= await blobSrcProvider?.MakeResourceAsync(credential); + destinationResource ??= await blobSrcProvider?.MakeResourceAsync(credential); + } + // ask DataMovement if it can recreate source or destination resources to local storage + if (LocalStorageResources.TryGetResourceProviders( + info, + out LocalStorageResourceProvider localSrcProvider, + out LocalStorageResourceProvider localDstProvider)) + { + sourceResource ??= localSrcProvider?.MakeResource(); + destinationResource ??= localDstProvider?.MakeResource(); + } + return (sourceResource, destinationResource); + } + #endregion + + // Get a reference to a destination blob + TransferManager transferManager = new TransferManager(); + DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource, destinationResource); + + #region Snippet:ResumeAllTransfers + List resumedTransfers = new(); + await foreach (DataTransferProperties transferProperties in transferManager.GetResumableTransfersAsync()) + { + (StorageResource resumeSource, StorageResource resumeDestination) = await MakeResourcesAsync(transferProperties); + resumedTransfers.Add(await transferManager.ResumeTransferAsync(transferProperties.TransferId, resumeSource, resumeDestination)); + } + #endregion + + await Task.WhenAll(resumedTransfers.Select(transfer => transfer.WaitForCompletionAsync())); + } + finally + { + await container.DeleteIfExistsAsync(); + } + } + + [Test] + public async Task HandleFailedTransfer() + { + string sourceLocalPath = CreateTempFile(SampleFileContent); + BlobContainerClient container = new BlobContainerClient(ConnectionString, Randomize("sample-container")); + await container.CreateIfNotExistsAsync(); + string logFile = CreateTempPath(); + + try + { + StorageResource sourceResource = new LocalFileStorageResource(sourceLocalPath); + StorageResource destinationResource = new BlockBlobStorageResource(container.GetBlockBlobClient("sample-blob")); + + TransferOptions transferOptions = new(); + #region Snippet:LogIndividualTransferFailures + transferOptions.TransferFailed += (TransferFailedEventArgs args) => + { + using (StreamWriter logStream = File.AppendText(logFile)) + { + // Specifying specific resources that failed, since its a directory transfer + // maybe only one file failed out of many + logStream.WriteLine($"Exception occured with TransferId: {args.TransferId}," + + $"Source Resource: {args.SourceResource.Path}, +" + + $"Destination Resource: {args.DestinationResource.Path}," + + $"Exception Message: {args.Exception.Message}"); + } + return Task.CompletedTask; + }; + #endregion + + TransferManager transferManager = new TransferManager(); + DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource, destinationResource); + + #region Snippet:LogTotalTransferFailure + await dataTransfer.WaitForCompletionAsync(); + if (dataTransfer.TransferStatus == StorageTransferStatus.CompletedWithFailedTransfers) + { + using (StreamWriter logStream = File.AppendText(logFile)) + { + logStream.WriteLine($"Failure for TransferId: {args.TransferId}"); + } + } + #endregion + } + finally + { + await container.DeleteIfExistsAsync(); + } + } + + public async Task CreateBlobContainerTestDirectory(BlobContainerClient client, int depth = 0, string basePath = default) + { + basePath = basePath ?? Path.GetTempFileName(); + + var dirPath = string.IsNullOrEmpty(basePath) ? Path.GetTempFileName() : $"{basePath}/{Path.GetTempFileName()}"; + + await CreateBlobTestFiles(client, dirPath, 5); + + if (depth > 0) + { + await CreateBlobContainerTestDirectory(client, --depth, dirPath); + } + + return dirPath; + } + + public async Task CreateBlobTestFiles(BlobContainerClient client, string dirPath = default, int count = 1) + { + var buff = new byte[1000]; + + for (int i = 0; i < count; i++) + { + var blobPath = string.IsNullOrEmpty(dirPath) ? $"{Path.GetTempFileName()}.txt" : $"{dirPath}/{Path.GetTempFileName()}.txt"; + + _rand.NextBytes(buff); + + await client.UploadBlobAsync(blobPath, new MemoryStream(buff)); + } + } + + public string CreateLocalTestDirectory(int depth = 0, string basePath = default) + { + basePath = basePath ?? Path.GetTempPath(); + + var dirPath = Path.Combine(basePath, Path.GetTempFileName()); + + Directory.CreateDirectory(dirPath); + + CreateLocalTestFiles(dirPath, 5); + + if (depth > 0) + { + CreateLocalTestDirectory(--depth, dirPath); + } + + return dirPath; + } + + public void CreateLocalTestFiles(string dirPath, int count = 1) + { + var buff = new byte[1000]; + + for (int i = 0; i < count; i++) + { + var filePath = Path.Combine(dirPath, Path.GetTempFileName() + ".txt"); + + _rand.NextBytes(buff); + + File.WriteAllText(filePath, Convert.ToBase64String(buff)); + } + } + + public struct StoredCredentials + { + public StorageResourceContainer SourceContainer { get; set; } + public StorageResourceContainer DestinationContainer { get; set; } + + public StoredCredentials( + StorageResourceContainer sourceContainer, + StorageResourceContainer destinationContainer) + { + SourceContainer = sourceContainer; + DestinationContainer = destinationContainer; + } + } + } +} From c71b0534b15265ea6c1acb3cd6261fa83209cb14 Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Tue, 25 Jul 2023 10:35:51 -0400 Subject: [PATCH 8/9] update snippets base package --- .../Azure.Storage.DataMovement/README.md | 18 +++++++++--------- .../samples/Sample01b_HelloWorldAsync.cs | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement/README.md b/sdk/storage/Azure.Storage.DataMovement/README.md index 76c1a429810c..79820866edff 100644 --- a/sdk/storage/Azure.Storage.DataMovement/README.md +++ b/sdk/storage/Azure.Storage.DataMovement/README.md @@ -67,7 +67,7 @@ This section demonstrates usage of Data Movement regardless of extension package Singleton usage of `TransferManager` is recommended. Providing `TransferManagerOptions` is optional. -```C# Snippet:CreateTransferManagerSimple +```C# Snippet:CreateTransferManagerSimple_BasePackage TransferManager transferManager = new TransferManager(new TransferManagerOptions()); ``` @@ -79,7 +79,7 @@ Configurations for accessing data are configured on the `StorageResource`. See f A function that starts a transfer and then awaits it's completion: -```C# Snippet:SimpleBlobUpload +```C# Snippet:SimpleBlobUpload_BasePackage async Task TransferAsync(StorageResource source, StorageResource destination, TransferOptions transferOptions = default, CancellationToken cancellationToken = default) { @@ -124,7 +124,7 @@ When starting a transfer, `TransferOptions` contains multiple events that can be A function that listens to status events for a given transfer: ```C# Snippet:ListenToTransferEvents -async Task ListenToTransfersAsync(TransferManager transferManager, +async Task ListenToTransfersAsync(TransferManager transferManager, StorageResource source, StorageResource destination) { TransferOptions transferOptions = new(); @@ -134,7 +134,7 @@ async Task ListenToTransfersAsync(TransferManager transferManager, logStream.WriteLine($"File Completed Transfer: {args.SourceResource.Path}"); return Task.CompletedTask; }; - DataTransfer transfer = await transferManager.StartTransferAsync( + return await transferManager.StartTransferAsync( source, destination, transferOptions); @@ -147,20 +147,20 @@ When starting a transfer, `TransferOptions` allows setting a progress handler th A function that listens to progress updates for a given transfer with a supplied `IProgress`: -```C# Snippet:TODO -async Task ListenToProgressAsync(TransferManager transferManager, IProgress progress, +```C# Snippet:ListenToProgress +async Task ListenToProgressAsync(TransferManager transferManager, IProgress progress, StorageResource source, StorageResource destination) { TransferOptions transferOptions = new() { ProgressHandler = progress, - // optionally include the below if progress updates on total bytes transferred are desired + // optionally include the below if progress updates on bytes transferred are desired ProgressHandlerOptions = new() { TrackBytesTransferred = true } }; - DataTransfer transfer = await transferManager.StartTransferAsync( + return await transferManager.StartTransferAsync( source, destination, transferOptions); @@ -236,7 +236,7 @@ if (dataTransfer.TransferStatus == StorageTransferStatus.CompletedWithFailedTran { using (StreamWriter logStream = File.AppendText(logFile)) { - logStream.WriteLine($"Failure for TransferId: {args.TransferId}"); + logStream.WriteLine($"Failure for TransferId: {dataTransfer.Id}"); } } ``` diff --git a/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_HelloWorldAsync.cs b/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_HelloWorldAsync.cs index 34636e021a3a..bf4f900b7cd1 100644 --- a/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_HelloWorldAsync.cs +++ b/sdk/storage/Azure.Storage.DataMovement/samples/Sample01b_HelloWorldAsync.cs @@ -59,12 +59,12 @@ public async Task UploadSingle_ConnectionStringAsync() StorageResource destinationResource = new BlockBlobStorageResource(destinationBlob); // Create the transfer manager - #region Snippet:CreateTransferManagerSimple + #region Snippet:CreateTransferManagerSimple_BasePackage TransferManager transferManager = new TransferManager(new TransferManagerOptions()); #endregion // Create simple transfer single blob upload job - #region Snippet:SimpleBlobUpload + #region Snippet:SimpleBlobUpload_BasePackage async Task TransferAsync(StorageResource source, StorageResource destination, TransferOptions transferOptions = default, CancellationToken cancellationToken = default) { @@ -301,7 +301,7 @@ public async Task HandleFailedTransfer() { using (StreamWriter logStream = File.AppendText(logFile)) { - logStream.WriteLine($"Failure for TransferId: {args.TransferId}"); + logStream.WriteLine($"Failure for TransferId: {dataTransfer.Id}"); } } #endregion From 9e34393f439ff2f0b1e24e7f2dc361e36f989940 Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Tue, 25 Jul 2023 12:34:07 -0400 Subject: [PATCH 9/9] blobs samples generated --- .../README.md | 109 +++----- .../samples/Sample01b_HelloWorldAsync.cs | 260 ++++++++++++++---- 2 files changed, 238 insertions(+), 131 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md index 56d84b3fb2b8..beee7d543ee0 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md @@ -147,13 +147,8 @@ await tranfer.WaitForCompletionAsync(); Azure.Storage.DataMovement.Blobs exposes a `StorageResource` for each type of blob (block, page, append) as well as a blob container. Storage resources are initialized with the appropriate client object from Azure.Storage.Blobs. -```csharp -BlobContainerClient containerClient; -BlockBlobClient blockBlobClient; -PageBlobClient pageBlobClient; -AppendBlobClient appendBlobClient; - -StorageResource containerResource = new BlobStorageResourceContainer(containerClient); +```C# Snippet:ResourceConstruction_Blobs +StorageResource containerResource = new BlobStorageResourceContainer(blobContainerClient); StorageResource blockBlobResource = new BlockBlobStorageResource(blockBlobClient); StorageResource pageBlobResource = new PageBlobStorageResource(pageBlobClient); StorageResource appendBlobResource = new AppendBlobStorageResource(appendBlobClient); @@ -161,39 +156,35 @@ StorageResource appendBlobResource = new AppendBlobStorageResource(appendBlobCli Blob `StorageResource` objects can be constructed with optional "options" arguments specific to the type of resource. -```csharp -BlobContainerClient containerClient; -BlobStorageResourceContainerOptions options = new() +```C# Snippet:ResourceConstruction_Blobs_WithOptions_VirtualDirectory +BlobStorageResourceContainerOptions virtualDirectoryOptions = new() { DirectoryPrefix = "blob/directory/prefix" -} +}; StorageResource virtualDirectoryResource = new BlobStorageResourceContainer( - containerClient, - options); + blobContainerClient, + virtualDirectoryOptions); ``` -```csharp -BlockBlobClient blockBlobClient; -string leaseId; -BlockBlobStorageResourceOptions options = new() +```C# Snippet:ResourceConstruction_Blobs_WithOptions_BlockBlob +BlockBlobStorageResourceOptions leasedResourceOptions = new() { SourceConditions = new() { LeaseId = leaseId } -} - -StorageResource virtualDirectoryResource = new BlockBlobStorageResource( +}; +StorageResource leasedBlockBlobResource = new BlockBlobStorageResource( blockBlobClient, - options); + leasedResourceOptions); ``` When resuming a transfer, a credential to Azure Storage is likely needed. Credentials are not persisted by the transfer manager. When using `BlobStorageResources.TryGetResourceProviders()` to recreate a `StorageResource` for resume, the returned provider can create the resource with a credential specified by the calling code. This allows for workflows like scoping generation of a Shared Access Signature to the given resource path. Your application should provide its own mechanism for getting the appropriate credential, represented by `GenerateMySasCredential()` in the sample below. -```csharp -AzureSasCredential GenerateMySasCredential(string blobUri); - +```C# Snippet:RehydrateBlobResource +StorageResource sourceResource = null; +StorageResource destinationResource = null; if (BlobStorageResources.TryGetResourceProviders( info, out BlobStorageResourceProvider blobSrcProvider, @@ -212,10 +203,7 @@ An upload takes place between a local file `StorageResource` as source and blob Upload a block blob. -```csharp -string sourceLocalPath; -BlockBlobClient destinationBlob; - +```C# Snippet:SimpleBlobUpload DataTransfer dataTransfer = await transferManager.StartTransferAsync( sourceResource: new LocalFileStorageResource(sourceLocalPath), destinationResource: new BlockBlobStorageResource(destinationBlob)); @@ -224,22 +212,19 @@ await dataTransfer.WaitForCompletionAsync(); Upload a directory as a specific blob type. -```csharp -string sourceLocalPath; -BlobContainerClient destinationContainer; -string optionalDestinationPrefix; - +```C# Snippet:SimpleDirectoryUpload DataTransfer dataTransfer = await transferManager.StartTransferAsync( - sourceResource: new LocalDirectoryStorageResourceContainer(sourceLocalPath), + sourceResource: new LocalDirectoryStorageResourceContainer(sourcePath), destinationResource: new BlobStorageResourceContainer( - destinationContainer, + blobContainerClient, new BlobStorageResourceContainerOptions() { // Block blobs are the default if not specified BlobType = BlobType.Block, - DirectoryPrefix = optionalDestinationPrefix - })); -await dataTransfer.AwaitCompletion(); + DirectoryPrefix = optionalDestinationPrefix, + }), + transferOptions: options); +await dataTransfer.WaitForCompletionAsync(); ``` ### Download @@ -248,32 +233,25 @@ A download takes place between a blob `StorageResource` as source and local file Download a block blob. -```csharp -BlockBlobClient sourceBlob; -string destinationLocalPath; - +```C# Snippet:SimpleBlockBlobDownload DataTransfer dataTransfer = await transferManager.StartTransferAsync( - sourceResource: new BlockBlobStorageResource(sourceBlob), - destinationResource: new LocalFileStorageResource(destinationLocalPath)); -await dataTransfer.AwaitCompletion(); + sourceResource: new BlockBlobStorageResource(sourceBlobClient), + destinationResource: new LocalFileStorageResource(downloadPath)); +await dataTransfer.WaitForCompletionAsync(); ``` Download a container which may contain a mix of blob types. -```csharp -BlobContainerClient sourceContainer; -string optionalSourcePrefix; -string destinationLocalPath; - +```C# Snippet:SimpleDirectoryDownload_Blob DataTransfer dataTransfer = await transferManager.StartTransferAsync( sourceResource: new BlobStorageResourceContainer( - sourceContainer, + blobContainerClient, new BlobStorageResourceContainerOptions() { DirectoryPrefix = optionalSourcePrefix }), - destinationResource: new LocalDirectoryStorageResourceContainer(destinationLocalPath)); -await dataTransfer.AwaitCompletion(); + destinationResource: new LocalDirectoryStorageResourceContainer(downloadPath)); +await dataTransfer.WaitForCompletionAsync(); ``` ### Blob Copy @@ -282,30 +260,22 @@ A copy takes place between two blob `StorageResource` instances. Copying between Copy a single blob. Note the change in blob type on this copy from block to append. -```csharp -BlockBlobClient sourceBlob; -AppendBlobClient destinationBlob; - +```C# Snippet:s2sCopyBlob DataTransfer dataTransfer = await transferManager.StartTransferAsync( - sourceResource: new BlockBlobStorageResource(sourceBlob), - destinationResource: new AppendBlobStorageResource(destinationBlob)); -await dataTransfer.AwaitCompletion(); + sourceResource: new BlockBlobStorageResource(sourceBlockBlobClient), + destinationResource: new AppendBlobStorageResource(destinationAppendBlobClient)); +await dataTransfer.WaitForCompletionAsync(); ``` Copy a blob container. -```csharp -BlobContainerClient sourceContainer; -string optionalSourcePrefix; -BlobContainerClient destinationContainer; -string optionalDestinationPrefix; - +```C# Snippet:s2sCopyBlobContainer DataTransfer dataTransfer = await transferManager.StartTransferAsync( - sourceResource: sourceResource: new BlobStorageResourceContainer( + sourceResource: new BlobStorageResourceContainer( sourceContainer, new BlobStorageResourceContainerOptions() { - DirectoryPrefix = optionalSourcePrefix + DirectoryPrefix = sourceDirectoryName }), destinationResource: new BlobStorageResourceContainer( destinationContainer, @@ -314,9 +284,8 @@ DataTransfer dataTransfer = await transferManager.StartTransferAsync( // all source blobs will be copied as a single type of destination blob // defaults to block blobs if unspecified BlobType = BlobType.Block, - DirectoryPrefix = optionalDestinationPrefix + DirectoryPrefix = downloadPath })); -await dataTransfer.AwaitCompletion(); ``` ## Troubleshooting diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs index f778be6a94ee..9ff24986ad81 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs @@ -24,6 +24,76 @@ public class Sample01b_HelloWorldAsync : SampleTest { public Random _rand = new Random(); + /// + /// Various ways of constructing blob resources. + /// + [Test] + public async Task ResourceConstructionDemonstration() + { + // Create a temporary Lorem Ipsum file on disk that we can upload + string sourceLocalPath = CreateTempFile(SampleFileContent); + string connectionString = ConnectionString; + string containerName = Randomize("sample-container"); + + // Create a client that can authenticate with a connection string + BlobContainerClient blobContainerClient = new BlobContainerClient(connectionString, containerName); + await blobContainerClient.CreateIfNotExistsAsync(); + try + { + TransferManager transferManager = new TransferManager(); + + // Get a reference to a source local file + StorageResource sourceResource = new LocalFileStorageResource(sourceLocalPath); + + // Get a reference to a destination blobs + BlockBlobClient blockBlobClient = blobContainerClient.GetBlockBlobClient("sample-blob-block"); + PageBlobClient pageBlobClient = blobContainerClient.GetPageBlobClient("sample-blob-page"); + AppendBlobClient appendBlobClient = blobContainerClient.GetAppendBlobClient("sample-blob-append"); + + await pageBlobClient.CreateAsync(Constants.KB); + await appendBlobClient.CreateAsync(); + + // Construct simple blob resources for data movement + #region Snippet:ResourceConstruction_Blobs + StorageResource containerResource = new BlobStorageResourceContainer(blobContainerClient); + StorageResource blockBlobResource = new BlockBlobStorageResource(blockBlobClient); + StorageResource pageBlobResource = new PageBlobStorageResource(pageBlobClient); + StorageResource appendBlobResource = new AppendBlobStorageResource(appendBlobClient); + #endregion + + // Construct a blob container resource that is scoped to a blob prefix (virtual directory). + #region Snippet:ResourceConstruction_Blobs_WithOptions_VirtualDirectory + BlobStorageResourceContainerOptions virtualDirectoryOptions = new() + { + DirectoryPrefix = "blob/directory/prefix" + }; + + StorageResource virtualDirectoryResource = new BlobStorageResourceContainer( + blobContainerClient, + virtualDirectoryOptions); + #endregion + + // Construct a blob resource that uses a given lease ID + string leaseId = "mylease"; + #region Snippet:ResourceConstruction_Blobs_WithOptions_BlockBlob + BlockBlobStorageResourceOptions leasedResourceOptions = new() + { + SourceConditions = new() + { + LeaseId = leaseId + } + }; + StorageResource leasedBlockBlobResource = new BlockBlobStorageResource( + blockBlobClient, + leasedResourceOptions); + #endregion + } + finally + { + await blobContainerClient.DeleteIfExistsAsync(); + } + } + /// /// Use a connection string to connect to a Storage account and upload two single blobs. /// @@ -57,11 +127,7 @@ public async Task UploadSingle_ConnectionStringAsync() // Get a reference to a destination blobs BlockBlobClient destinationBlob = container.GetBlockBlobClient("sample-blob"); StorageResource destinationResource = new BlockBlobStorageResource(destinationBlob); - - // Upload file data - #region Snippet:CreateTransferManagerSimple TransferManager transferManager = new TransferManager(new TransferManagerOptions()); - #endregion // Create simple transfer single blob upload job #region Snippet:SimpleBlobUpload @@ -77,6 +143,60 @@ public async Task UploadSingle_ConnectionStringAsync() } } + [Test] + public async Task RehydrateBlobResource_SasAsync() + { + string sourceLocalPath = CreateTempFile(SampleFileContent); + string connectionString = ConnectionString; + string containerName = Randomize("sample-container"); + + BlobContainerClient container = new BlobContainerClient(connectionString, containerName); + await container.CreateIfNotExistsAsync(); + try + { + StorageResource source = new LocalFileStorageResource(sourceLocalPath); + StorageResource destination = new BlockBlobStorageResource(container.GetBlockBlobClient("sample-blob")); + TransferManager transferManager = new TransferManager(new TransferManagerOptions()); + + DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource: source, + destinationResource: destination); + await dataTransfer.PauseIfRunningAsync(); + + DataTransferProperties info = await transferManager.GetResumableTransfersAsync().FirstAsync(); + + StorageSharedKeyCredential sharedKeyCredential = new(StorageAccountName, StorageAccountKey); + AzureSasCredential GenerateMySasCredential(string blobUri) + { + Uri fullSasUri = new BlobClient(new Uri(blobUri), sharedKeyCredential).GenerateSasUri( + BlobSasPermissions.All, DateTimeOffset.Now + TimeSpan.FromDays(1)); + return new AzureSasCredential(fullSasUri.Query); + } + + #region Snippet:RehydrateBlobResource + StorageResource sourceResource = null; + StorageResource destinationResource = null; + if (BlobStorageResources.TryGetResourceProviders( + info, + out BlobStorageResourceProvider blobSrcProvider, + out BlobStorageResourceProvider blobDstProvider)) + { + sourceResource ??= await blobSrcProvider?.MakeResourceAsync( + GenerateMySasCredential(info.SourcePath)); + destinationResource ??= await blobSrcProvider?.MakeResourceAsync( + GenerateMySasCredential(info.DestinationPath)); + } + #endregion + + DataTransfer resumedTransfer = await transferManager.ResumeTransferAsync(dataTransfer.Id, sourceResource, destinationResource); + await resumedTransfer.WaitForCompletionAsync(); + } + finally + { + await container.DeleteIfExistsAsync(); + } + } + /// /// Use a shared key to access a Storage Account to download two separate blobs. /// @@ -116,12 +236,12 @@ public async Task DownloadSingle_SharedKeyAuthAsync() try { // Get a reference to a source blobs and upload sample content to download - BlockBlobClient sourceBlob = container.GetBlockBlobClient("sample-blob"); + BlockBlobClient sourceBlobClient = container.GetBlockBlobClient("sample-blob"); BlockBlobClient sourceBlob2 = container.GetBlockBlobClient("sample-blob2"); using (FileStream stream = File.Open(originalPath, FileMode.Open)) { - await sourceBlob.UploadAsync(stream); + await sourceBlobClient.UploadAsync(stream); stream.Position = 0; await sourceBlob2.UploadAsync(stream); } @@ -134,23 +254,22 @@ public async Task DownloadSingle_SharedKeyAuthAsync() TransferManager transferManager = new TransferManager(options); // Simple Download Single Blob Job - + #region Snippet:SimpleBlockBlobDownload DataTransfer dataTransfer = await transferManager.StartTransferAsync( - sourceResource: new BlockBlobStorageResource(sourceBlob), + sourceResource: new BlockBlobStorageResource(sourceBlobClient), destinationResource: new LocalFileStorageResource(downloadPath)); await dataTransfer.WaitForCompletionAsync(); + #endregion - StorageResource sourceResource2 = new BlockBlobStorageResource(sourceBlob); + StorageResource sourceResource2 = new BlockBlobStorageResource(sourceBlobClient); StorageResource destinationResource2 = new LocalFileStorageResource(downloadPath2); - #region Snippet:BlockBlobDownloadOptions await transferManager.StartTransferAsync( - sourceResource: new BlockBlobStorageResource(sourceBlob, new BlockBlobStorageResourceOptions() + sourceResource: new BlockBlobStorageResource(sourceBlobClient, new BlockBlobStorageResourceOptions() { DestinationConditions = new BlobRequestConditions(){ LeaseId = "xyz" } }), destinationResource: new LocalFileStorageResource(downloadPath2)); - #endregion } finally { @@ -208,10 +327,10 @@ public async Task UploadDirectory_SasAsync() string containerName = Randomize("sample-container"); // Create a client that can authenticate with a connection string - BlobContainerClient container = service.GetBlobContainerClient(containerName); + BlobContainerClient blobContainerClient = service.GetBlobContainerClient(containerName); // Make a service request to verify we've successfully authenticated - await container.CreateIfNotExistsAsync(); + await blobContainerClient.CreateIfNotExistsAsync(); // Prepare for upload try @@ -220,10 +339,9 @@ public async Task UploadDirectory_SasAsync() StorageResourceContainer localDirectory = new LocalDirectoryStorageResourceContainer(sourcePath); // Get a storage resource to a destination blob directory StorageResourceContainer directoryDestination = new BlobStorageResourceContainer( - container, + blobContainerClient, new BlobStorageResourceContainerOptions() { DirectoryPrefix = "sample-directory" }); - #region Snippet:CreateTransferManagerWithOptions // Create BlobTransferManager with event handler in Options bag TransferManagerOptions transferManagerOptions = new TransferManagerOptions(); TransferOptions options = new TransferOptions() @@ -232,21 +350,27 @@ public async Task UploadDirectory_SasAsync() CreateMode = StorageResourceCreateMode.Overwrite, }; TransferManager transferManager = new TransferManager(transferManagerOptions); - #endregion - #region Snippet:SimpleDirectoryUpload // Create simple transfer directory upload job which uploads the directory and the contents of that directory + string optionalDestinationPrefix = "sample-directory2"; + #region Snippet:SimpleDirectoryUpload DataTransfer dataTransfer = await transferManager.StartTransferAsync( sourceResource: new LocalDirectoryStorageResourceContainer(sourcePath), destinationResource: new BlobStorageResourceContainer( - container, - new BlobStorageResourceContainerOptions() { DirectoryPrefix = "sample-directory2" }), + blobContainerClient, + new BlobStorageResourceContainerOptions() + { + // Block blobs are the default if not specified + BlobType = BlobType.Block, + DirectoryPrefix = optionalDestinationPrefix, + }), transferOptions: options); + await dataTransfer.WaitForCompletionAsync(); #endregion } finally { - await container.DeleteIfExistsAsync(); + await blobContainerClient.DeleteIfExistsAsync(); } } @@ -311,7 +435,6 @@ public async Task UploadDirectory_CompletedEventHandler() // Prepare for upload try { - #region Snippet:SimpleLoggingSample // Create BlobTransferManager with event handler in Options bag TransferManagerOptions options = new TransferManagerOptions(); TransferOptions transferOptions = new TransferOptions(); @@ -323,7 +446,6 @@ public async Task UploadDirectory_CompletedEventHandler() } return Task.CompletedTask; }; - #endregion TransferManager transferManager = new TransferManager(options); // Create simple transfer directory upload job which uploads the directory and the contents of that directory @@ -414,7 +536,6 @@ public async Task UploadDirectory_EventHandler_SasAsync() } return Task.CompletedTask; }; - #region Snippet:FailedEventDelegation transferOptions.TransferFailed += (TransferFailedEventArgs args) => { using (StreamWriter logStream = File.AppendText(logFile)) @@ -428,7 +549,6 @@ public async Task UploadDirectory_EventHandler_SasAsync() } return Task.CompletedTask; }; - #endregion TransferManager transferManager = new TransferManager(options); // Create simple transfer directory upload job which uploads the directory and the contents of that directory @@ -479,28 +599,28 @@ public async Task DownloadDirectory_EventHandler_ActiveDirectoryAuthAsync() string containerName = Randomize("sample-container"); // Create a client that can authenticate with a connection string - BlobContainerClient container = service.GetBlobContainerClient(containerName); + BlobContainerClient blobContainerClient = service.GetBlobContainerClient(containerName); // Make a service request to verify we've successfully authenticated - await container.CreateIfNotExistsAsync(); + await blobContainerClient.CreateIfNotExistsAsync(); // Prepare to download try { // Get a reference to a source blobs and upload sample content to download - StorageResourceContainer sourceDirectory = new BlobStorageResourceContainer(container, + StorageResourceContainer sourceDirectory = new BlobStorageResourceContainer(blobContainerClient, new BlobStorageResourceContainerOptions() { DirectoryPrefix = "sample-blob-directory" }); - StorageResourceContainer sourceDirectory2 = new BlobStorageResourceContainer(container, + StorageResourceContainer sourceDirectory2 = new BlobStorageResourceContainer(blobContainerClient, new BlobStorageResourceContainerOptions() { DirectoryPrefix = "sample-blob-directory2" }); StorageResourceContainer destinationDirectory = new LocalDirectoryStorageResourceContainer(downloadPath); StorageResourceContainer destinationDirectory2 = new LocalDirectoryStorageResourceContainer(downloadPath2); // Upload a couple of blobs so we have something to list - await container.UploadBlobAsync("first", File.OpenRead(CreateTempFile())); - await container.UploadBlobAsync("first/fourth", File.OpenRead(CreateTempFile())); - await container.UploadBlobAsync("first/fifth", File.OpenRead(CreateTempFile())); - await container.UploadBlobAsync("second", File.OpenRead(CreateTempFile())); - await container.UploadBlobAsync("third", File.OpenRead(CreateTempFile())); + await blobContainerClient.UploadBlobAsync("first", File.OpenRead(CreateTempFile())); + await blobContainerClient.UploadBlobAsync("first/fourth", File.OpenRead(CreateTempFile())); + await blobContainerClient.UploadBlobAsync("first/fifth", File.OpenRead(CreateTempFile())); + await blobContainerClient.UploadBlobAsync("second", File.OpenRead(CreateTempFile())); + await blobContainerClient.UploadBlobAsync("third", File.OpenRead(CreateTempFile())); // Create BlobTransferManager with event handler in Options bag TransferManagerOptions options = new TransferManagerOptions(); @@ -519,15 +639,22 @@ await transferManager.StartTransferAsync( sourceDirectory, destinationDirectory); // Create different download transfer - #region Snippet:SimpleDirectoryDownload - DataTransfer downloadDirectoryJobId2 = await transferManager.StartTransferAsync( - sourceDirectory2, - destinationDirectory2); + string optionalSourcePrefix = "sample-blob-directory2"; + #region Snippet:SimpleDirectoryDownload_Blob + DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource: new BlobStorageResourceContainer( + blobContainerClient, + new BlobStorageResourceContainerOptions() + { + DirectoryPrefix = optionalSourcePrefix + }), + destinationResource: new LocalDirectoryStorageResourceContainer(downloadPath)); + await dataTransfer.WaitForCompletionAsync(); #endregion } finally { - await container.DeleteIfExistsAsync(); + await blobContainerClient.DeleteIfExistsAsync(); } } @@ -564,31 +691,29 @@ public async Task CopySingle_ConnectionStringAsync() try { // Get a reference to a destination blobs - BlockBlobClient sourceBlob = container.GetBlockBlobClient("sample-blob"); + BlockBlobClient sourceBlockBlobClient = container.GetBlockBlobClient("sample-blob"); using (FileStream stream = File.Open(originalPath, FileMode.Open)) { - await sourceBlob.UploadAsync(stream); + await sourceBlockBlobClient.UploadAsync(stream); stream.Position = 0; } - StorageResource sourceResource = new BlockBlobStorageResource(sourceBlob); - BlockBlobClient destinationBlob = container.GetBlockBlobClient("sample-blob2"); - StorageResource destinationResource = new BlockBlobStorageResource(destinationBlob); + AppendBlobClient destinationAppendBlobClient = container.GetAppendBlobClient("sample-blob2"); // Upload file data TransferManager transferManager = new TransferManager(default); // Create simple transfer single blob upload job - DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource); - await transfer.WaitForCompletionAsync(); - - // Generous 10 second wait for our transfer to finish - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(10000); - await transfer.WaitForCompletionAsync(cancellationTokenSource.Token); + #region Snippet:s2sCopyBlob + DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource: new BlockBlobStorageResource(sourceBlockBlobClient), + destinationResource: new AppendBlobStorageResource(destinationAppendBlobClient)); + await dataTransfer.WaitForCompletionAsync(); + #endregion - Assert.IsTrue(await destinationBlob.ExistsAsync()); - Assert.AreEqual(transfer.TransferStatus, StorageTransferStatus.Completed); + Assert.IsTrue(await destinationAppendBlobClient.ExistsAsync()); + Assert.AreEqual(dataTransfer.TransferStatus, StorageTransferStatus.Completed); } finally { @@ -667,9 +792,28 @@ public async Task CopyDirectory() //await LogFailedFileAsync(args.SourceFileUri, args.DestinationFileClient.Uri, args.Exception.Message); }; #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - await transferManager.StartTransferAsync( - sourceDirectory1, - destinationDirectory1); + + // this is just a directory copy within a container, but they can be separate containers as well + BlobContainerClient sourceContainer = container; + BlobContainerClient destinationContainer = container; + #region Snippet:s2sCopyBlobContainer + DataTransfer dataTransfer = await transferManager.StartTransferAsync( + sourceResource: new BlobStorageResourceContainer( + sourceContainer, + new BlobStorageResourceContainerOptions() + { + DirectoryPrefix = sourceDirectoryName + }), + destinationResource: new BlobStorageResourceContainer( + destinationContainer, + new BlobStorageResourceContainerOptions() + { + // all source blobs will be copied as a single type of destination blob + // defaults to block blobs if unspecified + BlobType = BlobType.Block, + DirectoryPrefix = downloadPath + })); + #endregion } finally { @@ -707,7 +851,6 @@ public async Task PauseAndResumeAsync_ManagerId() StorageResource destinationResource = new LocalFileStorageResource(downloadPath); // Create simple transfer single blob download job - #region Snippet:TransferManagerTryPauseId_Async DataTransfer dataTransfer = await transferManager.StartTransferAsync( sourceResource: sourceResource, destinationResource: destinationResource); @@ -715,12 +858,10 @@ public async Task PauseAndResumeAsync_ManagerId() // Pause from the Transfer Manager using the Transfer Id await transferManager.PauseTransferIfRunningAsync(transferId); - #endregion Snippet:TransferManagerTryPauseId_Async StorageSharedKeyCredential GetMyCredential(string uri) => new StorageSharedKeyCredential(StorageAccountName, StorageAccountKey); - #region Snippet:TransferManagerResume_Async async Task<(StorageResource Source, StorageResource Destination)> MakeResourcesAsync(DataTransferProperties info) { StorageResource sourceResource = null, destinationResource = null; @@ -748,7 +889,6 @@ StorageSharedKeyCredential GetMyCredential(string uri) (StorageResource resumeSource, StorageResource resumeDestination) = await MakeResourcesAsync(transferProperties); resumedTransfers.Add(await transferManager.ResumeTransferAsync(transferProperties.TransferId, resumeSource, resumeDestination)); } - #endregion Snippet:TransferManagerResume_Async // Wait for download to finish await Task.WhenAll(resumedTransfers.Select(t => t.WaitForCompletionAsync())); @@ -797,14 +937,12 @@ public async Task PauseAndResumeAsync_DataTransferPause() StorageResource destinationResource = new LocalFileStorageResource(downloadPath); // Create simple transfer single blob download job - #region Snippet:DataTransferTryPause_Async DataTransfer dataTransfer = await transferManager.StartTransferAsync( sourceResource: sourceResource, destinationResource: destinationResource); // Pause from the DataTransfer object await dataTransfer.PauseIfRunningAsync(); - #endregion Snippet:DataTransferTryPause_Async DataTransfer resumedTransfer = await transferManager.ResumeTransferAsync( transferId: dataTransfer.Id,