From bf0e76631b6df569648a4a586a952aee3acb73e9 Mon Sep 17 00:00:00 2001 From: Bethany Date: Mon, 5 Feb 2024 13:19:02 -0500 Subject: [PATCH] Specify `Content-Type` for BlockBlob upload (#3119) * add content-type to block blob upload * Add content-type for sdk path * fix spacing * merge headers and only when file extension is .txt * add conditions * tweak conditions and path matching * pass in headers * add content-type for appendblob --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 86 +++++++++++++++++++--- 1 file changed, 77 insertions(+), 9 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 53338dd15a2..d206306f59f 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -222,14 +222,34 @@ private AppendBlobClient GetAppendBlobClient(string url) return new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts); } - private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) + private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken, Dictionary customHeaders = null) { if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage) { var blobClient = GetBlobClient(url); + var httpHeaders = new BlobHttpHeaders(); + if (customHeaders != null) + { + foreach (var header in customHeaders) + { + switch (header.Key) + { + case Constants.ContentTypeHeader: + httpHeaders.ContentType = header.Value; + break; + } + } + } try { - await blobClient.UploadAsync(file, cancellationToken); + await blobClient.UploadAsync(file, new BlobUploadOptions() + { + HttpHeaders = httpHeaders, + Conditions = new BlobRequestConditions + { + IfNoneMatch = new ETag("*") + } + }, cancellationToken); } catch (RequestFailedException e) { @@ -249,6 +269,14 @@ private async Task UploadBlockFileAsync(string url, string blobStorageType, File request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob); } + if (customHeaders != null) + { + foreach (var header in customHeaders) + { + request.Content.Headers.Add(header.Key, header.Value); + } + }; + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) { if (!response.IsSuccessStatusCode) @@ -259,14 +287,34 @@ private async Task UploadBlockFileAsync(string url, string blobStorageType, File } } - private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken) + private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken, Dictionary customHeaders = null) { if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage) { var appendBlobClient = GetAppendBlobClient(url); + var httpHeaders = new BlobHttpHeaders(); + if (customHeaders != null) + { + foreach (var header in customHeaders) + { + switch (header.Key) + { + case Constants.ContentTypeHeader: + httpHeaders.ContentType = header.Value; + break; + } + } + } try { - await appendBlobClient.CreateAsync(cancellationToken: cancellationToken); + await appendBlobClient.CreateAsync(new AppendBlobCreateOptions() + { + HttpHeaders = httpHeaders, + Conditions = new AppendBlobRequestConditions + { + IfNoneMatch = new ETag("*") + } + }, cancellationToken: cancellationToken); } catch (RequestFailedException e) { @@ -284,6 +332,13 @@ private async Task CreateAppendFileAsync(string url, string blobStorageType, Can request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob); request.Content.Headers.Add("Content-Length", "0"); } + if (customHeaders != null) + { + foreach (var header in customHeaders) + { + request.Content.Headers.Add(header.Key, header.Value); + } + }; using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) { @@ -366,14 +421,14 @@ public async Task UploadStepSummaryAsync(string planId, string jobId, Guid stepI } private async Task UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType, - CancellationToken cancellationToken) + CancellationToken cancellationToken, Dictionary customHeaders = null) { if (firstBlock && finalize) { // This is the one and only block, just use a block blob using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken); + await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken, customHeaders); } } else @@ -382,7 +437,7 @@ private async Task UploadLogFile(string file, bool finalize, bool firstBlock, st // Create the Append blob if (firstBlock) { - await CreateAppendFileAsync(sasUrl, blobStorageType, cancellationToken); + await CreateAppendFileAsync(sasUrl, blobStorageType, cancellationToken, customHeaders); } // Upload content @@ -404,7 +459,12 @@ public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid st throw new Exception("Failed to get step log upload url"); } - await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken); + var customHeaders = new Dictionary + { + { Constants.ContentTypeHeader, Constants.TextPlainContentType } + }; + + await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken, customHeaders); // Update metadata if (finalize) @@ -424,7 +484,12 @@ public async Task UploadResultsJobLogAsync(string planId, string jobId, string f throw new Exception("Failed to get job log upload url"); } - await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken); + var customHeaders = new Dictionary + { + { Constants.ContentTypeHeader, Constants.TextPlainContentType } + }; + + await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken, customHeaders); // Update metadata if (finalize) @@ -547,6 +612,9 @@ public static class Constants public static readonly string AzureBlobTypeHeader = "x-ms-blob-type"; public static readonly string AzureBlockBlob = "BlockBlob"; public static readonly string AzureAppendBlob = "AppendBlob"; + + public const string ContentTypeHeader = "Content-Type"; + public const string TextPlainContentType = "text/plain"; } }