Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Specify Content-Type for BlockBlob upload #3119

Merged
merged 8 commits into from
Feb 5, 2024
Merged

Specify Content-Type for BlockBlob upload #3119

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7778991
add content-type to block blob upload
bethanyj28 Jan 30, 2024
8901ee6
Add content-type for sdk path
bethanyj28 Jan 31, 2024
ca07991
fix spacing
bethanyj28 Jan 31, 2024
37263f1
merge headers and only when file extension is .txt
bethanyj28 Jan 31, 2024
7a0db61
add conditions
bethanyj28 Feb 1, 2024
dc146c2
tweak conditions and path matching
bethanyj28 Feb 2, 2024
fd90e7c
pass in headers
bethanyj28 Feb 2, 2024
5c3baef
add content-type for appendblob
bethanyj28 Feb 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 77 additions & 9 deletions src/Sdk/WebApi/WebApi/ResultsHttpClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,14 +222,34 @@ private AppendBlobClient GetAppendBlobClient(string url)
return new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts);
}

private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken)
private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken, Dictionary<string, string> customHeaders = null)
{
if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
var blobClient = GetBlobClient(url);
var httpHeaders = new BlobHttpHeaders();
if (customHeaders != null)
{
foreach (var header in customHeaders)
{
switch (header.Key)
{
case Constants.ContentTypeHeader:
httpHeaders.ContentType = header.Value;
break;
}
}
}
try
{
await blobClient.UploadAsync(file, cancellationToken);
await blobClient.UploadAsync(file, new BlobUploadOptions()
{
HttpHeaders = httpHeaders,
Conditions = new BlobRequestConditions
{
IfNoneMatch = new ETag("*")
}
}, cancellationToken);
}
catch (RequestFailedException e)
{
Expand All @@ -249,6 +269,14 @@ private async Task UploadBlockFileAsync(string url, string blobStorageType, File
request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob);
}

if (customHeaders != null)
{
foreach (var header in customHeaders)
{
request.Content.Headers.Add(header.Key, header.Value);
}
};

using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
if (!response.IsSuccessStatusCode)
Expand All @@ -259,14 +287,34 @@ private async Task UploadBlockFileAsync(string url, string blobStorageType, File
}
}

private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken)
private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken, Dictionary<string, string> customHeaders = null)
{
if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
var appendBlobClient = GetAppendBlobClient(url);
var httpHeaders = new BlobHttpHeaders();
if (customHeaders != null)
{
foreach (var header in customHeaders)
{
switch (header.Key)
{
case Constants.ContentTypeHeader:
httpHeaders.ContentType = header.Value;
break;
}
}
}
try
{
await appendBlobClient.CreateAsync(cancellationToken: cancellationToken);
await appendBlobClient.CreateAsync(new AppendBlobCreateOptions()
{
HttpHeaders = httpHeaders,
Conditions = new AppendBlobRequestConditions
{
IfNoneMatch = new ETag("*")
}
}, cancellationToken: cancellationToken);
}
catch (RequestFailedException e)
{
Expand All @@ -284,6 +332,13 @@ private async Task CreateAppendFileAsync(string url, string blobStorageType, Can
request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob);
request.Content.Headers.Add("Content-Length", "0");
}
if (customHeaders != null)
{
foreach (var header in customHeaders)
{
request.Content.Headers.Add(header.Key, header.Value);
}
};

using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
Expand Down Expand Up @@ -366,14 +421,14 @@ public async Task UploadStepSummaryAsync(string planId, string jobId, Guid stepI
}

private async Task UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType,
CancellationToken cancellationToken)
CancellationToken cancellationToken, Dictionary<string, string> customHeaders = null)
{
if (firstBlock && finalize)
{
// This is the one and only block, just use a block blob
using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
{
await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken);
await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken, customHeaders);
}
}
else
Expand All @@ -382,7 +437,7 @@ private async Task UploadLogFile(string file, bool finalize, bool firstBlock, st
// Create the Append blob
if (firstBlock)
{
await CreateAppendFileAsync(sasUrl, blobStorageType, cancellationToken);
await CreateAppendFileAsync(sasUrl, blobStorageType, cancellationToken, customHeaders);
}

// Upload content
Expand All @@ -404,7 +459,12 @@ public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid st
throw new Exception("Failed to get step log upload url");
}

await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken);
var customHeaders = new Dictionary<string, string>
{
{ Constants.ContentTypeHeader, Constants.TextPlainContentType }
};

await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken, customHeaders);

// Update metadata
if (finalize)
Expand All @@ -424,7 +484,12 @@ public async Task UploadResultsJobLogAsync(string planId, string jobId, string f
throw new Exception("Failed to get job log upload url");
}

await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken);
var customHeaders = new Dictionary<string, string>
{
{ Constants.ContentTypeHeader, Constants.TextPlainContentType }
};

await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken, customHeaders);

// Update metadata
if (finalize)
Expand Down Expand Up @@ -547,6 +612,9 @@ public static class Constants
public static readonly string AzureBlobTypeHeader = "x-ms-blob-type";
public static readonly string AzureBlockBlob = "BlockBlob";
public static readonly string AzureAppendBlob = "AppendBlob";

public const string ContentTypeHeader = "Content-Type";
public const string TextPlainContentType = "text/plain";
}

}
Loading