From 42580e173a97d1b41ee26b1f3bf1eb21e19b90d3 Mon Sep 17 00:00:00 2001 From: Ersan Bozduman Date: Wed, 13 Dec 2023 00:11:19 -0800 Subject: [PATCH 1/3] re-adjusted the progress reporting --- Minio.Functional.Tests/FunctionalTest.cs | 20 +++++++++++------- Minio/ApiEndpoints/ObjectOperations.cs | 26 ++++++++++++++---------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index de8c76ddd2..5c771fdcae 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -3494,13 +3494,19 @@ internal static async Task PutObject_Test9(IMinioClient minio) var startTime = DateTime.Now; var bucketName = GetRandomName(15); var objectName = GetRandomObjectName(10); - var contentType = "application/octet-stream"; + const int objSize = 1 * MB; + const string contentType = "application/octet-stream"; var percentage = 0; var totalBytesTransferred = 0L; var progress = new Progress(progressReport => { percentage = progressReport.Percentage; totalBytesTransferred = progressReport.TotalBytesTransferred; + // Console.WriteLine( + // $"PutObject_Test9 - Percentage: {progressReport.Percentage}% TotalBytesTransferred: {progressReport.TotalBytesTransferred} bytes"); + // if (progressReport.Percentage != 100) + // Console.SetCursorPosition(0, Console.CursorTop - 1); + // else Console.WriteLine(); }); var args = new Dictionary (StringComparer.Ordinal) @@ -3514,9 +3520,9 @@ internal static async Task PutObject_Test9(IMinioClient minio) { await Setup_Test(minio, bucketName).ConfigureAwait(false); _ = await PutObject_Tester(minio, bucketName, objectName, null, contentType, 0, null, - rsg.GenerateStreamFromSeed(1 * MB), progress).ConfigureAwait(false); + rsg.GenerateStreamFromSeed(objSize), progress).ConfigureAwait(false); Assert.IsTrue(percentage == 100); - Assert.IsTrue(totalBytesTransferred == 1 * MB); + Assert.IsTrue(totalBytesTransferred == objSize); new MintLogger(nameof(PutObject_Test9), putObjectSignature, "Tests whether PutObject with progress passes for small object", TestStatus.PASS, DateTime.Now - startTime, @@ -3548,11 +3554,11 @@ internal static async Task PutObject_Test10(IMinioClient minio) { percentage = progressReport.Percentage; totalBytesTransferred = progressReport.TotalBytesTransferred; - //Console.WriteLine( - // $"Percentage: {progressReport.Percentage}% TotalBytesTransferred: {progressReport.TotalBytesTransferred} bytes"); - //if (progressReport.Percentage != 100) + // Console.WriteLine( + // $"PutObject_Test10 - Percentage: {progressReport.Percentage}% TotalBytesTransferred: {progressReport.TotalBytesTransferred} bytes"); + // if (progressReport.Percentage != 100) // Console.SetCursorPosition(0, Console.CursorTop - 1); - //else Console.WriteLine(); + // else Console.WriteLine(); }); var args = new Dictionary (StringComparer.Ordinal) diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index 5252aa38d6..1a7561395e 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -17,6 +17,7 @@ using System.Diagnostics.CodeAnalysis; using System.Globalization; +using System.Net; using System.Reactive.Linq; using Minio.ApiEndpoints; using Minio.DataModel; @@ -841,20 +842,29 @@ await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, /// The file stream is currently in a read operation /// For encrypted PUT operation, Access is denied if the key is wrong private async Task PutObjectSinglePartAsync(PutObjectArgs args, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + bool singleFile = true) { //Skipping validate as we need the case where stream sends 0 bytes var progressReport = new ProgressReport(); - args.Progress?.Report(progressReport); + if (singleFile) args.Progress?.Report(progressReport); var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false); using var response = await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, cancellationToken: cancellationToken) .ConfigureAwait(false); - progressReport.Percentage = 100; - progressReport.TotalBytesTransferred = args.ObjectSize; - args.Progress?.Report(progressReport); + if (singleFile) + { + if (response.StatusCode == HttpStatusCode.OK) + { + progressReport.Percentage = 100; + progressReport.TotalBytesTransferred = args.ObjectSize; + } + + args.Progress?.Report(progressReport); + } + return new PutObjectResponse(response.StatusCode, response.Content, response.Headers, args.ObjectSize, args.ObjectName); } @@ -924,12 +934,6 @@ private async Task> PutObjectPartAsync(PutObjectPartArg return null; } - if (args.ObjectSize == -1) - { - progressReport.Percentage = 100; - args.Progress?.Report(progressReport); - } - return etags; } From 636739d2c488a1a2fdbfe3fa0bac3a910f7fc1d5 Mon Sep 17 00:00:00 2001 From: Ersan Bozduman Date: Wed, 13 Dec 2023 00:54:57 -0800 Subject: [PATCH 2/3] adds missing param tag for 'singleFile' --- Minio/ApiEndpoints/ObjectOperations.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index 1a7561395e..24d5a1d52e 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -832,6 +832,10 @@ await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, /// Headers, SSE Headers /// /// Optional cancellation token to cancel the operation + /// + /// This boolean parameter differentiates single part file upload and + /// multi part file upload as this function is shared by both. + /// /// /// When access or secret key is invalid /// When bucket name is invalid From a507ca62002c0604f7042468052e582a5ecd73b0 Mon Sep 17 00:00:00 2001 From: Ersan Bozduman Date: Thu, 14 Dec 2023 04:34:28 -0800 Subject: [PATCH 3/3] gets single file upload size info from stat api --- Minio/ApiEndpoints/ObjectOperations.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index 24d5a1d52e..ebcb9cc5a1 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -593,7 +593,7 @@ public async Task PutObjectAsync(PutObjectArgs args, args = args.WithRequestBody(bytes) .WithStreamData(null) .WithObjectSize(bytesRead); - return await PutObjectSinglePartAsync(args, cancellationToken).ConfigureAwait(false); + return await PutObjectSinglePartAsync(args, cancellationToken, true).ConfigureAwait(false); } // For all sizes greater than 5MiB do multipart. @@ -847,7 +847,7 @@ await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, /// For encrypted PUT operation, Access is denied if the key is wrong private async Task PutObjectSinglePartAsync(PutObjectArgs args, CancellationToken cancellationToken = default, - bool singleFile = true) + bool singleFile = false) { //Skipping validate as we need the case where stream sends 0 bytes var progressReport = new ProgressReport(); @@ -858,15 +858,19 @@ await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, cancellationToken: cancellationToken) .ConfigureAwait(false); - if (singleFile) + if (singleFile && args.Progress is not null) { + var statArgs = new StatObjectArgs() + .WithBucket(args.BucketName) + .WithObject(args.ObjectName); + var stat = await StatObjectAsync(statArgs, cancellationToken).ConfigureAwait(false); if (response.StatusCode == HttpStatusCode.OK) { progressReport.Percentage = 100; - progressReport.TotalBytesTransferred = args.ObjectSize; + progressReport.TotalBytesTransferred = stat.Size; } - args.Progress?.Report(progressReport); + args.Progress.Report(progressReport); } return new PutObjectResponse(response.StatusCode, response.Content, response.Headers,