From 0a91ce7f06d0da43fe3f9b6e108b107ed39f5043 Mon Sep 17 00:00:00 2001 From: hadashiA Date: Fri, 15 Mar 2024 12:45:13 +0900 Subject: [PATCH 1/3] Fix BrotliCompressor.CopyToAsync buffer is not enough --- src/MemoryPack.Core/Compression/BrotliCompressor.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/MemoryPack.Core/Compression/BrotliCompressor.cs b/src/MemoryPack.Core/Compression/BrotliCompressor.cs index af13bcd..971aa20 100644 --- a/src/MemoryPack.Core/Compression/BrotliCompressor.cs +++ b/src/MemoryPack.Core/Compression/BrotliCompressor.cs @@ -157,6 +157,11 @@ public async ValueTask CopyToAsync(Stream stream, int bufferSize = 65535, Cancel foreach (var item in bufferWriter) { var source = item; + if (source.Length > buffer.Length) + { + ArrayPool.Shared.Return(buffer); + buffer = ArrayPool.Shared.Rent(source.Length); + } var lastResult = OperationStatus.DestinationTooSmall; while (lastResult == OperationStatus.DestinationTooSmall) { @@ -175,6 +180,10 @@ public async ValueTask CopyToAsync(Stream stream, int bufferSize = 65535, Cancel // call BrotliEncoderOperation.Finish var finalStatus = encoder.Compress(ReadOnlySpan.Empty, buffer, out var consumed, out var written, isFinalBlock: true); + if (finalStatus != OperationStatus.Done) + { + MemoryPackSerializationException.ThrowCompressionFailed(finalStatus); + } if (written > 0) { await stream.WriteAsync(buffer.AsMemory(0, written), cancellationToken).ConfigureAwait(false); From 2131f6fa666429639968978ce07938e4d3982887 Mon Sep 17 00:00:00 2001 From: hadashiA Date: Fri, 15 Mar 2024 14:14:58 +0900 Subject: [PATCH 2/3] tweaks --- src/MemoryPack.Core/Compression/BrotliCompressor.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/MemoryPack.Core/Compression/BrotliCompressor.cs b/src/MemoryPack.Core/Compression/BrotliCompressor.cs index 971aa20..692916c 100644 --- a/src/MemoryPack.Core/Compression/BrotliCompressor.cs +++ b/src/MemoryPack.Core/Compression/BrotliCompressor.cs @@ -151,17 +151,20 @@ public async ValueTask CopyToAsync(Stream stream, int bufferSize = 65535, Cancel using var encoder = new BrotliEncoder(quality, window); + foreach (var item in bufferWriter) + { + if (item.Length > bufferSize) + { + bufferSize = item.Length; + } + } + var buffer = ArrayPool.Shared.Rent(bufferSize); try { foreach (var item in bufferWriter) { var source = item; - if (source.Length > buffer.Length) - { - ArrayPool.Shared.Return(buffer); - buffer = ArrayPool.Shared.Rent(source.Length); - } var lastResult = OperationStatus.DestinationTooSmall; while (lastResult == OperationStatus.DestinationTooSmall) { From 6bd9b325d4116d67744b4c50d2258567fe420584 Mon Sep 17 00:00:00 2001 From: hadashiA Date: Fri, 15 Mar 2024 14:46:16 +0900 Subject: [PATCH 3/3] Remove unneccessory large buffer --- .../Compression/BrotliCompressor.cs | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/MemoryPack.Core/Compression/BrotliCompressor.cs b/src/MemoryPack.Core/Compression/BrotliCompressor.cs index 692916c..c4754b8 100644 --- a/src/MemoryPack.Core/Compression/BrotliCompressor.cs +++ b/src/MemoryPack.Core/Compression/BrotliCompressor.cs @@ -151,14 +151,6 @@ public async ValueTask CopyToAsync(Stream stream, int bufferSize = 65535, Cancel using var encoder = new BrotliEncoder(quality, window); - foreach (var item in bufferWriter) - { - if (item.Length > bufferSize) - { - bufferSize = item.Length; - } - } - var buffer = ArrayPool.Shared.Rent(bufferSize); try { @@ -182,14 +174,18 @@ public async ValueTask CopyToAsync(Stream stream, int bufferSize = 65535, Cancel } // call BrotliEncoderOperation.Finish - var finalStatus = encoder.Compress(ReadOnlySpan.Empty, buffer, out var consumed, out var written, isFinalBlock: true); - if (finalStatus != OperationStatus.Done) + var finalStatus = OperationStatus.DestinationTooSmall; + while (finalStatus == OperationStatus.DestinationTooSmall) { - MemoryPackSerializationException.ThrowCompressionFailed(finalStatus); + finalStatus = encoder.Compress(ReadOnlySpan.Empty, buffer, out var consumed, out var written, isFinalBlock: true); + if (written > 0) + { + await stream.WriteAsync(buffer.AsMemory(0, written), cancellationToken).ConfigureAwait(false); + } } - if (written > 0) + if (finalStatus != OperationStatus.Done) { - await stream.WriteAsync(buffer.AsMemory(0, written), cancellationToken).ConfigureAwait(false); + MemoryPackSerializationException.ThrowCompressionFailed(finalStatus); } } finally