Skip to content

Commit 68e2f92

Browse files
pikselpiksel
authored
fix(zip): fully implement async deflate (#813)
1 parent c4009fd commit 68e2f92

File tree

6 files changed

+156
-39
lines changed

6 files changed

+156
-39
lines changed

‎src/ICSharpCode.SharpZipLib/GZip/GzipInputStream.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,7 @@ private void ReadFooter()
334334
int crcval = (footer[0] & 0xff) | ((footer[1] & 0xff) << 8) | ((footer[2] & 0xff) << 16) | (footer[3] << 24);
335335
if (crcval != (int)crc.Value)
336336
{
337-
throw new GZipException("GZIP crc sum mismatch, theirs \"" + crcval + "\" and ours \"" + (int)crc.Value);
337+
throw new GZipException($"GZIP crc sum mismatch, theirs \"{crcval:x8}\" and ours \"{(int)crc.Value:x8}\"");
338338
}
339339

340340
// NOTE The total here is the original total modulo 2 ^ 32.

‎src/ICSharpCode.SharpZipLib/GZip/GzipOutputStream.cs‎

Lines changed: 52 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,11 @@ public string FileName
138138
}
139139
}
140140

141+
/// <summary>
142+
/// If defined, will use this time instead of the current for the output header
143+
/// </summary>
144+
public DateTime? ModifiedTime { get; set; }
145+
141146
#endregion Public API
142147

143148
#region Stream overrides
@@ -149,21 +154,47 @@ public string FileName
149154
/// <param name="offset">Offset of first byte in buf to write</param>
150155
/// <param name="count">Number of bytes to write</param>
151156
public override void Write(byte[] buffer, int offset, int count)
157+
=> WriteSyncOrAsync(buffer, offset, count, null).GetAwaiter().GetResult();
158+
159+
private async Task WriteSyncOrAsync(byte[] buffer, int offset, int count, CancellationToken? ct)
152160
{
153161
if (state_ == OutputState.Header)
154162
{
155-
WriteHeader();
163+
if (ct.HasValue)
164+
{
165+
await WriteHeaderAsync(ct.Value).ConfigureAwait(false);
166+
}
167+
else
168+
{
169+
WriteHeader();
170+
}
156171
}
157172

158173
if (state_ != OutputState.Footer)
159-
{
160174
throw new InvalidOperationException("Write not permitted in current state");
161-
}
162-
175+
163176
crc.Update(new ArraySegment<byte>(buffer, offset, count));
164-
base.Write(buffer, offset, count);
177+
178+
if (ct.HasValue)
179+
{
180+
await base.WriteAsync(buffer, offset, count, ct.Value).ConfigureAwait(false);
181+
}
182+
else
183+
{
184+
base.Write(buffer, offset, count);
185+
}
165186
}
166187

188+
/// <summary>
189+
/// Asynchronously write given buffer to output updating crc
190+
/// </summary>
191+
/// <param name="buffer">Buffer to write</param>
192+
/// <param name="offset">Offset of first byte in buf to write</param>
193+
/// <param name="count">Number of bytes to write</param>
194+
/// <param name="ct">The token to monitor for cancellation requests</param>
195+
public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken ct)
196+
=> await WriteSyncOrAsync(buffer, offset, count, ct).ConfigureAwait(false);
197+
167198
/// <summary>
168199
/// Writes remaining compressed output data to the output stream
169200
/// and closes it.
@@ -187,7 +218,7 @@ protected override void Dispose(bool disposing)
187218
}
188219
}
189220

190-
#if NETSTANDARD2_1_OR_GREATER
221+
#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_1_OR_GREATER
191222
/// <inheritdoc cref="DeflaterOutputStream.Dispose"/>
192223
public override async ValueTask DisposeAsync()
193224
{
@@ -225,6 +256,16 @@ public override void Flush()
225256
base.Flush();
226257
}
227258

259+
/// <inheritdoc cref="Flush"/>
260+
public override async Task FlushAsync(CancellationToken ct)
261+
{
262+
if (state_ == OutputState.Header)
263+
{
264+
await WriteHeaderAsync(ct).ConfigureAwait(false);
265+
}
266+
await base.FlushAsync(ct).ConfigureAwait(false);
267+
}
268+
228269
#endregion Stream overrides
229270

230271
#region DeflaterOutputStream overrides
@@ -249,21 +290,13 @@ public override void Finish()
249290
}
250291
}
251292

252-
/// <inheritdoc cref="Flush"/>
253-
public override async Task FlushAsync(CancellationToken ct)
254-
{
255-
await WriteHeaderAsync().ConfigureAwait(false);
256-
await base.FlushAsync(ct).ConfigureAwait(false);
257-
}
258-
259-
260293
/// <inheritdoc cref="Finish"/>
261294
public override async Task FinishAsync(CancellationToken ct)
262295
{
263296
// If no data has been written a header should be added.
264297
if (state_ == OutputState.Header)
265298
{
266-
await WriteHeaderAsync().ConfigureAwait(false);
299+
await WriteHeaderAsync(ct).ConfigureAwait(false);
267300
}
268301

269302
if (state_ == OutputState.Footer)
@@ -305,7 +338,8 @@ private byte[] GetFooter()
305338

306339
private byte[] GetHeader()
307340
{
308-
var modTime = (int)((DateTime.Now.Ticks - new DateTime(1970, 1, 1).Ticks) / 10000000L); // Ticks give back 100ns intervals
341+
var modifiedUtc = ModifiedTime?.ToUniversalTime() ?? DateTime.UtcNow;
342+
var modTime = (int)((modifiedUtc - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).Ticks / 10000000L); // Ticks give back 100ns intervals
309343
byte[] gzipHeader = {
310344
// The two magic bytes
311345
GZipConstants.ID1,
@@ -351,12 +385,12 @@ private void WriteHeader()
351385
baseOutputStream_.Write(gzipHeader, 0, gzipHeader.Length);
352386
}
353387

354-
private async Task WriteHeaderAsync()
388+
private async Task WriteHeaderAsync(CancellationToken ct)
355389
{
356390
if (state_ != OutputState.Header) return;
357391
state_ = OutputState.Footer;
358392
var gzipHeader = GetHeader();
359-
await baseOutputStream_.WriteAsync(gzipHeader, 0, gzipHeader.Length).ConfigureAwait(false);
393+
await baseOutputStream_.WriteAsync(gzipHeader, 0, gzipHeader.Length, ct).ConfigureAwait(false);
360394
}
361395

362396
#endregion Support Routines

‎src/ICSharpCode.SharpZipLib/Zip/Compression/Streams/DeflaterOutputStream.cs‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -240,11 +240,9 @@ protected void EncryptBlock(byte[] buffer, int offset, int length)
240240
/// are processed.
241241
/// </summary>
242242
protected void Deflate()
243-
{
244-
Deflate(false);
245-
}
243+
=> DeflateSyncOrAsync(false, null).GetAwaiter().GetResult();
246244

247-
private void Deflate(bool flushing)
245+
private async Task DeflateSyncOrAsync(bool flushing, CancellationToken? ct)
248246
{
249247
while (flushing || !deflater_.IsNeedingInput)
250248
{
@@ -257,7 +255,14 @@ private void Deflate(bool flushing)
257255

258256
EncryptBlock(buffer_, 0, deflateCount);
259257

260-
baseOutputStream_.Write(buffer_, 0, deflateCount);
258+
if (ct.HasValue)
259+
{
260+
await baseOutputStream_.WriteAsync(buffer_, 0, deflateCount, ct.Value).ConfigureAwait(false);
261+
}
262+
else
263+
{
264+
baseOutputStream_.Write(buffer_, 0, deflateCount);
265+
}
261266
}
262267

263268
if (!deflater_.IsNeedingInput)
@@ -383,10 +388,18 @@ public override int Read(byte[] buffer, int offset, int count)
383388
public override void Flush()
384389
{
385390
deflater_.Flush();
386-
Deflate(true);
391+
DeflateSyncOrAsync(true, null).GetAwaiter().GetResult();
387392
baseOutputStream_.Flush();
388393
}
389394

395+
/// <inheritdoc/>
396+
public override async Task FlushAsync(CancellationToken cancellationToken)
397+
{
398+
deflater_.Flush();
399+
await DeflateSyncOrAsync(true, cancellationToken).ConfigureAwait(false);
400+
await baseOutputStream_.FlushAsync(cancellationToken).ConfigureAwait(false);
401+
}
402+
390403
/// <summary>
391404
/// Calls <see cref="Finish"/> and closes the underlying
392405
/// stream when <see cref="IsStreamOwner"></see> is true.
@@ -491,6 +504,13 @@ public override void Write(byte[] buffer, int offset, int count)
491504
Deflate();
492505
}
493506

507+
/// <inheritdoc />
508+
public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken ct)
509+
{
510+
deflater_.SetInput(buffer, offset, count);
511+
await DeflateSyncOrAsync(false, ct).ConfigureAwait(false);
512+
}
513+
494514
#endregion Stream Overrides
495515

496516
#region Instance Fields

‎src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,7 @@ await baseOutputStream_.WriteProcToStreamAsync(s =>
552552
public void CloseEntry()
553553
{
554554
// Note: This method will run synchronously
555-
FinishCompression(null).Wait();
555+
FinishCompressionSyncOrAsync(null).GetAwaiter().GetResult();
556556
WriteEntryFooter(baseOutputStream_);
557557

558558
// Patch the header if possible
@@ -566,7 +566,7 @@ public void CloseEntry()
566566
curEntry = null;
567567
}
568568

569-
private async Task FinishCompression(CancellationToken? ct)
569+
private async Task FinishCompressionSyncOrAsync(CancellationToken? ct)
570570
{
571571
// Compression handled externally
572572
if (entryIsPassthrough) return;
@@ -600,7 +600,7 @@ private async Task FinishCompression(CancellationToken? ct)
600600
/// <inheritdoc cref="CloseEntry"/>
601601
public async Task CloseEntryAsync(CancellationToken ct)
602602
{
603-
await FinishCompression(ct).ConfigureAwait(false);
603+
await FinishCompressionSyncOrAsync(ct).ConfigureAwait(false);
604604
await baseOutputStream_.WriteProcToStreamAsync(WriteEntryFooter, ct).ConfigureAwait(false);
605605

606606
// Patch the header if possible
@@ -767,9 +767,7 @@ private byte[] CreateZipCryptoHeader(long crcValue)
767767
private void InitializeZipCryptoPassword(string password)
768768
{
769769
var pkManaged = new PkzipClassicManaged();
770-
Console.WriteLine($"Output Encoding: {ZipCryptoEncoding.EncodingName}");
771770
byte[] key = PkzipClassic.GenerateKeys(ZipCryptoEncoding.GetBytes(password));
772-
Console.WriteLine($"Output Bytes: {string.Join(", ", key.Select(b => $"{b:x2}").ToArray())}");
773771
cryptoTransform_ = pkManaged.CreateEncryptor(key, null);
774772
}
775773

@@ -782,6 +780,13 @@ private void InitializeZipCryptoPassword(string password)
782780
/// <exception cref="ZipException">Archive size is invalid</exception>
783781
/// <exception cref="System.InvalidOperationException">No entry is active.</exception>
784782
public override void Write(byte[] buffer, int offset, int count)
783+
=> WriteSyncOrAsync(buffer, offset, count, null).GetAwaiter().GetResult();
784+
785+
/// <inheritdoc />
786+
public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken ct)
787+
=> await WriteSyncOrAsync(buffer, offset, count, ct).ConfigureAwait(false);
788+
789+
private async Task WriteSyncOrAsync(byte[] buffer, int offset, int count, CancellationToken? ct)
785790
{
786791
if (curEntry == null)
787792
{
@@ -816,20 +821,34 @@ public override void Write(byte[] buffer, int offset, int count)
816821

817822
size += count;
818823

819-
if(curMethod == CompressionMethod.Stored || entryIsPassthrough)
824+
if (curMethod == CompressionMethod.Stored || entryIsPassthrough)
820825
{
821826
if (Password != null)
822827
{
823828
CopyAndEncrypt(buffer, offset, count);
824829
}
825830
else
826831
{
827-
baseOutputStream_.Write(buffer, offset, count);
832+
if (ct.HasValue)
833+
{
834+
await baseOutputStream_.WriteAsync(buffer, offset, count, ct.Value).ConfigureAwait(false);
835+
}
836+
else
837+
{
838+
baseOutputStream_.Write(buffer, offset, count);
839+
}
828840
}
829841
}
830842
else
831843
{
832-
base.Write(buffer, offset, count);
844+
if (ct.HasValue)
845+
{
846+
await base.WriteAsync(buffer, offset, count, ct.Value).ConfigureAwait(false);
847+
}
848+
else
849+
{
850+
base.Write(buffer, offset, count);
851+
}
833852
}
834853
}
835854

‎test/ICSharpCode.SharpZipLib.Tests/GZip/GZipAsyncTests.cs‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using System.IO;
1+
using System;
2+
using System.IO;
23
using System.Text;
34
using System.Threading.Tasks;
45
using ICSharpCode.SharpZipLib.GZip;
@@ -7,8 +8,6 @@
78

89
namespace ICSharpCode.SharpZipLib.Tests.GZip
910
{
10-
11-
1211
[TestFixture]
1312
public class GZipAsyncTests
1413
{
@@ -140,5 +139,48 @@ public async Task EmptyGZipStreamAsync()
140139
Assert.IsEmpty(content);
141140
}
142141
}
142+
143+
[Test]
144+
[Category("GZip")]
145+
[Category("Async")]
146+
public async Task WriteGZipStreamToAsyncOnlyStream()
147+
{
148+
#if NETSTANDARD2_1 || NETCOREAPP3_0_OR_GREATER
149+
var content = Encoding.ASCII.GetBytes("a");
150+
var modTime = DateTime.UtcNow;
151+
152+
await using (var msAsync = new MemoryStreamWithoutSync())
153+
{
154+
await using (var outStream = new GZipOutputStream(msAsync) { IsStreamOwner = false })
155+
{
156+
outStream.ModifiedTime = modTime;
157+
await outStream.WriteAsync(content);
158+
}
159+
160+
using var msSync = new MemoryStream();
161+
using (var outStream = new GZipOutputStream(msSync) { IsStreamOwner = false })
162+
{
163+
outStream.ModifiedTime = modTime;
164+
outStream.Write(content);
165+
}
166+
167+
var syncBytes = string.Join(' ', msSync.ToArray());
168+
var asyncBytes = string.Join(' ', msAsync.ToArray());
169+
170+
Assert.AreEqual(syncBytes, asyncBytes, "Sync and Async compressed streams are not equal");
171+
172+
// Since GZipInputStream isn't async yet we need to read from it from a regular MemoryStream
173+
using (var readStream = new MemoryStream(msAsync.ToArray()))
174+
using (var inStream = new GZipInputStream(readStream))
175+
using (var reader = new StreamReader(inStream))
176+
{
177+
Assert.AreEqual(content, await reader.ReadToEndAsync());
178+
}
179+
}
180+
#else
181+
await Task.CompletedTask;
182+
Assert.Ignore("AsyncDispose is not supported");
183+
#endif
184+
}
143185
}
144186
}

‎test/ICSharpCode.SharpZipLib.Tests/Zip/ZipStreamAsyncTests.cs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -124,17 +124,19 @@ public async Task WriteReadOnlyZipStreamAsync ()
124124
[Test]
125125
[Category("Zip")]
126126
[Category("Async")]
127-
public async Task WriteZipStreamToAsyncOnlyStream ()
127+
[TestCase(12, Description = "Small files")]
128+
[TestCase(12000, Description = "Large files")]
129+
public async Task WriteZipStreamToAsyncOnlyStream (int fileSize)
128130
{
129131
#if NETSTANDARD2_1 || NETCOREAPP3_0_OR_GREATER
130132
await using(var ms = new MemoryStreamWithoutSync()){
131133
await using(var outStream = new ZipOutputStream(ms) { IsStreamOwner = false })
132134
{
133135
await outStream.PutNextEntryAsync(new ZipEntry("FirstFile"));
134-
await Utils.WriteDummyDataAsync(outStream, 12);
136+
await Utils.WriteDummyDataAsync(outStream, fileSize);
135137

136138
await outStream.PutNextEntryAsync(new ZipEntry("SecondFile"));
137-
await Utils.WriteDummyDataAsync(outStream, 12);
139+
await Utils.WriteDummyDataAsync(outStream, fileSize);
138140

139141
await outStream.FinishAsync(CancellationToken.None);
140142
await outStream.DisposeAsync();

0 commit comments

Comments
 (0)