Add Brotli Compression to CoreFX

pkg/Microsoft.Private.PackageBaseline/packageIndex.json

@@ -1886,6 +1886,15 @@
         "4.1.2.0": "4.3.0"
       }
     },
+    "System.IO.Compression.Brotli": {
+      "InboxOn": {
+        "netcoreapp2.1": "4.2.1.0"
+      },
+      "AssemblyVersionInPackageVersion": {
+        "4.2.0.0": "4.4.0",
+        "4.2.1.0": "4.5.0"
+      }
+    },
     "System.IO.Compression.FileSystem": {
       "InboxOn": {
         "netcoreapp2.0": "4.0.0.0",

src/Common/src/Interop/Windows/Interop.Libraries.cs

@@ -30,6 +30,6 @@ internal static partial class Libraries
         internal const string WebSocket = "websocket.dll";
         internal const string WinHttp = "winhttp.dll";
         internal const string Ws2_32 = "ws2_32.dll";
-        internal const string Zlib = "clrcompression.dll";
+        internal const string CompressionNative = "clrcompression.dll";
     }
 }

src/Common/src/Microsoft/Win32/SafeHandles/SafeBrotliHandle.cs

@@ -0,0 +1,35 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.Win32.SafeHandles
+{
+    internal sealed class SafeBrotliEncoderHandle : SafeHandle
+    {
+        public SafeBrotliEncoderHandle() : base(IntPtr.Zero, true) { }
+
+        protected override bool ReleaseHandle()
+        {
+            Interop.Brotli.BrotliEncoderDestroyInstance(handle);
+            return true;
+        }
+
+        public override bool IsInvalid => handle == IntPtr.Zero;
+    }
+
+    internal sealed class SafeBrotliDecoderHandle : SafeHandle
+    {
+        public SafeBrotliDecoderHandle() : base(IntPtr.Zero, true) { }
+
+        protected override bool ReleaseHandle()
+        {
+            Interop.Brotli.BrotliDecoderDestroyInstance(handle);
+            return true;
+        }
+
+        public override bool IsInvalid => handle == IntPtr.Zero;
+    }
+}

src/Common/tests/System/IO/Compression/CompressionStreamPerfTestBase.cs

@@ -2,39 +2,43 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
 using Microsoft.Xunit.Performance;
 using Xunit;
 
 namespace System.IO.Compression
 {
     public abstract class CompressionStreamPerfTestBase : CompressionStreamTestBase
     {
-        public static IEnumerable<object[]> CanterburyCorpus()
+        public static IEnumerable<object[]> CanterburyCorpus_WithCompressionLevel()
         {
             foreach (CompressionLevel compressionLevel in Enum.GetValues(typeof(CompressionLevel)))
             {
-                foreach (int innerIterations in new int[] { 1, 10 })
+                foreach (object[] canterburyWithoutLevel in CanterburyCorpus())
                 {
-                    foreach (var fileName in UncompressedTestFiles())
-                    {
-                        yield return new object[] { innerIterations, fileName[0], compressionLevel };
-                    }
+                    yield return new object[] { canterburyWithoutLevel[0], canterburyWithoutLevel[1], compressionLevel };
                 }
             }
         }
 
+        public static IEnumerable<object[]> CanterburyCorpus()
+        {
+            foreach (int innerIterations in new int[] { 1, 10 })
+            {
+                foreach (var fileName in UncompressedTestFiles())
+                {
+                    yield return new object[] { innerIterations, fileName[0] };
+                }
+            }
+        }
+
+
         /// <summary>
         /// Benchmark tests to measure the performance of individually compressing each file in the
         /// Canterbury Corpus
         /// </summary>
         [Benchmark]
-        [MemberData(nameof(CanterburyCorpus))]
+        [MemberData(nameof(CanterburyCorpus_WithCompressionLevel))]
         public void Compress_Canterbury(int innerIterations, string uncompressedFileName, CompressionLevel compressLevel)
         {
             byte[] bytes = File.ReadAllBytes(uncompressedFileName);
@@ -61,7 +65,7 @@ public void Compress_Canterbury(int innerIterations, string uncompressedFileName
         /// </summary>
         [Benchmark]
         [MemberData(nameof(CanterburyCorpus))]
-        public void Decompress_Canterbury(int innerIterations, string uncompressedFilePath, CompressionLevel compressLevel)
+        public void Decompress_Canterbury(int innerIterations, string uncompressedFilePath)
         {
             string compressedFilePath = CompressedTestFile(uncompressedFilePath);
             byte[] outputRead = new byte[new FileInfo(uncompressedFilePath).Length];

src/Common/tests/System/IO/Compression/CompressionStreamTestBase.cs

@@ -2,13 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-using Xunit;
 
 namespace System.IO.Compression
 {

src/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs

@@ -2,9 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -15,31 +13,34 @@ namespace System.IO.Compression
     public abstract class CompressionStreamUnitTestBase : CompressionStreamTestBase
     {
         [Fact]
-        public async Task FlushAsync_DuringWriteAsync()
+        public virtual void FlushAsync_DuringWriteAsync()
         {
             if (FlushNoOps)
                 return;
-            byte[] buffer = null;
-            string testFilePath = UncompressedTestFile();
-            using (var origStream = await LocalMemoryStream.readAppFileAsync(testFilePath))
-            {
-                buffer = origStream.ToArray();
-            }
+            byte[] buffer = new byte[100000];
+            Random rand = new Random();
+            rand.NextBytes(buffer);
 
             using (var writeStream = new ManualSyncMemoryStream(false))
             using (var compressor = CreateStream(writeStream, CompressionMode.Compress))
             {
                 Task task = null;
                 try
                 {
+                    // Write needs to be big enough to trigger a write to the underlying base stream so the WriteAsync call doesn't immediately complete.
                     task = compressor.WriteAsync(buffer, 0, buffer.Length);
+                    while (task.IsCompleted)
+                    {
+                        rand.NextBytes(buffer);
+                        task = compressor.WriteAsync(buffer, 0, buffer.Length);
+                    }
                     Assert.Throws<InvalidOperationException>(() => { compressor.FlushAsync(); }); // "overlapping flushes"
                 }
                 finally
                 {
                     // Unblock Async operations
                     writeStream.manualResetEvent.Set();
-                    // The original WriteAsync should be able to complete, though it may not have been enough input to trigger a write to the base stream
+                    // The original WriteAsync should be able to complete
                     Assert.True(task.Wait(10 * 500), "Original WriteAsync Task did not complete in time");
                     Assert.True(writeStream.WriteHit, "BaseStream Write function was not called");
                 }
@@ -119,32 +120,34 @@ public async Task FlushAsync_DuringFlushAsync()
         }
 
         [Fact]
-        public async Task WriteAsync_DuringWriteAsync()
+        public virtual void WriteAsync_DuringWriteAsync()
         {
-            byte[] buffer = null;
-            string testFilePath = UncompressedTestFile();
-            using (var origStream = await LocalMemoryStream.readAppFileAsync(testFilePath))
-            {
-                buffer = origStream.ToArray();
-            }
+            byte[] buffer = new byte[100000];
+            Random rand = new Random();
+            rand.NextBytes(buffer);
 
             using (var writeStream = new ManualSyncMemoryStream(false))
             using (var compressor = CreateStream(writeStream, CompressionMode.Compress))
             {
                 Task task = null;
                 try
                 {
-                    task = compressor.WriteAsync(buffer, 0, buffer.Length);    // write needs to be bigger than the internal write buffer
+                    // Write needs to be big enough to trigger a write to the underlying base stream so the WriteAsync call doesn't immediately complete.
+                    task = compressor.WriteAsync(buffer, 0, buffer.Length);
+                    while (task.IsCompleted)
+                    {
+                        rand.NextBytes(buffer);
+                        task = compressor.WriteAsync(buffer, 0, buffer.Length);
+                    }
                     Assert.Throws<InvalidOperationException>(() => { compressor.WriteAsync(buffer, 32, 32); }); // "overlapping write"
                 }
                 finally
                 {
                     // Unblock Async operations
                     writeStream.manualResetEvent.Set();
-                    // The original WriteAsync should be able to complete, though it may not have been enough input to trigger a write to the base stream
+                    // The original WriteAsync should be able to complete
                     Assert.True(task.Wait(10 * 500), "Original WriteAsync Task did not complete in time");
-                    if (writeStream.Length > 0)
-                        Assert.True(writeStream.WriteHit, "BaseStream Write function was not called");
+                    Assert.True(writeStream.WriteHit, "BaseStream Write function was not called");
                 }
             }
         }
@@ -175,37 +178,44 @@ public async Task ReadAsync_DuringReadAsync()
         }
 
         [Fact]
-        public virtual async Task Dispose_WithUnfinishedWriteAsync()
+        public virtual void Dispose_WithUnfinishedWriteAsync()
         {
-            byte[] uncompressedBytes = null;
-            string uncompressedPath = UncompressedTestFile();
-            using (var uncompressedStream = await LocalMemoryStream.readAppFileAsync(uncompressedPath))
-            {
-                uncompressedBytes = uncompressedStream.ToArray();
-            }
+            byte[] buffer = new byte[100000];
+            Random rand = new Random();
+            rand.NextBytes(buffer);
 
             using (var writeStream = new ManualSyncMemoryStream(false))
             {
-                var compressor = CreateStream(writeStream, CompressionMode.Compress, true);
-                compressor.Write(uncompressedBytes, 0, uncompressedBytes.Length);
+                var compressor = CreateStream(writeStream, CompressionMode.Compress, leaveOpen: true);
+                compressor.Write(buffer, 0, buffer.Length);
                 int writesBeingFlushed = 2;
-                Task task = compressor.WriteAsync(uncompressedBytes, 0, uncompressedBytes.Length);    // write needs to be bigger than the internal write buffer
-                while (!writeStream.WriteHit && task.IsCompleted)
+                Task task = null;
+                try
                 {
-                    task = compressor.WriteAsync(uncompressedBytes, 0, uncompressedBytes.Length);
-                    writesBeingFlushed++;
-                }
+                    // Write needs to be big enough to trigger a write to the underlying base stream so the WriteAsync call doesn't immediately complete.
+                    task = compressor.WriteAsync(buffer, 0, buffer.Length);
+                    while (task.IsCompleted)
+                    {
+                        rand.NextBytes(buffer);
+                        task = compressor.WriteAsync(buffer, 0, buffer.Length);
+                        writesBeingFlushed++;
+                    }
 
-                // WriteAsync will be blocked on writing the output to the underlying stream. Calling Dispose will trigger a Finish call with unwritten output
-                // still available.
-                Assert.InRange(writeStream.Length, 0, uncompressedBytes.Length);
-                compressor.Dispose();
-                Assert.InRange(writeStream.Length, 0, uncompressedBytes.Length * writesBeingFlushed);
-                Assert.False(task.IsCompleted);
-                writeStream.manualResetEvent.Set();
-                // WriteAsync call will return to the compression stream's WriteAsync which will attempt to 
-                // access members of the now disposed stream. 
-                Assert.Throws<AggregateException>(() => task.Wait(1000)); 
+                    // WriteAsync will be blocked on writing the output to the underlying stream. Calling Dispose will trigger a Finish call with unwritten output
+                    // still available.
+                    Assert.InRange(writeStream.Length, 0, buffer.Length);
+                    compressor.Dispose();
+                    Assert.InRange(writeStream.Length, 0, buffer.Length * writesBeingFlushed);
+                    Assert.False(task.IsCompleted);
+                }
+                finally
+                {
+                    // Unblock Async operations
+                    writeStream.manualResetEvent.Set();
+                    // WriteAsync call will return to the compression stream's WriteAsync which will attempt to 
+                    // access members of the now disposed stream. 
+                    Assert.Throws<AggregateException>(() => task.Wait(1000));
+                }
             }
         }
 
@@ -1129,7 +1139,7 @@ public async Task WrapStreamReturningBadReadValues()
         [Theory]
         [OuterLoop]
         [InlineData(true)]
-        [InlineData(true)]
+        [InlineData(false)]
         public async Task CopyTo_Roundtrip_OutputMatchesInput(bool useAsync)
         {
             var rand = new Random();

src/Native/Unix/CMakeLists.txt

@@ -9,12 +9,10 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
 set(CMAKE_SHARED_LIBRARY_PREFIX "")
 set(VERSION_FILE_PATH "${CMAKE_BINARY_DIR}/../../version.c")
 
-add_compile_options(-Weverything)
 add_compile_options(-Wno-format-nonliteral)
 add_compile_options(-Wno-missing-prototypes)
 add_compile_options(-Wno-disabled-macro-expansion)
 add_compile_options(-Wno-c++98-compat)
-add_compile_options(-Wno-c++98-compat-pedantic)
 add_compile_options(-Wno-padded)
 add_compile_options(-Wno-empty-translation-unit)
 add_compile_options(-Wno-old-style-cast)
@@ -23,12 +21,11 @@ add_compile_options(-Wno-typedef-redefinition)
 add_compile_options(-Wno-unused-local-typedef)
 add_compile_options(-Wno-c99-extensions)
 add_compile_options(-Wno-c11-extensions)
-add_compile_options(-Werror)
 add_compile_options(-fPIC)
 add_compile_options(-I${CMAKE_CURRENT_SOURCE_DIR}/Common)
 add_compile_options(-I${CMAKE_CURRENT_BINARY_DIR}/Common)
 add_compile_options(-g)
-
+add_compile_options(-Werror)
 if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.5)
     add_compile_options(-Wno-unreachable-code)
 endif ()
@@ -247,6 +244,10 @@ endfunction()
 include(configure.cmake)
 
 add_subdirectory(System.IO.Compression.Native)
+
+add_compile_options(-Weverything)
+add_compile_options(-Wno-c++98-compat-pedantic)
+
 add_subdirectory(System.Native)
 add_subdirectory(System.Net.Http.Native)
 add_subdirectory(System.Net.Security.Native)

src/Native/Unix/System.IO.Compression.Native/CMakeLists.txt

@@ -6,6 +6,35 @@ set(NATIVECOMPRESSION_SOURCES
     pal_zlib.cpp
 )
 
+#Include Brotli include files
+include_directories("../../AnyOS/brotli/include")
+
+set (NATIVECOMPRESSION_SOURCES
+    ${NATIVECOMPRESSION_SOURCES}
+    ../../AnyOS/brotli/common/dictionary.c
+    ../../AnyOS/brotli/dec/bit_reader.c
+    ../../AnyOS/brotli/dec/decode.c
+    ../../AnyOS/brotli/dec/huffman.c
+    ../../AnyOS/brotli/dec/state.c
+    ../../AnyOS/brotli/enc/backward_references.c
+    ../../AnyOS/brotli/enc/backward_references_hq.c
+    ../../AnyOS/brotli/enc/bit_cost.c
+    ../../AnyOS/brotli/enc/block_splitter.c
+    ../../AnyOS/brotli/enc/brotli_bit_stream.c
+    ../../AnyOS/brotli/enc/cluster.c
+    ../../AnyOS/brotli/enc/compress_fragment.c
+    ../../AnyOS/brotli/enc/compress_fragment_two_pass.c
+    ../../AnyOS/brotli/enc/dictionary_hash.c
+    ../../AnyOS/brotli/enc/encode.c
+    ../../AnyOS/brotli/enc/entropy_encode.c
+    ../../AnyOS/brotli/enc/histogram.c
+    ../../AnyOS/brotli/enc/literal_cost.c
+    ../../AnyOS/brotli/enc/memory.c
+    ../../AnyOS/brotli/enc/metablock.c
+    ../../AnyOS/brotli/enc/static_dict.c
+    ../../AnyOS/brotli/enc/utf8_util.c
+)
+
 add_library(System.IO.Compression.Native
     SHARED
     ${NATIVECOMPRESSION_SOURCES}