BinaryWriter perf and memory improvements

src/libraries/Common/tests/TestUtilities/System/Runtime/InteropServices/SafeBufferUtil.cs

@@ -0,0 +1,44 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+
+namespace System.Runtime.InteropServices
+{
+    public static class SafeBufferUtil
+    {
+        /// <summary>
+        /// Creates an unmanaged buffer of the specified length.
+        /// </summary>
+        public static SafeBuffer CreateSafeBuffer(nuint byteLength)
+        {
+            return new AllocHGlobalSafeHandle(byteLength);
+        }
+
+        private sealed class AllocHGlobalSafeHandle : SafeBuffer
+        {
+            public AllocHGlobalSafeHandle(nuint cb) : base(ownsHandle: true)
+            {
+#if !NETCOREAPP
+                RuntimeHelpers.PrepareConstrainedRegions();
+#endif
+                try
+                {
+                    // intentionally empty to avoid ThreadAbortException in netfx runtimes
+                }
+                finally
+                {
+                    SetHandle(Marshal.AllocHGlobal((nint)cb));
+                }
+
+                Initialize(cb);
+            }
+
+            protected override bool ReleaseHandle()
+            {
+                Marshal.FreeHGlobal(handle);
+                return true;
+            }
+        }
+    }
+}

src/libraries/Common/tests/TestUtilities/TestUtilities.csproj

@@ -19,6 +19,7 @@
     <Compile Include="System\Buffers\BoundedMemory.Unix.cs" />
     <Compile Include="System\Buffers\BoundedMemory.Windows.cs" />
     <Compile Include="System\Buffers\PoisonPagePlacement.cs" />
+    <Compile Include="System\Runtime\InteropServices\SafeBufferUtil.cs" />
     <Compile Include="System\IO\FileCleanupTestBase.cs" />
     <Compile Include="System\TestEnvironment.cs" />
     <Compile Include="System\ThreadCultureChange.cs" />

src/libraries/System.IO/tests/BinaryWriter/BinaryWriter.EncodingTests.cs

@@ -0,0 +1,251 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Numerics;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using System.Text;
+using Xunit;
+
+namespace System.IO.Tests
+{
+    public class BinaryWriter_EncodingTests
+    {
+        [Fact]
+        public void Ctor_Default_UsesFastUtf8()
+        {
+            BinaryWriter writer = new BinaryWriter(new MemoryStream());
+            Assert.True(IsUsingFastUtf8(writer));
+        }
+
+        [Fact]
+        public void Ctor_EncodingUtf8Singleton_UsesFastUtf8()
+        {
+            BinaryWriter writer = new BinaryWriter(new MemoryStream(), Encoding.UTF8);
+            Assert.True(IsUsingFastUtf8(writer));
+        }
+
+        [Theory]
+        [InlineData(true, true)]
+        [InlineData(true, false)]
+        [InlineData(false, true)]
+        [InlineData(false, false)]
+        public void Ctor_NewUtf8Encoding_UsesFastUtf8(bool emitIdentifier, bool throwOnInvalidBytes)
+        {
+            BinaryWriter writer = new BinaryWriter(new MemoryStream(), new UTF8Encoding(emitIdentifier, throwOnInvalidBytes));
+            Assert.True(IsUsingFastUtf8(writer));
+        }
+
+        [Fact]
+        public void Ctor_Utf8EncodingWithSingleCharReplacementChar_UsesFastUtf8()
+        {
+            Encoding encoding = Encoding.GetEncoding("utf-8", new EncoderReplacementFallback("x"), DecoderFallback.ExceptionFallback);
+            BinaryWriter writer = new BinaryWriter(new MemoryStream(), encoding);
+            Assert.True(IsUsingFastUtf8(writer));
+        }
+
+        [Fact]
+        public void Ctor_Utf8EncodingWithMultiCharReplacementChar_DoesNotUseFastUtf8()
+        {
+            Encoding encoding = Encoding.GetEncoding("utf-8", new EncoderReplacementFallback("xx"), DecoderFallback.ExceptionFallback);
+            BinaryWriter writer = new BinaryWriter(new MemoryStream(), encoding);
+            Assert.False(IsUsingFastUtf8(writer));
+        }
+
+        [Fact]
+        public void Ctor_NotUtf8EncodingType_DoesNotUseFastUtf8()
+        {
+            BinaryWriter writer = new BinaryWriter(new MemoryStream(), new UnicodeEncoding());
+            Assert.False(IsUsingFastUtf8(writer));
+        }
+
+        [Fact]
+        public void Ctor_Utf8EncodingDerivedTypeWithWrongCodePage_DoesNotUseFastUtf8()
+        {
+            BinaryWriter writer = new BinaryWriter(new MemoryStream(), new NotActuallyUTF8Encoding());
+            Assert.False(IsUsingFastUtf8(writer));
+        }
+
+        [Fact]
+        public void Ctor_Utf8EncodingDerivedTypeWithCorrectCodePage_DoesNotUseFastUtf8()
+        {
+            BinaryWriter writer = new BinaryWriter(new MemoryStream(), new MyCustomUTF8Encoding());
+            Assert.True(IsUsingFastUtf8(writer));
+        }
+
+        [Theory]
+        [InlineData('x')] // 1 UTF-8 byte
+        [InlineData('\u00e9')] // LATIN SMALL LETTER E WITH ACUTE (2 UTF-8 bytes)
+        [InlineData('\u2130')] // SCRIPT CAPITAL E (3 UTF-8 bytes)
+        public void WriteSingleChar_FastUtf8(char ch)
+        {
+            MemoryStream stream = new MemoryStream();
+            BinaryWriter writer = new BinaryWriter(stream);
+
+            writer.Write(ch);
+
+            Assert.Equal(Encoding.UTF8.GetBytes(new char[] { ch }), stream.ToArray());
+        }
+
+        [Theory]
+        [InlineData('x')] // 1 UTF-8 byte
+        [InlineData('\u00e9')] // LATIN SMALL LETTER E WITH ACUTE (2 UTF-8 bytes)
+        [InlineData('\u2130')] // SCRIPT CAPITAL E (3 UTF-8 bytes)
+        public void WriteSingleChar_NotUtf8NoArrayPoolRentalNeeded(char ch)
+        {
+            MemoryStream stream = new MemoryStream();
+            BinaryWriter writer = new BinaryWriter(stream, Encoding.Unicode /* little endian */);
+
+            writer.Write(ch);
+
+            Assert.Equal(Encoding.Unicode.GetBytes(new char[] { ch }), stream.ToArray());
+        }
+
+        [Fact]
+        public void WriteSingleChar_ArrayPoolRentalNeeded()
+        {
+            string replacementString = new string('v', 10_000);
+            Encoding encoding = Encoding.GetEncoding("ascii", new EncoderReplacementFallback(replacementString), DecoderFallback.ExceptionFallback);
+            MemoryStream stream = new MemoryStream();
+            BinaryWriter writer = new BinaryWriter(stream, encoding);
+
+            writer.Write('\uFFFD'); // not ASCII
+
+            Assert.Equal(Encoding.ASCII.GetBytes(replacementString), stream.ToArray());
+        }
+
+        [Theory]
+        [InlineData(8 * 1024)] // both char count & byte count within 64k rental boundary
+        [InlineData(32 * 1024)] // char count within 64k rental boundary, byte count not
+        [InlineData(256 * 1024)] // neither char count nor byte count within 64k rental boundary
+        public void WriteChars_FastUtf8(int stringLengthInChars)
+        {
+            string stringToWrite = GenerateLargeUnicodeString(stringLengthInChars);
+            byte[] expectedBytes = Encoding.UTF8.GetBytes(stringToWrite);
+
+            MemoryStream stream = new MemoryStream();
+            BinaryWriter writer = new BinaryWriter(stream);
+
+            writer.Write(stringToWrite.ToCharArray()); // writing a char buffer doesn't emit the length upfront 
+            Assert.Equal(expectedBytes, stream.GetBuffer()[..expectedBytes.Length]);
+        }
+
+        [Theory]
+        [InlineData(24)] // within stackalloc path
+        [InlineData(8 * 1024)] // both char count & byte count within 64k rental boundary
+        [InlineData(32 * 1024)] // char count within 64k rental boundary, byte count not
+        [InlineData(256 * 1024)] // neither char count nor byte count within 64k rental boundary
+        public void WriteString_FastUtf8(int stringLengthInChars)
+        {
+            string stringToWrite = GenerateLargeUnicodeString(stringLengthInChars);
+            byte[] expectedBytes = Encoding.UTF8.GetBytes(stringToWrite);
+
+            MemoryStream stream = new MemoryStream();
+            BinaryWriter writer = new BinaryWriter(stream);
+
+            writer.Write(stringToWrite);
+            stream.Position = 0;
+
+            Assert.Equal(expectedBytes.Length /* byte count */, new BinaryReader(stream).Read7BitEncodedInt());
+            Assert.Equal(expectedBytes, stream.GetBuffer()[Get7BitEncodedIntByteLength((uint)expectedBytes.Length)..(int)stream.Length]);
+        }
+
+        [Theory]
+        [InlineData(127 / 3)] // within stackalloc fast path
+        [InlineData(127 / 3 + 1)] // not within stackalloc fast path
+        public void WriteString_FastUtf8_UsingThreeByteChars(int stringLengthInChars)
+        {
+            string stringToWrite = new string('\u2023', stringLengthInChars); // TRIANGULAR BULLET
+            byte[] expectedBytes = Encoding.UTF8.GetBytes(stringToWrite);
+
+            MemoryStream stream = new MemoryStream();
+            BinaryWriter writer = new BinaryWriter(stream);
+
+            writer.Write(stringToWrite);
+            stream.Position = 0;
+
+            Assert.Equal(expectedBytes.Length /* byte count */, new BinaryReader(stream).Read7BitEncodedInt());
+            Assert.Equal(expectedBytes, stream.GetBuffer()[Get7BitEncodedIntByteLength((uint)expectedBytes.Length)..(int)stream.Length]);
+        }
+
+        [Theory]
+        [InlineData(8 * 1024)] // both char count & byte count within 64k rental boundary
+        [InlineData(48 * 1024)] // char count within 64k rental boundary, byte count not
+        [InlineData(256 * 1024)] // neither char count nor byte count within 64k rental boundary
+        public void WriteString_NotUtf8(int stringLengthInChars)
+        {
+            string stringToWrite = GenerateLargeUnicodeString(stringLengthInChars);
+            byte[] expectedBytes = Encoding.Unicode.GetBytes(stringToWrite);
+
+            MemoryStream stream = new MemoryStream();
+            BinaryWriter writer = new BinaryWriter(stream, Encoding.Unicode /* little endian */);
+
+            writer.Write(stringToWrite);
+            stream.Position = 0;
+
+            Assert.Equal(expectedBytes.Length /* byte count */, new BinaryReader(stream).Read7BitEncodedInt());
+            Assert.Equal(expectedBytes, stream.GetBuffer()[Get7BitEncodedIntByteLength((uint)expectedBytes.Length)..(int)stream.Length]);
+        }
+
+        [Fact]
+        public unsafe void WriteChars_VeryLargeArray_DoesNotOverflow()
+        {
+            const nuint INPUT_LEN_IN_CHARS = 1_500_000_000;
+            const nuint OUTPUT_LEN_IN_BYTES = 3_500_000_000; // overallocate
+
+            SafeBuffer unmanagedInputBuffer;
+            SafeBuffer unmanagedOutputBufer;
+            try
+            {
+                unmanagedInputBuffer = SafeBufferUtil.CreateSafeBuffer(INPUT_LEN_IN_CHARS * sizeof(char));
+                unmanagedOutputBufer = SafeBufferUtil.CreateSafeBuffer(OUTPUT_LEN_IN_BYTES * sizeof(byte));
+            }
+            catch (OutOfMemoryException)
+            {
+                return; // skip test in low-mem conditions
+            }
+
+            using (unmanagedInputBuffer)
+            using (unmanagedOutputBufer)
+            {
+                Span<char> inputSpan = new Span<char>((char*)unmanagedInputBuffer.DangerousGetHandle(), (int)INPUT_LEN_IN_CHARS);
+                inputSpan.Fill('\u0224'); // LATIN CAPITAL LETTER Z WITH HOOK
+                Stream outStream = new UnmanagedMemoryStream(unmanagedOutputBufer, 0, (long)unmanagedOutputBufer.ByteLength, FileAccess.ReadWrite);
+                BinaryWriter writer = new BinaryWriter(outStream);
+
+                writer.Write(inputSpan); // will write 3 billion bytes to the output
+
+                Assert.Equal(3_000_000_000, outStream.Position);
+            }
+        }
+
+        private static bool IsUsingFastUtf8(BinaryWriter writer)
+        {
+            return (bool)writer.GetType().GetField("_useFastUtf8", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(writer);
+        }
+
+        private static string GenerateLargeUnicodeString(int charCount)
+        {
+            return string.Create(charCount, (object)null, static (buffer, _) =>
+            {
+                for (int i = 0; i < buffer.Length; i++)
+                {
+                    buffer[i] = (char)((i % 0xF00) + 0x100); // U+0100..U+0FFF (mix of 2-byte and 3-byte chars)
+                }
+            });
+        }
+
+        private static int Get7BitEncodedIntByteLength(uint value) => (BitOperations.Log2(value) / 7) + 1;
+
+        // subclasses UTF8Encoding, but returns a non-UTF8 code page
+        private class NotActuallyUTF8Encoding : UTF8Encoding
+        {
+            public override int CodePage => 65000; // UTF-7 code page
+        }
+
+        // subclasses UTF8Encoding, returns UTF-8 code page
+        private class MyCustomUTF8Encoding : UTF8Encoding
+        {
+        }
+    }
+}

src/libraries/System.IO/tests/System.IO.Tests.csproj

@@ -15,6 +15,7 @@
     <Compile Include="StreamWriter\StreamWriter.cs" />
     <Compile Include="StreamWriter\StreamWriter.StringCtorTests.cs" />
     <Compile Include="BinaryWriter\BinaryWriter.DisposeAsync.cs" />
+    <Compile Include="BinaryWriter\BinaryWriter.EncodingTests.cs" />
     <Compile Include="BinaryWriter\BinaryWriter.WriteByteCharTests.cs" />
     <Compile Include="BinaryWriter\BinaryWriter.WriteTests.cs" />
     <Compile Include="BinaryWriter\BinaryWriterTests.cs" />

src/libraries/System.Private.CoreLib/src/System/Decimal.cs

@@ -587,16 +587,14 @@ public static bool TryGetBits(decimal d, Span<int> destination, out int valuesWr
             return true;
         }
 
-        internal static void GetBytes(in decimal d, byte[] buffer)
+        internal static void GetBytes(in decimal d, Span<byte> buffer)
         {
-            Debug.Assert(buffer != null && buffer.Length >= 16, "[GetBytes]buffer != null && buffer.Length >= 16");
+            Debug.Assert(buffer.Length >= 16, "buffer.Length >= 16");
 
-            Span<byte> span = buffer;
-
-            BinaryPrimitives.WriteInt32LittleEndian(span, (int)d.Low);
-            BinaryPrimitives.WriteInt32LittleEndian(span.Slice(4), (int)d.Mid);
-            BinaryPrimitives.WriteInt32LittleEndian(span.Slice(8), (int)d.High);
-            BinaryPrimitives.WriteInt32LittleEndian(span.Slice(12), d._flags);
+            BinaryPrimitives.WriteInt32LittleEndian(buffer, (int)d.Low);
+            BinaryPrimitives.WriteInt32LittleEndian(buffer.Slice(4), (int)d.Mid);
+            BinaryPrimitives.WriteInt32LittleEndian(buffer.Slice(8), (int)d.High);
+            BinaryPrimitives.WriteInt32LittleEndian(buffer.Slice(12), d._flags);
         }
 
         internal static decimal ToDecimal(ReadOnlySpan<byte> span)