Includes type forwarding for System.Text.Unicode.Utf8 to the Microsoft.Bcl.Memory library

src/libraries/Microsoft.Bcl.Memory/src/Microsoft.Bcl.Memory.Forwards.cs

@@ -3,6 +3,9 @@
 
 [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.Index))]
 [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.Range))]
+#if NET
+[assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.Text.Unicode.Utf8))]
+#endif
 #if NET9_0_OR_GREATER
 [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.Buffers.Text.Base64Url))]
 #endif

src/libraries/Microsoft.Bcl.Memory/src/Microsoft.Bcl.Memory.csproj

@@ -1,11 +1,11 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFrameworks>$(NetCoreAppCurrent);$(NetCoreAppPrevious);$(NetCoreAppMinimum);netstandard2.1;netstandard2.0;$(NetFrameworkMinimum)</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <DefineConstants>$(DefineConstants);MICROSOFT_BCL_MEMORY</DefineConstants>
     <IsPackable>true</IsPackable>
-    <PackageDescription>Provides Base64Url, Index and Range types support for .NET Framework and .NET Standard.</PackageDescription>
+    <PackageDescription>Provides Base64Url, Utf8, Index, and Range types support for .NET Framework and .NET Standard.</PackageDescription>
   </PropertyGroup>
 
   <!-- DesignTimeBuild requires all the TargetFramework Derived Properties to not be present in the first property group. -->
@@ -27,6 +27,22 @@
     <Compile Include="$(CoreLibSharedDir)System\Buffers\Text\Base64Url\Base64UrlValidator.cs" Link="System\Buffers\Text\Base64Url\Base64UrlValidator.cs" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(TargetFrameworkIdentifier)' != '.NETCoreApp'">
+    <Compile Include="Polyfills\System.Numerics.BitOperations.netstandard20.cs" />
+    <Compile Include="System\ThrowHelper.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\Ascii.Utility.cs" Link="System\Text\Ascii.Utility.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\Ascii.Utility.Helpers.cs" Link="System\Text\Ascii.Utility.Helpers.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\Rune.cs" Link="System\Text\Rune.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\UnicodeDebug.cs" Link="System\Text\UnicodeDebug.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\UnicodeUtility.cs" Link="System\Text\UnicodeUtility.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\Unicode\Utf16Utility.cs" Link="System\Text\Unicode\Utf16Utility.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\Unicode\Utf8.cs" Link="System\Text\Unicode\Utf8.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\Unicode\Utf8Utility.cs" Link="System\Text\Unicode\Utf8Utility.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\Unicode\Utf8Utility.Helpers.cs" Link="System\Text\Unicode\Utf8Utility.Helpers.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\Unicode\Utf8Utility.Transcoding.cs" Link="System\Text\Unicode\Utf8Utility.Transcoding.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\Unicode\Utf8Utility.Validation.cs" Link="System\Text\Unicode\Utf8Utility.Validation.cs" />
+  </ItemGroup>
+
   <ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'netstandard2.1'))">
     <Compile Include="$(CoreLibSharedDir)System\Index.cs" />
     <Compile Include="$(CoreLibSharedDir)System\Range.cs" />

src/libraries/Microsoft.Bcl.Memory/src/PACKAGE.md

@@ -1,16 +1,18 @@
 ## About
 
 Provides `Index` and `Range` types to simplify slicing operations on collections for .NET Framework and .NET Standard 2.0.
-Provides `Base64Url` for encoding data in a URL-safe manner on .NET Framework and .NET Standard.
+Provides `Base64Url` for encoding data in a URL-safe manner on older .NET platforms.
+Provides `Utf8` for converting chunked data between UTF-8 and UTF-16 encodings on .NET Framework and .NET Standard 2.0.
 
 This library is not necessary nor recommended when targeting versions of .NET that include the relevant support.
 
 ## Key Features
 
 <!-- The key features of this package -->
 
-* Enables the use of `Index` and `Range` types on older .NET platforms.
+* Enables the use of `Index` and `Range` types on .NET Framework and .NET Standard 2.0.
 * Provides `Base64Url` encoding, decoding, and validation for URL-safe data processing on older .NET platforms.
+* Provides `Utf8` encoding, decoding, and validation for chunked data between UTF-8 and UTF-16 on .NET Framework and .NET Standard 2.0.
 
 ## How to Use
 
@@ -64,6 +66,7 @@ The main types provided by this library are:
 * `System.Index`
 * `System.Range`
 * `System.Buffers.Text.Base64Url`
+* `System.Text.Unicode.Utf8`
 
 ## Additional Documentation
 
@@ -74,6 +77,7 @@ API documentation
 * [System.Index](https://learn.microsoft.com/dotnet/api/system.index)
 * [System.Range](https://learn.microsoft.com/dotnet/api/system.range)
 * [System.Buffers.Text.Base64Url](https://learn.microsoft.com/dotnet/api/system.buffers.text.base64url)
+* [System.Text.Unicode.Utf8](https://learn.microsoft.com/dotnet/api/system.text.unicode.utf8)
 
 ## Feedback & Contributing
 

src/libraries/Microsoft.Bcl.Memory/src/Polyfills/System.Numerics.BitOperations.netstandard20.cs

@@ -0,0 +1,75 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// Contains a polyfill implementation of System.Numerics.BitOperations that works on netstandard2.0.
+// Implementation copied from:
+// https://github.com/dotnet/runtime/blob/82ab89241b90ca3d64b22971f3a1e248da72828a/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs
+//
+// Some routines inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson:
+// http://graphics.stanford.edu/~seander/bithacks.html
+
+namespace System.Numerics
+{
+    internal static class BitOperations
+    {
+        // C# no-alloc optimization that directly wraps the data section of the dll (similar to string constants)
+        // https://github.com/dotnet/roslyn/pull/24621
+
+        private static ReadOnlySpan<byte> TrailingZeroCountDeBruijn => // 32
+        [
+            00, 01, 28, 02, 29, 14, 24, 03,
+            30, 22, 20, 15, 25, 17, 04, 08,
+            31, 27, 13, 23, 21, 19, 16, 07,
+            26, 12, 18, 06, 11, 05, 10, 09
+        ];
+
+        /// <summary>
+        /// Count the number of trailing zero bits in an integer value.
+        /// Similar in behavior to the x86 instruction TZCNT.
+        /// </summary>
+        /// <param name="value">The value.</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int TrailingZeroCount(uint value)
+        {
+            // Unguarded fallback contract is 0->0, BSF contract is 0->undefined
+            if (value == 0)
+            {
+                return 32;
+            }
+
+            // uint.MaxValue >> 27 is always in range [0 - 31] so we use Unsafe.AddByteOffset to avoid bounds check
+            return Unsafe.AddByteOffset(
+                // Using deBruijn sequence, k=2, n=5 (2^5=32) : 0b_0000_0111_0111_1100_1011_0101_0011_0001u
+                ref MemoryMarshal.GetReference(TrailingZeroCountDeBruijn),
+                // uint|long -> IntPtr cast on 32-bit platforms does expensive overflow checks not needed here
+                (IntPtr)(int)(((value & (uint)-(int)value) * 0x077CB531u) >> 27)); // Multi-cast mitigates redundant conv.u8
+        }
+
+        /// <summary>
+        /// Rotates the specified value left by the specified number of bits.
+        /// Similar in behavior to the x86 instruction ROL.
+        /// </summary>
+        /// <param name="value">The value to rotate.</param>
+        /// <param name="offset">The number of bits to rotate by.
+        /// Any value outside the range [0..31] is treated as congruent mod 32.</param>
+        /// <returns>The rotated value.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static uint RotateLeft(uint value, int offset)
+            => (value << offset) | (value >> (32 - offset));
+
+        /// <summary>
+        /// Rotates the specified value right by the specified number of bits.
+        /// Similar in behavior to the x86 instruction ROR.
+        /// </summary>
+        /// <param name="value">The value to rotate.</param>
+        /// <param name="offset">The number of bits to rotate by.
+        /// Any value outside the range [0..31] is treated as congruent mod 32.</param>
+        /// <returns>The rotated value.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static uint RotateRight(uint value, int offset)
+            => (value >> offset) | (value << (32 - offset));
+    }
+}

src/libraries/Microsoft.Bcl.Memory/src/System/ThrowHelper.cs

@@ -0,0 +1,62 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+
+namespace System
+{
+    internal static class ThrowHelper
+    {
+        [DoesNotReturn]
+        internal static void ThrowArgumentException_DestinationTooShort()
+        {
+            throw new ArgumentException(SR.Argument_DestinationTooShort, "destination");
+        }
+
+        [DoesNotReturn]
+        internal static void ThrowArgumentNullException(ExceptionArgument argument)
+        {
+            throw new ArgumentNullException(GetArgumentName(argument));
+        }
+
+        [DoesNotReturn]
+        internal static void ThrowArgumentOutOfRangeException(ExceptionArgument argument)
+        {
+            throw new ArgumentOutOfRangeException(GetArgumentName(argument));
+        }
+
+        private static string GetArgumentName(ExceptionArgument argument)
+        {
+            switch (argument)
+            {
+                case ExceptionArgument.ch:
+                    return nameof(ExceptionArgument.ch);
+                case ExceptionArgument.culture:
+                    return nameof(ExceptionArgument.culture);
+                case ExceptionArgument.index:
+                    return nameof(ExceptionArgument.index);
+                case ExceptionArgument.input:
+                    return nameof(ExceptionArgument.input);
+                case ExceptionArgument.value:
+                    return nameof(ExceptionArgument.value);
+                default:
+                    Debug.Fail("The enum value is not defined, please check the ExceptionArgument Enum.");
+                    return "";
+
+            }
+        }
+    }
+
+    //
+    // The convention for this enum is using the argument name as the enum name
+    //
+    internal enum ExceptionArgument
+    {
+        ch,
+        culture,
+        index,
+        input,
+        value,
+    }
+}

src/libraries/Microsoft.Bcl.Memory/tests/Microsoft.Bcl.Memory.Tests.csproj

@@ -30,6 +30,11 @@
     <Compile Include="..\..\System.Runtime\tests\System.Runtime.Tests\System\IndexTests.cs">
       <Link>System\IndexTests.cs</Link>
     </Compile>
+    <Compile Include="$(CoreLibSharedDir)System\Text\UnicodeDebug.cs" Link="System\Text\UnicodeDebug.cs" />
+    <Compile Include="$(CoreLibSharedDir)System\Text\UnicodeUtility.cs" Link="System\Text\UnicodeUtility.cs" />
+    <Compile Include="..\..\System.Runtime\tests\System.Runtime.Tests\System\Text\Unicode\Utf8Tests.cs">
+      <Link>System\Text\Unicode\Utf8Tests.cs</Link>
+    </Compile>
   </ItemGroup>
 
   <!-- Targetting  NetFramework & Netstandard 2.0 -->

src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.Helpers.cs

@@ -4,10 +4,16 @@
 using System.Diagnostics;
 using System.Numerics;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 
 namespace System.Text
 {
-    public static partial class Ascii
+#if SYSTEM_PRIVATE_CORELIB
+    public
+#else
+    internal
+#endif
+        static partial class Ascii
     {
         /// <summary>
         /// A mask which selects only the high bit of each byte of the given <see cref="uint"/>.