Add EnumerateRunes() ref APIs and unit tests

src/Common/src/CoreLib/System/MemoryExtensions.Fast.cs

@@ -2,10 +2,12 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using Xunit;
 
 using Internal.Runtime.CompilerServices;
 
@@ -371,6 +373,58 @@ public static bool StartsWith(this ReadOnlySpan<char> span, ReadOnlySpan<char> v
                     CultureInfo.CurrentCulture.CompareInfo.IsPrefix(span, value, string.GetCaseCompareOfComparisonCulture(comparisonType));
         }
 
+        [Theory]
+        [InlineData(new char[0], new int[0])] // empty
+        [InlineData(new char[] { 'x', 'y', 'z' }, new int[] { 'x', 'y', 'z' })]
+        [InlineData(new char[] { 'x', '\uD86D', '\uDF54', 'y' }, new int[] { 'x', 0x2B754, 'y' })] // valid surrogate pair
+        [InlineData(new char[] { 'x', '\uD86D', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // standalone high surrogate
+        [InlineData(new char[] { 'x', '\uDF54', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // standalone low surrogate
+        [InlineData(new char[] { 'x', '\uD86D' }, new int[] { 'x', 0xFFFD })] // standalone high surrogate at end of string
+        [InlineData(new char[] { 'x', '\uDF54' }, new int[] { 'x', 0xFFFD })] // standalone low surrogate at end of string
+        [InlineData(new char[] { 'x', '\uD86D', '\uD86D', 'y' }, new int[] { 'x', 0xFFFD, 0xFFFD, 'y' })] // two high surrogates should be two replacement chars
+        [InlineData(new char[] { 'x', '\uFFFD', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // literal U+FFFD
+        public static void EnumerateRunes(char[] chars, int[] expected)
+        {
+            // Test data is smuggled as char[] instead of straight-up string since the test framework
+            // doesn't like invalid UTF-16 literals.
+
+            // first, test Span<char>
+
+            List<int> enumeratedValues = new List<int>();
+            foreach (Rune rune in ((Span<char>)chars).EnumerateRunes())
+            {
+                enumeratedValues.Add(rune.Value);
+            }
+            Assert.Equal(expected, enumeratedValues.ToArray());
+
+
+            // next, ROS<char>
+
+            enumeratedValues.Clear();
+            foreach (Rune rune in ((ReadOnlySpan<char>)chars).EnumerateRunes())
+            {
+                enumeratedValues.Add(rune.Value);
+            }
+            Assert.Equal(expected, enumeratedValues.ToArray());
+        }
+
+        [Fact]
+        public static void EnumerateRunes_DoesNotReadPastEndOfSpan(char[] chars, int[] expected)
+        {
+            // As an optimization, reading scalars from a string *may* read past the end of the string
+            // to the terminating null. This optimization is invalid for arbitrary spans, so this test
+            // ensures that we're not performing this optimization here.
+
+            ReadOnlySpan<char> span = "xy\U0002B754z".AsSpan(1, 2); // well-formed string, but span splits surrogate pair
+
+            List<int> enumeratedValues = new List<int>();
+            foreach (Rune rune in span.EnumerateRunes())
+            {
+                enumeratedValues.Add(rune.Value);
+            }
+            Assert.Equal(new int[] { 'y', '\uFFFD' }, enumeratedValues.ToArray());
+        }
+
         /// <summary>
         /// Creates a new span over the portion of the target array.
         /// </summary>

src/System.Memory/ref/System.Memory.cs

@@ -42,6 +42,8 @@ public static void CopyTo<T>(this T[] source, System.Span<T> destination) { }
         public static bool EndsWith(this System.ReadOnlySpan<char> span, System.ReadOnlySpan<char> value, System.StringComparison comparisonType) { throw null; }
         public static bool EndsWith<T>(this System.ReadOnlySpan<T> span, System.ReadOnlySpan<T> value) where T : System.IEquatable<T> { throw null; }
         public static bool EndsWith<T>(this System.Span<T> span, System.ReadOnlySpan<T> value) where T : System.IEquatable<T> { throw null; }
+        public static System.Text.SpanRuneEnumerator EnumerateRunes(this ReadOnlySpan<char> span) { throw null; }
+        public static System.Text.SpanRuneEnumerator EnumerateRunes(this Span<char> span) { throw null; }
         public static bool Equals(this System.ReadOnlySpan<char> span, System.ReadOnlySpan<char> other, System.StringComparison comparisonType) { throw null; }
         public static int IndexOf(this System.ReadOnlySpan<char> span, System.ReadOnlySpan<char> value, System.StringComparison comparisonType) { throw null; }
         public static int IndexOfAny<T>(this System.ReadOnlySpan<T> span, System.ReadOnlySpan<T> values) where T : System.IEquatable<T> { throw null; }
@@ -430,3 +432,14 @@ public static partial class SequenceMarshal
         public static bool TryGetReadOnlySequenceSegment<T>(System.Buffers.ReadOnlySequence<T> sequence, out System.Buffers.ReadOnlySequenceSegment<T> startSegment, out int startIndex, out System.Buffers.ReadOnlySequenceSegment<T> endSegment, out int endIndex) { throw null; }
     }
 }
+namespace System.Text
+{
+    public ref partial struct SpanRuneEnumerator
+    {
+        private readonly object _dummyReference;
+        private readonly int _dummyPrimitive;
+        public Rune Current { get { throw null; } }
+        public SpanRuneEnumerator GetEnumerator() { throw null; }
+        public bool MoveNext() { throw null; }
+    }
+}

src/System.Memory/src/ApiCompatBaseline.netcoreappaot.txt

@@ -1,9 +1,12 @@
 Compat issues with assembly System.Memory:
+MembersMustExist : Member 'System.MemoryExtensions.EnumerateRunes(System.ReadOnlySpan<System.Char>)' does not exist in the implementation but it does exist in the contract.
+MembersMustExist : Member 'System.MemoryExtensions.EnumerateRunes(System.Span<System.Char>)' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.ReadOnlySpan<T>.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.ReadOnlySpan<T>.Item.get(System.Range)' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.Span<T>.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.Span<T>.Item.get(System.Range)' does not exist in the implementation but it does exist in the contract.
 TypesMustExist : Type 'System.Buffers.StandardFormat' does not exist in the implementation but it does exist in the contract.
 TypesMustExist : Type 'System.Buffers.Text.Utf8Formatter' does not exist in the implementation but it does exist in the contract.
 TypesMustExist : Type 'System.Buffers.Text.Utf8Parser' does not exist in the implementation but it does exist in the contract.
-Total Issues: 7
+TypesMustExist : Type 'System.Text.SpanRuneEnumerator' does not exist in the implementation but it does exist in the contract.
+Total Issues: 10

src/System.Memory/src/ApiCompatBaseline.uapaot.txt

@@ -1,9 +1,12 @@
 Compat issues with assembly System.Memory:
+MembersMustExist : Member 'System.MemoryExtensions.EnumerateRunes(System.ReadOnlySpan<System.Char>)' does not exist in the implementation but it does exist in the contract.
+MembersMustExist : Member 'System.MemoryExtensions.EnumerateRunes(System.Span<System.Char>)' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.ReadOnlySpan<T>.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.ReadOnlySpan<T>.Item.get(System.Range)' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.Span<T>.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.Span<T>.Item.get(System.Range)' does not exist in the implementation but it does exist in the contract.
 TypesMustExist : Type 'System.Buffers.StandardFormat' does not exist in the implementation but it does exist in the contract.
 TypesMustExist : Type 'System.Buffers.Text.Utf8Formatter' does not exist in the implementation but it does exist in the contract.
 TypesMustExist : Type 'System.Buffers.Text.Utf8Parser' does not exist in the implementation but it does exist in the contract.
-Total Issues: 7
+TypesMustExist : Type 'System.Text.SpanRuneEnumerator' does not exist in the implementation but it does exist in the contract.
+Total Issues: 10

src/System.Runtime/ref/System.Runtime.cs

@@ -2330,6 +2330,7 @@ public void CopyTo(int sourceIndex, char[] destination, int destinationIndex, in
         public bool EndsWith(System.String value) { throw null; }
         public bool EndsWith(System.String value, bool ignoreCase, System.Globalization.CultureInfo culture) { throw null; }
         public bool EndsWith(System.String value, System.StringComparison comparisonType) { throw null; }
+        public System.Text.StringRuneEnumerator EnumerateRunes() { throw null; }
         public override bool Equals(object obj) { throw null; }
         public bool Equals(System.String value) { throw null; }
         public static bool Equals(System.String a, System.String b) { throw null; }
@@ -7784,6 +7785,19 @@ public partial struct ChunkEnumerator
             public bool MoveNext() { throw null; }
         }
     }
+    public partial struct StringRuneEnumerator : System.Collections.Generic.IEnumerable<System.Text.Rune>, System.Collections.Generic.IEnumerator<System.Text.Rune>
+    {
+        private readonly object _dummyReference;
+        private readonly int _dummyPrimitive;
+        public System.Text.Rune Current { get { throw null; } }
+        public StringRuneEnumerator GetEnumerator() { throw null; }
+        public bool MoveNext() { throw null; }
+        object System.Collections.IEnumerator.Current { get { throw null; } }
+        void IDisposable.Dispose() { }
+        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { throw null; }
+        System.Collections.Generic.IEnumerator<System.Text.Rune> System.Collections.Generic.IEnumerable<System.Text.Rune>.GetEnumerator() { throw null; }
+        void System.Collections.IEnumerator.Reset() { }
+    }
 }
 namespace System.Threading
 {

src/System.Runtime/src/ApiCompatBaseline.netcoreappaot.txt

@@ -7,6 +7,7 @@ TypesMustExist : Type 'System.ArgIterator' does not exist in the implementation
 TypesMustExist : Type 'System.Index' does not exist in the implementation but it does exist in the contract. 
 TypesMustExist : Type 'System.Range' does not exist in the implementation but it does exist in the contract.
 TypesMustExist : Type 'System.Text.Rune' does not exist in the implementation but it does exist in the contract.
+TypesMustExist : Type 'System.Text.StringRuneEnumerator' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.ReadOnlySpan<T>.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract. 
 MembersMustExist : Member 'System.ReadOnlySpan<T>.Item.get(System.Range)' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.Span<T>.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract.

src/System.Runtime/src/ApiCompatBaseline.uapaot.txt

@@ -2,6 +2,7 @@ TypesMustExist : Type 'System.ArgIterator' does not exist in the implementation
 TypesMustExist : Type 'System.Index' does not exist in the implementation but it does exist in the contract. 
 TypesMustExist : Type 'System.Range' does not exist in the implementation but it does exist in the contract.
 TypesMustExist : Type 'System.Text.Rune' does not exist in the implementation but it does exist in the contract.
+TypesMustExist : Type 'System.Text.StringRuneEnumerator' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.ReadOnlySpan<T>.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract. 
 MembersMustExist : Member 'System.ReadOnlySpan<T>.Item.get(System.Range)' does not exist in the implementation but it does exist in the contract.
 MembersMustExist : Member 'System.Span<T>.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract.

src/System.Runtime/tests/System/StringTests.netcoreapp.cs

@@ -6,6 +6,7 @@
 using System.Globalization;
 using System.Linq;
 using System.Runtime.InteropServices;
+using System.Text;
 using Xunit;
 
 namespace System.Tests
@@ -411,6 +412,38 @@ public static void EndsWith(string s, char value, bool expected)
             Assert.Equal(expected, s.EndsWith(value));
         }
 
+        [Theory]
+        [InlineData(new char[0], new int[0])] // empty
+        [InlineData(new char[] { 'x', 'y', 'z' }, new int[] { 'x', 'y', 'z' })]
+        [InlineData(new char[] { 'x', '\uD86D', '\uDF54', 'y' }, new int[] { 'x', 0x2B754, 'y' })] // valid surrogate pair
+        [InlineData(new char[] { 'x', '\uD86D', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // standalone high surrogate
+        [InlineData(new char[] { 'x', '\uDF54', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // standalone low surrogate
+        [InlineData(new char[] { 'x', '\uD86D' }, new int[] { 'x', 0xFFFD })] // standalone high surrogate at end of string
+        [InlineData(new char[] { 'x', '\uDF54' }, new int[] { 'x', 0xFFFD })] // standalone low surrogate at end of string
+        [InlineData(new char[] { 'x', '\uD86D', '\uD86D', 'y' }, new int[] { 'x', 0xFFFD, 0xFFFD, 'y' })] // two high surrogates should be two replacement chars
+        [InlineData(new char[] { 'x', '\uFFFD', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // literal U+FFFD
+        public static void EnumerateRunes(char[] chars, int[] expected)
+        {
+            // Test data is smuggled as char[] instead of straight-up string since the test framework
+            // doesn't like invalid UTF-16 literals.
+
+            string asString = new string(chars);
+
+            // First, use a straight-up foreach keyword to ensure pattern matching works as expected
+
+            List<int> enumeratedScalarValues = new List<int>();
+            foreach (Rune rune in asString.EnumerateRunes())
+            {
+                enumeratedScalarValues.Add(rune.Value);
+            }
+            Assert.Equal(expected, enumeratedScalarValues.ToArray());
+
+            // Then use LINQ to ensure IEnumerator<...> works as expected
+
+            int[] enumeratedValues = new string(chars).EnumerateRunes().Select(r => r.Value).ToArray();
+            Assert.Equal(expected, enumeratedValues);
+        }
+
         [Theory]
         [InlineData("Hello", 'H', true)]
         [InlineData("Hello", 'h', false)]