diff --git a/src/Common/src/CoreLib/System/MemoryExtensions.Fast.cs b/src/Common/src/CoreLib/System/MemoryExtensions.Fast.cs index 99fb83e635c7..5a45b708aeb3 100644 --- a/src/Common/src/CoreLib/System/MemoryExtensions.Fast.cs +++ b/src/Common/src/CoreLib/System/MemoryExtensions.Fast.cs @@ -2,10 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using Xunit; using Internal.Runtime.CompilerServices; @@ -371,6 +373,58 @@ public static bool StartsWith(this ReadOnlySpan span, ReadOnlySpan v CultureInfo.CurrentCulture.CompareInfo.IsPrefix(span, value, string.GetCaseCompareOfComparisonCulture(comparisonType)); } + [Theory] + [InlineData(new char[0], new int[0])] // empty + [InlineData(new char[] { 'x', 'y', 'z' }, new int[] { 'x', 'y', 'z' })] + [InlineData(new char[] { 'x', '\uD86D', '\uDF54', 'y' }, new int[] { 'x', 0x2B754, 'y' })] // valid surrogate pair + [InlineData(new char[] { 'x', '\uD86D', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // standalone high surrogate + [InlineData(new char[] { 'x', '\uDF54', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // standalone low surrogate + [InlineData(new char[] { 'x', '\uD86D' }, new int[] { 'x', 0xFFFD })] // standalone high surrogate at end of string + [InlineData(new char[] { 'x', '\uDF54' }, new int[] { 'x', 0xFFFD })] // standalone low surrogate at end of string + [InlineData(new char[] { 'x', '\uD86D', '\uD86D', 'y' }, new int[] { 'x', 0xFFFD, 0xFFFD, 'y' })] // two high surrogates should be two replacement chars + [InlineData(new char[] { 'x', '\uFFFD', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // literal U+FFFD + public static void EnumerateRunes(char[] chars, int[] expected) + { + // Test data is smuggled as char[] instead of straight-up string since the test framework + // doesn't like invalid UTF-16 literals. + + // first, test Span + + List enumeratedValues = new List(); + foreach (Rune rune in ((Span)chars).EnumerateRunes()) + { + enumeratedValues.Add(rune.Value); + } + Assert.Equal(expected, enumeratedValues.ToArray()); + + + // next, ROS + + enumeratedValues.Clear(); + foreach (Rune rune in ((ReadOnlySpan)chars).EnumerateRunes()) + { + enumeratedValues.Add(rune.Value); + } + Assert.Equal(expected, enumeratedValues.ToArray()); + } + + [Fact] + public static void EnumerateRunes_DoesNotReadPastEndOfSpan(char[] chars, int[] expected) + { + // As an optimization, reading scalars from a string *may* read past the end of the string + // to the terminating null. This optimization is invalid for arbitrary spans, so this test + // ensures that we're not performing this optimization here. + + ReadOnlySpan span = "xy\U0002B754z".AsSpan(1, 2); // well-formed string, but span splits surrogate pair + + List enumeratedValues = new List(); + foreach (Rune rune in span.EnumerateRunes()) + { + enumeratedValues.Add(rune.Value); + } + Assert.Equal(new int[] { 'y', '\uFFFD' }, enumeratedValues.ToArray()); + } + /// /// Creates a new span over the portion of the target array. /// diff --git a/src/System.Memory/ref/System.Memory.cs b/src/System.Memory/ref/System.Memory.cs index 99e84dcd5049..0bb6fd1dedd7 100644 --- a/src/System.Memory/ref/System.Memory.cs +++ b/src/System.Memory/ref/System.Memory.cs @@ -42,6 +42,8 @@ public static void CopyTo(this T[] source, System.Span destination) { } public static bool EndsWith(this System.ReadOnlySpan span, System.ReadOnlySpan value, System.StringComparison comparisonType) { throw null; } public static bool EndsWith(this System.ReadOnlySpan span, System.ReadOnlySpan value) where T : System.IEquatable { throw null; } public static bool EndsWith(this System.Span span, System.ReadOnlySpan value) where T : System.IEquatable { throw null; } + public static System.Text.SpanRuneEnumerator EnumerateRunes(this ReadOnlySpan span) { throw null; } + public static System.Text.SpanRuneEnumerator EnumerateRunes(this Span span) { throw null; } public static bool Equals(this System.ReadOnlySpan span, System.ReadOnlySpan other, System.StringComparison comparisonType) { throw null; } public static int IndexOf(this System.ReadOnlySpan span, System.ReadOnlySpan value, System.StringComparison comparisonType) { throw null; } public static int IndexOfAny(this System.ReadOnlySpan span, System.ReadOnlySpan values) where T : System.IEquatable { throw null; } @@ -430,3 +432,14 @@ public static partial class SequenceMarshal public static bool TryGetReadOnlySequenceSegment(System.Buffers.ReadOnlySequence sequence, out System.Buffers.ReadOnlySequenceSegment startSegment, out int startIndex, out System.Buffers.ReadOnlySequenceSegment endSegment, out int endIndex) { throw null; } } } +namespace System.Text +{ + public ref partial struct SpanRuneEnumerator + { + private readonly object _dummyReference; + private readonly int _dummyPrimitive; + public Rune Current { get { throw null; } } + public SpanRuneEnumerator GetEnumerator() { throw null; } + public bool MoveNext() { throw null; } + } +} diff --git a/src/System.Memory/src/ApiCompatBaseline.netcoreappaot.txt b/src/System.Memory/src/ApiCompatBaseline.netcoreappaot.txt index 3201fcb77937..d6ecab9d206b 100644 --- a/src/System.Memory/src/ApiCompatBaseline.netcoreappaot.txt +++ b/src/System.Memory/src/ApiCompatBaseline.netcoreappaot.txt @@ -1,4 +1,6 @@ Compat issues with assembly System.Memory: +MembersMustExist : Member 'System.MemoryExtensions.EnumerateRunes(System.ReadOnlySpan)' does not exist in the implementation but it does exist in the contract. +MembersMustExist : Member 'System.MemoryExtensions.EnumerateRunes(System.Span)' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.ReadOnlySpan.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.ReadOnlySpan.Item.get(System.Range)' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.Span.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract. @@ -6,4 +8,5 @@ MembersMustExist : Member 'System.Span.Item.get(System.Range)' does not exist TypesMustExist : Type 'System.Buffers.StandardFormat' does not exist in the implementation but it does exist in the contract. TypesMustExist : Type 'System.Buffers.Text.Utf8Formatter' does not exist in the implementation but it does exist in the contract. TypesMustExist : Type 'System.Buffers.Text.Utf8Parser' does not exist in the implementation but it does exist in the contract. -Total Issues: 7 +TypesMustExist : Type 'System.Text.SpanRuneEnumerator' does not exist in the implementation but it does exist in the contract. +Total Issues: 10 diff --git a/src/System.Memory/src/ApiCompatBaseline.uapaot.txt b/src/System.Memory/src/ApiCompatBaseline.uapaot.txt index 3201fcb77937..d6ecab9d206b 100644 --- a/src/System.Memory/src/ApiCompatBaseline.uapaot.txt +++ b/src/System.Memory/src/ApiCompatBaseline.uapaot.txt @@ -1,4 +1,6 @@ Compat issues with assembly System.Memory: +MembersMustExist : Member 'System.MemoryExtensions.EnumerateRunes(System.ReadOnlySpan)' does not exist in the implementation but it does exist in the contract. +MembersMustExist : Member 'System.MemoryExtensions.EnumerateRunes(System.Span)' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.ReadOnlySpan.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.ReadOnlySpan.Item.get(System.Range)' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.Span.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract. @@ -6,4 +8,5 @@ MembersMustExist : Member 'System.Span.Item.get(System.Range)' does not exist TypesMustExist : Type 'System.Buffers.StandardFormat' does not exist in the implementation but it does exist in the contract. TypesMustExist : Type 'System.Buffers.Text.Utf8Formatter' does not exist in the implementation but it does exist in the contract. TypesMustExist : Type 'System.Buffers.Text.Utf8Parser' does not exist in the implementation but it does exist in the contract. -Total Issues: 7 +TypesMustExist : Type 'System.Text.SpanRuneEnumerator' does not exist in the implementation but it does exist in the contract. +Total Issues: 10 diff --git a/src/System.Runtime/ref/System.Runtime.cs b/src/System.Runtime/ref/System.Runtime.cs index 0ed3c6990749..5f315ebb8f75 100644 --- a/src/System.Runtime/ref/System.Runtime.cs +++ b/src/System.Runtime/ref/System.Runtime.cs @@ -2330,6 +2330,7 @@ public void CopyTo(int sourceIndex, char[] destination, int destinationIndex, in public bool EndsWith(System.String value) { throw null; } public bool EndsWith(System.String value, bool ignoreCase, System.Globalization.CultureInfo culture) { throw null; } public bool EndsWith(System.String value, System.StringComparison comparisonType) { throw null; } + public System.Text.StringRuneEnumerator EnumerateRunes() { throw null; } public override bool Equals(object obj) { throw null; } public bool Equals(System.String value) { throw null; } public static bool Equals(System.String a, System.String b) { throw null; } @@ -7784,6 +7785,19 @@ public partial struct ChunkEnumerator public bool MoveNext() { throw null; } } } + public partial struct StringRuneEnumerator : System.Collections.Generic.IEnumerable, System.Collections.Generic.IEnumerator + { + private readonly object _dummyReference; + private readonly int _dummyPrimitive; + public System.Text.Rune Current { get { throw null; } } + public StringRuneEnumerator GetEnumerator() { throw null; } + public bool MoveNext() { throw null; } + object System.Collections.IEnumerator.Current { get { throw null; } } + void IDisposable.Dispose() { } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { throw null; } + System.Collections.Generic.IEnumerator System.Collections.Generic.IEnumerable.GetEnumerator() { throw null; } + void System.Collections.IEnumerator.Reset() { } + } } namespace System.Threading { diff --git a/src/System.Runtime/src/ApiCompatBaseline.netcoreappaot.txt b/src/System.Runtime/src/ApiCompatBaseline.netcoreappaot.txt index 251d803963de..a305c33080dd 100644 --- a/src/System.Runtime/src/ApiCompatBaseline.netcoreappaot.txt +++ b/src/System.Runtime/src/ApiCompatBaseline.netcoreappaot.txt @@ -7,6 +7,7 @@ TypesMustExist : Type 'System.ArgIterator' does not exist in the implementation TypesMustExist : Type 'System.Index' does not exist in the implementation but it does exist in the contract. TypesMustExist : Type 'System.Range' does not exist in the implementation but it does exist in the contract. TypesMustExist : Type 'System.Text.Rune' does not exist in the implementation but it does exist in the contract. +TypesMustExist : Type 'System.Text.StringRuneEnumerator' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.ReadOnlySpan.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.ReadOnlySpan.Item.get(System.Range)' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.Span.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract. diff --git a/src/System.Runtime/src/ApiCompatBaseline.uapaot.txt b/src/System.Runtime/src/ApiCompatBaseline.uapaot.txt index b1c3f4ba9069..72bc8dca0cf1 100644 --- a/src/System.Runtime/src/ApiCompatBaseline.uapaot.txt +++ b/src/System.Runtime/src/ApiCompatBaseline.uapaot.txt @@ -2,6 +2,7 @@ TypesMustExist : Type 'System.ArgIterator' does not exist in the implementation TypesMustExist : Type 'System.Index' does not exist in the implementation but it does exist in the contract. TypesMustExist : Type 'System.Range' does not exist in the implementation but it does exist in the contract. TypesMustExist : Type 'System.Text.Rune' does not exist in the implementation but it does exist in the contract. +TypesMustExist : Type 'System.Text.StringRuneEnumerator' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.ReadOnlySpan.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.ReadOnlySpan.Item.get(System.Range)' does not exist in the implementation but it does exist in the contract. MembersMustExist : Member 'System.Span.Item.get(System.Index)' does not exist in the implementation but it does exist in the contract. diff --git a/src/System.Runtime/tests/System/StringTests.netcoreapp.cs b/src/System.Runtime/tests/System/StringTests.netcoreapp.cs index 3ddfcf49bfd9..555513973dbe 100644 --- a/src/System.Runtime/tests/System/StringTests.netcoreapp.cs +++ b/src/System.Runtime/tests/System/StringTests.netcoreapp.cs @@ -6,6 +6,7 @@ using System.Globalization; using System.Linq; using System.Runtime.InteropServices; +using System.Text; using Xunit; namespace System.Tests @@ -411,6 +412,38 @@ public static void EndsWith(string s, char value, bool expected) Assert.Equal(expected, s.EndsWith(value)); } + [Theory] + [InlineData(new char[0], new int[0])] // empty + [InlineData(new char[] { 'x', 'y', 'z' }, new int[] { 'x', 'y', 'z' })] + [InlineData(new char[] { 'x', '\uD86D', '\uDF54', 'y' }, new int[] { 'x', 0x2B754, 'y' })] // valid surrogate pair + [InlineData(new char[] { 'x', '\uD86D', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // standalone high surrogate + [InlineData(new char[] { 'x', '\uDF54', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // standalone low surrogate + [InlineData(new char[] { 'x', '\uD86D' }, new int[] { 'x', 0xFFFD })] // standalone high surrogate at end of string + [InlineData(new char[] { 'x', '\uDF54' }, new int[] { 'x', 0xFFFD })] // standalone low surrogate at end of string + [InlineData(new char[] { 'x', '\uD86D', '\uD86D', 'y' }, new int[] { 'x', 0xFFFD, 0xFFFD, 'y' })] // two high surrogates should be two replacement chars + [InlineData(new char[] { 'x', '\uFFFD', 'y' }, new int[] { 'x', 0xFFFD, 'y' })] // literal U+FFFD + public static void EnumerateRunes(char[] chars, int[] expected) + { + // Test data is smuggled as char[] instead of straight-up string since the test framework + // doesn't like invalid UTF-16 literals. + + string asString = new string(chars); + + // First, use a straight-up foreach keyword to ensure pattern matching works as expected + + List enumeratedScalarValues = new List(); + foreach (Rune rune in asString.EnumerateRunes()) + { + enumeratedScalarValues.Add(rune.Value); + } + Assert.Equal(expected, enumeratedScalarValues.ToArray()); + + // Then use LINQ to ensure IEnumerator<...> works as expected + + int[] enumeratedValues = new string(chars).EnumerateRunes().Select(r => r.Value).ToArray(); + Assert.Equal(expected, enumeratedValues); + } + [Theory] [InlineData("Hello", 'H', true)] [InlineData("Hello", 'h', false)]