Revert "Use spans in low level lexer char array handling code (#79232)"

333fred · web-flow · commit 0d92d27d14b3 · 2025-07-03T10:29:05.000-07:00
This reverts commit fb38d6b.
diff --git a/src/Compilers/CSharp/Portable/Parser/Lexer.cs b/src/Compilers/CSharp/Portable/Parser/Lexer.cs
@@ -2337,7 +2337,9 @@ private SyntaxTrivia ScanWhitespace()
                 if (width < MaxCachedTokenSize)
                 {
                     return _cache.LookupTrivia(
-                        TextWindow.CharacterWindow.AsSpan(TextWindow.LexemeRelativeStart, width),
+                        TextWindow.CharacterWindow,
+                        TextWindow.LexemeRelativeStart,
+                        width,
                         hashCode,
                         CreateWhitespaceTrivia,
                         TextWindow);
diff --git a/src/Compilers/CSharp/Portable/Parser/LexerCache.cs b/src/Compilers/CSharp/Portable/Parser/LexerCache.cs
@@ -182,17 +182,19 @@ internal bool TryGetKeywordKind(string key, out SyntaxKind kind)
         }
 
         internal SyntaxTrivia LookupTrivia<TArg>(
-            ReadOnlySpan<char> textBuffer,
+            char[] textBuffer,
+            int keyStart,
+            int keyLength,
             int hashCode,
             Func<TArg, SyntaxTrivia> createTriviaFunction,
             TArg data)
         {
-            var value = TriviaMap.FindItem(textBuffer, hashCode);
+            var value = TriviaMap.FindItem(textBuffer, keyStart, keyLength, hashCode);
 
             if (value == null)
             {
                 value = createTriviaFunction(data);
-                TriviaMap.AddItem(textBuffer, hashCode, value);
+                TriviaMap.AddItem(textBuffer, keyStart, keyLength, hashCode, value);
             }
 
             return value;
@@ -220,20 +222,22 @@ private static void Miss()
 #endif
 
         internal SyntaxToken LookupToken<TArg>(
-            ReadOnlySpan<char> textBuffer,
+            char[] textBuffer,
+            int keyStart,
+            int keyLength,
             int hashCode,
             Func<TArg, SyntaxToken> createTokenFunction,
             TArg data)
         {
-            var value = TokenMap.FindItem(textBuffer, hashCode);
+            var value = TokenMap.FindItem(textBuffer, keyStart, keyLength, hashCode);
 
             if (value == null)
             {
 #if COLLECT_STATS
                     Miss();
 #endif
                 value = createTokenFunction(data);
-                TokenMap.AddItem(textBuffer, hashCode, value);
+                TokenMap.AddItem(textBuffer, keyStart, keyLength, hashCode, value);
             }
             else
             {
diff --git a/src/Compilers/CSharp/Portable/Parser/QuickScanner.cs b/src/Compilers/CSharp/Portable/Parser/QuickScanner.cs
@@ -235,7 +235,9 @@ private enum CharFlags : byte
             {
                 // this is a good token!
                 var token = _cache.LookupToken(
-                    TextWindow.CharacterWindow.AsSpan(TextWindow.LexemeRelativeStart, i - TextWindow.LexemeRelativeStart),
+                    TextWindow.CharacterWindow,
+                    TextWindow.LexemeRelativeStart,
+                    i - TextWindow.LexemeRelativeStart,
                     hashCode,
                     CreateQuickToken,
                     this);
diff --git a/src/Compilers/CSharp/Portable/Parser/SlidingTextWindow.cs b/src/Compilers/CSharp/Portable/Parser/SlidingTextWindow.cs
@@ -416,7 +416,7 @@ public string Intern(StringBuilder text)
 
         public string Intern(char[] array, int start, int length)
         {
-            return _strings.Add(array.AsSpan(start, length));
+            return _strings.Add(array, start, length);
         }
 
         public string GetInternedText()
diff --git a/src/Compilers/Core/CodeAnalysisTest/StringTableTests.cs b/src/Compilers/Core/CodeAnalysisTest/StringTableTests.cs
@@ -48,7 +48,7 @@ public void TestAddSameWithVariousInputsProducesSameStringInstance()
             var s3 = st.Add(" ");
             Assert.Same(s2, s3);
 
-            var s4 = st.Add([' ']);
+            var s4 = st.Add(new char[1] { ' ' }, 0, 1);
             Assert.Same(s3, s4);
 
             var s5 = st.Add("ABC DEF", 3, 1);
diff --git a/src/Compilers/Core/Portable/InternalUtilities/Hash.cs b/src/Compilers/Core/Portable/InternalUtilities/Hash.cs
@@ -351,7 +351,17 @@ internal static int GetFNVHashCode(System.Text.StringBuilder text)
         /// <param name="length">The number of characters, beginning with <paramref name="start"/> to hash</param>
         /// <returns>The FNV-1a hash code of the substring beginning at <paramref name="start"/> and ending after <paramref name="length"/> characters.</returns>
         internal static int GetFNVHashCode(char[] text, int start, int length)
-            => GetFNVHashCode(text.AsSpan(start, length));
+        {
+            int hashCode = Hash.FnvOffsetBias;
+            int end = start + length;
+
+            for (int i = start; i < end; i++)
+            {
+                hashCode = unchecked((hashCode ^ text[i]) * Hash.FnvPrime);
+            }
+
+            return hashCode;
+        }
 
         /// <summary>
         /// Compute the hashcode of a single character using the FNV-1a algorithm
diff --git a/src/Compilers/Core/Portable/InternalUtilities/StringTable.cs b/src/Compilers/Core/Portable/InternalUtilities/StringTable.cs
@@ -111,9 +111,10 @@ public void Free()
 
         #endregion // Poolable
 
-        internal string Add(ReadOnlySpan<char> chars)
+        internal string Add(char[] chars, int start, int len)
         {
-            var hashCode = Hash.GetFNVHashCode(chars);
+            var span = chars.AsSpan(start, len);
+            var hashCode = Hash.GetFNVHashCode(chars, start, len);
 
             // capture array to avoid extra range checks
             var arr = _localTable;
@@ -124,13 +125,13 @@ internal string Add(ReadOnlySpan<char> chars)
             if (text != null && arr[idx].HashCode == hashCode)
             {
                 var result = arr[idx].Text;
-                if (TextEquals(result, chars))
+                if (StringTable.TextEquals(result, span))
                 {
                     return result;
                 }
             }
 
-            string? shared = FindSharedEntry(chars, hashCode);
+            string? shared = FindSharedEntry(chars, start, len, hashCode);
             if (shared != null)
             {
                 // PERF: the following code does element-wise assignment of a struct
@@ -142,7 +143,7 @@ internal string Add(ReadOnlySpan<char> chars)
                 return shared;
             }
 
-            return AddItem(chars, hashCode);
+            return AddItem(chars, start, len, hashCode);
         }
 
         internal string Add(string chars, int start, int len)
@@ -282,7 +283,7 @@ internal string Add(string chars)
             return chars;
         }
 
-        private static string? FindSharedEntry(ReadOnlySpan<char> chars, int hashCode)
+        private static string? FindSharedEntry(char[] chars, int start, int len, int hashCode)
         {
             var arr = s_sharedTable;
             int idx = SharedIdxFromHash(hashCode);
@@ -297,7 +298,7 @@ internal string Add(string chars)
 
                 if (e != null)
                 {
-                    if (hash == hashCode && TextEquals(e, chars))
+                    if (hash == hashCode && TextEquals(e, chars.AsSpan(start, len)))
                     {
                         break;
                     }
@@ -491,9 +492,9 @@ internal string Add(string chars)
             return e;
         }
 
-        private string AddItem(ReadOnlySpan<char> chars, int hashCode)
+        private string AddItem(char[] chars, int start, int len, int hashCode)
         {
-            var text = chars.ToString();
+            var text = new String(chars, start, len);
             AddCore(text, hashCode);
             return text;
         }
diff --git a/src/Compilers/Core/Portable/InternalUtilities/TextKeyedCache.cs b/src/Compilers/Core/Portable/InternalUtilities/TextKeyedCache.cs
@@ -105,7 +105,7 @@ public void Free()
 
         #endregion // Poolable
 
-        internal T? FindItem(ReadOnlySpan<char> chars, int hashCode)
+        internal T? FindItem(char[] chars, int start, int len, int hashCode)
         {
             // get direct element reference to avoid extra range checks
             ref var localSlot = ref _localTable[LocalIdxFromHash(hashCode)];
@@ -114,13 +114,13 @@ public void Free()
 
             if (text != null && localSlot.HashCode == hashCode)
             {
-                if (StringTable.TextEquals(text, chars))
+                if (StringTable.TextEquals(text, chars.AsSpan(start, len)))
                 {
                     return localSlot.Item;
                 }
             }
 
-            SharedEntryValue? e = FindSharedEntry(chars, hashCode);
+            SharedEntryValue? e = FindSharedEntry(chars, start, len, hashCode);
             if (e != null)
             {
                 localSlot.HashCode = hashCode;
@@ -135,7 +135,7 @@ public void Free()
             return null!;
         }
 
-        private SharedEntryValue? FindSharedEntry(ReadOnlySpan<char> chars, int hashCode)
+        private SharedEntryValue? FindSharedEntry(char[] chars, int start, int len, int hashCode)
         {
             var arr = _sharedTableInst;
             int idx = SharedIdxFromHash(hashCode);
@@ -151,7 +151,7 @@ public void Free()
 
                 if (e != null)
                 {
-                    if (hash == hashCode && StringTable.TextEquals(e.Text, chars))
+                    if (hash == hashCode && StringTable.TextEquals(e.Text, chars.AsSpan(start, len)))
                     {
                         break;
                     }
@@ -171,9 +171,9 @@ public void Free()
             return e;
         }
 
-        internal void AddItem(ReadOnlySpan<char> chars, int hashCode, T item)
+        internal void AddItem(char[] chars, int start, int len, int hashCode, T item)
         {
-            var text = _strings.Add(chars);
+            var text = _strings.Add(chars, start, len);
 
             // add to the shared table first (in case someone looks for same item)
             var e = new SharedEntryValue(text, item);
diff --git a/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb b/src/Compilers/VisualBasic/Portable/Scanner/Scanner.vb
@@ -166,7 +166,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
             Dim quickToken = QuickScanToken(allowLeadingMultilineTrivia)
 
             If quickToken.Succeeded Then
-                Dim token = _quickTokenTable.FindItem(quickToken.Chars.AsSpan(quickToken.Start, quickToken.Length), quickToken.HashCode)
+                Dim token = _quickTokenTable.FindItem(quickToken.Chars, quickToken.Start, quickToken.Length, quickToken.HashCode)
                 If token IsNot Nothing Then
                     AdvanceChar(quickToken.Length)
                     If quickToken.TerminatorLength <> 0 Then
@@ -185,7 +185,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
             If quickToken.Succeeded Then
                 Debug.Assert(quickToken.Length = scannedToken.FullWidth)
 
-                _quickTokenTable.AddItem(quickToken.Chars.AsSpan(quickToken.Start, quickToken.Length), quickToken.HashCode, scannedToken)
+                _quickTokenTable.AddItem(quickToken.Chars, quickToken.Start, quickToken.Length, quickToken.HashCode, scannedToken)
             End If
 
             Return scannedToken
@@ -399,7 +399,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
             Return _stringTable.Add(ch)
         End Function
         Friend Function Intern(arr As Char()) As String
-            Return _stringTable.Add(arr.AsSpan())
+            Return _stringTable.Add(arr)
         End Function
 #End Region
 
diff --git a/src/Compilers/VisualBasic/Test/Syntax/QuickTokenTableTests.vb b/src/Compilers/VisualBasic/Test/Syntax/QuickTokenTableTests.vb
@@ -89,8 +89,8 @@ Public Class QuickTokenTableTests
             Dim text = New String(qt.Chars, 0, qt.Length)
             Dim token = InternalSyntax.SyntaxFactory.Identifier(text)
 
-            Assert.Null(table.FindItem(qt.Chars.AsSpan(qt.Start, qt.Length), qt.HashCode))
-            table.AddItem(qt.Chars.AsSpan(qt.Start, qt.Length), qt.HashCode, DirectCast(token, InternalSyntax.SyntaxToken))
+            Assert.Null(table.FindItem(qt.Chars, qt.Start, qt.Length, qt.HashCode))
+            table.AddItem(qt.Chars, qt.Start, qt.Length, qt.HashCode, DirectCast(token, InternalSyntax.SyntaxToken))
             Return New Tuple(Of String, InternalSyntax.SyntaxToken)(text, token)
         End Using
     End Function
@@ -107,7 +107,7 @@ Public Class QuickTokenTableTests
             Dim qt = scanner.QuickScanToken(False)
             Assert.True(qt.Succeeded)
 
-            Dim tokFound = table.FindItem(qt.Chars.AsSpan(qt.Start, qt.Length), qt.HashCode)
+            Dim tokFound = table.FindItem(qt.Chars, qt.Start, qt.Length, qt.HashCode)
             Assert.Same(e.Item2, tokFound)
         End Using
     End Sub

Original file line number	Diff line number	Diff line change
`@@ -182,17 +182,19 @@ internal bool TryGetKeywordKind(string key, out SyntaxKind kind)`
`182`	`182`	`}`
`183`	`183`
`184`	`184`	`internal SyntaxTrivia LookupTrivia<TArg>(`
`185`		`- ReadOnlySpan<char> textBuffer,`
	`185`	`+ char[] textBuffer,`
	`186`	`+ int keyStart,`
	`187`	`+ int keyLength,`
`186`	`188`	`int hashCode,`
`187`	`189`	`Func<TArg, SyntaxTrivia> createTriviaFunction,`
`188`	`190`	`TArg data)`
`189`	`191`	`{`
`190`		`- var value = TriviaMap.FindItem(textBuffer, hashCode);`
	`192`	`+ var value = TriviaMap.FindItem(textBuffer, keyStart, keyLength, hashCode);`
`191`	`193`
`192`	`194`	`if (value == null)`
`193`	`195`	`{`
`194`	`196`	`value = createTriviaFunction(data);`
`195`		`- TriviaMap.AddItem(textBuffer, hashCode, value);`
	`197`	`+ TriviaMap.AddItem(textBuffer, keyStart, keyLength, hashCode, value);`
`196`	`198`	`}`
`197`	`199`
`198`	`200`	`return value;`
`@@ -220,20 +222,22 @@ private static void Miss()`
`220`	`222`	`#endif`
`221`	`223`
`222`	`224`	`internal SyntaxToken LookupToken<TArg>(`
`223`		`- ReadOnlySpan<char> textBuffer,`
	`225`	`+ char[] textBuffer,`
	`226`	`+ int keyStart,`
	`227`	`+ int keyLength,`
`224`	`228`	`int hashCode,`
`225`	`229`	`Func<TArg, SyntaxToken> createTokenFunction,`
`226`	`230`	`TArg data)`
`227`	`231`	`{`
`228`		`- var value = TokenMap.FindItem(textBuffer, hashCode);`
	`232`	`+ var value = TokenMap.FindItem(textBuffer, keyStart, keyLength, hashCode);`
`229`	`233`
`230`	`234`	`if (value == null)`
`231`	`235`	`{`
`232`	`236`	`#if COLLECT_STATS`
`233`	`237`	`Miss();`
`234`	`238`	`#endif`
`235`	`239`	`value = createTokenFunction(data);`
`236`		`- TokenMap.AddItem(textBuffer, hashCode, value);`
	`240`	`+ TokenMap.AddItem(textBuffer, keyStart, keyLength, hashCode, value);`
`237`	`241`	`}`
`238`	`242`	`else`
`239`	`243`	`{`
Original file line number	Diff line number	Diff line change
`@@ -416,7 +416,7 @@ public string Intern(StringBuilder text)`
`416`	`416`
`417`	`417`	`public string Intern(char[] array, int start, int length)`
`418`	`418`	`{`
`419`		`- return _strings.Add(array.AsSpan(start, length));`
	`419`	`+ return _strings.Add(array, start, length);`
`420`	`420`	`}`
`421`	`421`
`422`	`422`	`public string GetInternedText()`
Original file line number	Diff line number	Diff line change
`@@ -111,9 +111,10 @@ public void Free()`
`111`	`111`
`112`	`112`	`#endregion // Poolable`
`113`	`113`
`114`		`- internal string Add(ReadOnlySpan<char> chars)`
	`114`	`+ internal string Add(char[] chars, int start, int len)`
`115`	`115`	`{`
`116`		`- var hashCode = Hash.GetFNVHashCode(chars);`
	`116`	`+ var span = chars.AsSpan(start, len);`
	`117`	`+ var hashCode = Hash.GetFNVHashCode(chars, start, len);`
`117`	`118`
`118`	`119`	`// capture array to avoid extra range checks`
`119`	`120`	`var arr = _localTable;`
`@@ -124,13 +125,13 @@ internal string Add(ReadOnlySpan<char> chars)`
`124`	`125`	`if (text != null && arr[idx].HashCode == hashCode)`
`125`	`126`	`{`
`126`	`127`	`var result = arr[idx].Text;`
`127`		`- if (TextEquals(result, chars))`
	`128`	`+ if (StringTable.TextEquals(result, span))`
`128`	`129`	`{`
`129`	`130`	`return result;`
`130`	`131`	`}`
`131`	`132`	`}`
`132`	`133`
`133`		`- string? shared = FindSharedEntry(chars, hashCode);`
	`134`	`+ string? shared = FindSharedEntry(chars, start, len, hashCode);`
`134`	`135`	`if (shared != null)`
`135`	`136`	`{`
`136`	`137`	`// PERF: the following code does element-wise assignment of a struct`
`@@ -142,7 +143,7 @@ internal string Add(ReadOnlySpan<char> chars)`
`142`	`143`	`return shared;`
`143`	`144`	`}`
`144`	`145`
`145`		`- return AddItem(chars, hashCode);`
	`146`	`+ return AddItem(chars, start, len, hashCode);`
`146`	`147`	`}`
`147`	`148`
`148`	`149`	`internal string Add(string chars, int start, int len)`
`@@ -282,7 +283,7 @@ internal string Add(string chars)`
`282`	`283`	`return chars;`
`283`	`284`	`}`
`284`	`285`
`285`		`- private static string? FindSharedEntry(ReadOnlySpan<char> chars, int hashCode)`
	`286`	`+ private static string? FindSharedEntry(char[] chars, int start, int len, int hashCode)`
`286`	`287`	`{`
`287`	`288`	`var arr = s_sharedTable;`
`288`	`289`	`int idx = SharedIdxFromHash(hashCode);`
`@@ -297,7 +298,7 @@ internal string Add(string chars)`
`297`	`298`
`298`	`299`	`if (e != null)`
`299`	`300`	`{`
`300`		`- if (hash == hashCode && TextEquals(e, chars))`
	`301`	`+ if (hash == hashCode && TextEquals(e, chars.AsSpan(start, len)))`
`301`	`302`	`{`
`302`	`303`	`break;`
`303`	`304`	`}`
`@@ -491,9 +492,9 @@ internal string Add(string chars)`
`491`	`492`	`return e;`
`492`	`493`	`}`
`493`	`494`
`494`		`- private string AddItem(ReadOnlySpan<char> chars, int hashCode)`
	`495`	`+ private string AddItem(char[] chars, int start, int len, int hashCode)`
`495`	`496`	`{`
`496`		`- var text = chars.ToString();`
	`497`	`+ var text = new String(chars, start, len);`
`497`	`498`	`AddCore(text, hashCode);`
`498`	`499`	`return text;`
`499`	`500`	`}`
Original file line number	Diff line number	Diff line change
`@@ -105,7 +105,7 @@ public void Free()`
`105`	`105`
`106`	`106`	`#endregion // Poolable`
`107`	`107`
`108`		`- internal T? FindItem(ReadOnlySpan<char> chars, int hashCode)`
	`108`	`+ internal T? FindItem(char[] chars, int start, int len, int hashCode)`
`109`	`109`	`{`
`110`	`110`	`// get direct element reference to avoid extra range checks`
`111`	`111`	`ref var localSlot = ref _localTable[LocalIdxFromHash(hashCode)];`
`@@ -114,13 +114,13 @@ public void Free()`
`114`	`114`
`115`	`115`	`if (text != null && localSlot.HashCode == hashCode)`
`116`	`116`	`{`
`117`		`- if (StringTable.TextEquals(text, chars))`
	`117`	`+ if (StringTable.TextEquals(text, chars.AsSpan(start, len)))`
`118`	`118`	`{`
`119`	`119`	`return localSlot.Item;`
`120`	`120`	`}`
`121`	`121`	`}`
`122`	`122`
`123`		`- SharedEntryValue? e = FindSharedEntry(chars, hashCode);`
	`123`	`+ SharedEntryValue? e = FindSharedEntry(chars, start, len, hashCode);`
`124`	`124`	`if (e != null)`
`125`	`125`	`{`
`126`	`126`	`localSlot.HashCode = hashCode;`
`@@ -135,7 +135,7 @@ public void Free()`
`135`	`135`	`return null!;`
`136`	`136`	`}`
`137`	`137`
`138`		`- private SharedEntryValue? FindSharedEntry(ReadOnlySpan<char> chars, int hashCode)`
	`138`	`+ private SharedEntryValue? FindSharedEntry(char[] chars, int start, int len, int hashCode)`
`139`	`139`	`{`
`140`	`140`	`var arr = _sharedTableInst;`
`141`	`141`	`int idx = SharedIdxFromHash(hashCode);`
`@@ -151,7 +151,7 @@ public void Free()`
`151`	`151`
`152`	`152`	`if (e != null)`
`153`	`153`	`{`
`154`		`- if (hash == hashCode && StringTable.TextEquals(e.Text, chars))`
	`154`	`+ if (hash == hashCode && StringTable.TextEquals(e.Text, chars.AsSpan(start, len)))`
`155`	`155`	`{`
`156`	`156`	`break;`
`157`	`157`	`}`
`@@ -171,9 +171,9 @@ public void Free()`
`171`	`171`	`return e;`
`172`	`172`	`}`
`173`	`173`
`174`		`- internal void AddItem(ReadOnlySpan<char> chars, int hashCode, T item)`
	`174`	`+ internal void AddItem(char[] chars, int start, int len, int hashCode, T item)`
`175`	`175`	`{`
`176`		`- var text = _strings.Add(chars);`
	`176`	`+ var text = _strings.Add(chars, start, len);`
`177`	`177`
`178`	`178`	`// add to the shared table first (in case someone looks for same item)`
`179`	`179`	`var e = new SharedEntryValue(text, item);`