Fix BinaryReader.ReadChars for fragmented Streams (dotnet#26324) (#26…

…356)

BinaryReader.ReadChars incorrectly read more than necessary from the underlying Stream when multi-byte characters straddled the read chunks.

Fixes https://github.com/dotnet/corefx/issues/40455

Signed-off-by: dotnet-bot <dotnet-bot@microsoft.com>

src/Common/src/CoreLib/System/IO/BinaryReader.cs

@@ -390,6 +390,28 @@ private int InternalReadChars(Span<char> buffer)
                 {
                     numBytes <<= 1;
                 }
+
+                // We do not want to read even a single byte more than necessary.
+                //
+                // Subtract pending bytes that the decoder may be holding onto. This assumes that each
+                // decoded char corresponds to one or more bytes. Note that custom encodings or encodings with
+                // a custom replacement sequence may violate this assumption.
+                if (numBytes > 1)
+                {
+                    DecoderNLS? decoder = _decoder as DecoderNLS;
+                    // For internal decoders, we can check whether the decoder has any pending state.
+                    // For custom decoders, assume that the decoder has pending state.
+                    if (decoder == null || decoder.HasState)
+                    {
+                        numBytes -= 1;
+
+                        // The worst case is charsRemaining = 2 and UTF32Decoder holding onto 3 pending bytes. We need to read just
+                        // one byte in this case.
+                        if (_2BytesPerChar && numBytes > 2)
+                            numBytes -= 2;
+                    }
+                }
+
                 if (numBytes > MaxCharBytesSize)
                 {
                     numBytes = MaxCharBytesSize;

src/Common/src/CoreLib/System/Text/DecoderNLS.cs

@@ -227,7 +227,7 @@ internal virtual bool HasState
         {
             get
             {
-                return false;
+                return _leftoverByteCount != 0;
             }
         }