JSON continuation tests

[devsko]

See #42158
comment and comment and comment

Test only

Test several scenarios in System.Text.Json where deserialization has to continue after the next chunk of data is available.

[devsko]

@layomia This is not yet ready for review. I hope to get done what I have in mind tomorrow.

[devsko]

Is there anything else about continuation / chunked buffer?

[ahsonkhan]

Is there anything else about continuation / chunked buffer?

Since you asked, how about continuation where the token being split isn't null but other types, like whitespace (\r\n), or true/false boolean, or some large string token, or a number?

In the Utf8JsonReader specific tests, a lot of those scenarios are covered, by building ROSequence with segments splits across a whole variety of locations within the JSON, and with partial data/state to test re-entrancy. But most of those are for relatively small payloads (due to test time), and the async Deserializer API for streams could benefit from that type of extensive coverage too :)

runtime/src/libraries/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs

Lines 474 to 643 in e0e1919

	// TestCaseType is only used to give the json strings a descriptive name.
	[Theory]
	// Skipping large JSON since slicing them (O(n^2)) is too slow.
	[MemberData(nameof(SmallTestCases))]
	public static void TestJsonReaderUtf8SegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
	{
	ReadPartialSegmentSizeOne(compactData, type, jsonString);
	}
	// TestCaseType is only used to give the json strings a descriptive name.
	[Theory]
	[MemberData(nameof(LargeTestCases))]
	public static void TestJsonReaderLargeUtf8SegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
	{
	// Skipping really large JSON on Browser to prevent OOM
	if (PlatformDetection.IsBrowser && (type == TestCaseType.Json40KB || type == TestCaseType.Json400KB || type == TestCaseType.ProjectLockJson))
	{
	return;
	}
	ReadFullySegmentSizeOne(compactData, type, jsonString);
	}
	// TestCaseType is only used to give the json strings a descriptive name.
	[Theory]
	[OuterLoop]
	[MemberData(nameof(LargeTestCases))]
	public static void TestJsonReaderLargestUtf8SegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
	{
	// Skipping really large JSON since slicing them (O(n^2)) is too slow.
	if (type == TestCaseType.Json40KB || type == TestCaseType.Json400KB || type == TestCaseType.ProjectLockJson)
	{
	return;
	}
	ReadPartialSegmentSizeOne(compactData, type, jsonString);
	}
	private static void ReadPartialSegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
	{
	// Remove all formatting/indendation
	if (compactData)
	{
	jsonString = JsonTestHelper.GetCompactString(jsonString);
	}
	byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
	Stream stream = new MemoryStream(dataUtf8);
	TextReader reader = new StreamReader(stream, Encoding.UTF8, false, 1024, true);
	string expectedStr = JsonTestHelper.NewtonsoftReturnStringHelper(reader);
	ReadOnlySequence<byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
	for (int j = 0; j < dataUtf8.Length; j++)
	{
	var utf8JsonReader = new Utf8JsonReader(sequence.Slice(0, j), isFinalBlock: false, default);
	byte[] resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out int length, ref utf8JsonReader);
	string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length);
	long consumed = utf8JsonReader.BytesConsumed;
	utf8JsonReader = new Utf8JsonReader(sequence.Slice(consumed), isFinalBlock: true, utf8JsonReader.CurrentState);
	resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out length, ref utf8JsonReader);
	actualStrSequence += Encoding.UTF8.GetString(resultSequence, 0, length);
	string message = $"Expected consumed: {dataUtf8.Length - consumed}, Actual consumed: {utf8JsonReader.BytesConsumed}, Index: {j}";
	Assert.True(dataUtf8.Length - consumed == utf8JsonReader.BytesConsumed, message);
	Assert.Equal(expectedStr, actualStrSequence);
	}
	}
	private static void ReadFullySegmentSizeOne(bool compactData, TestCaseType type, string jsonString)
	{
	// Remove all formatting/indendation
	if (compactData)
	{
	jsonString = JsonTestHelper.GetCompactString(jsonString);
	}
	byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
	Stream stream = new MemoryStream(dataUtf8);
	TextReader reader = new StreamReader(stream, Encoding.UTF8, false, 1024, true);
	string expectedStr = JsonTestHelper.NewtonsoftReturnStringHelper(reader);
	ReadOnlySequence<byte> sequence = JsonTestHelper.GetSequence(dataUtf8, 1);
	var utf8JsonReader = new Utf8JsonReader(sequence, isFinalBlock: true, default);
	byte[] resultSequence = JsonTestHelper.ReaderLoop(dataUtf8.Length, out int length, ref utf8JsonReader);
	string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length);
	Assert.Equal(expectedStr, actualStrSequence);
	}
	[Theory]
	[MemberData(nameof(SmallTestCases))]
	public static void TestPartialJsonReaderMultiSegment(bool compactData, TestCaseType type, string jsonString)
	{
	_ = type;
	// Remove all formatting/indendation
	if (compactData)
	{
	jsonString = JsonTestHelper.GetCompactString(jsonString);
	}
	byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
	ReadOnlyMemory<byte> dataMemory = dataUtf8;
	List<ReadOnlySequence<byte>> sequences = JsonTestHelper.GetSequences(dataMemory);
	for (int i = 0; i < sequences.Count; i++)
	{
	ReadOnlySequence<byte> sequence = sequences[i];
	var json = new Utf8JsonReader(sequence, isFinalBlock: true, default);
	while (json.Read())
	;
	Assert.Equal(sequence.Length, json.BytesConsumed);
	Assert.True(sequence.Slice(json.Position).IsEmpty);
	}
	for (int i = 0; i < sequences.Count; i++)
	{
	ReadOnlySequence<byte> sequence = sequences[i];
	var json = new Utf8JsonReader(sequence);
	while (json.Read())
	;
	Assert.Equal(sequence.Length, json.BytesConsumed);
	Assert.True(sequence.Slice(json.Position).IsEmpty);
	}
	}
	[Theory]
	[OuterLoop]
	[MemberData(nameof(SmallTestCases))]
	public static void TestPartialJsonReaderSlicesMultiSegment(bool compactData, TestCaseType type, string jsonString)
	{
	_ = type;
	// Remove all formatting/indendation
	if (compactData)
	{
	jsonString = JsonTestHelper.GetCompactString(jsonString);
	}
	byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString);
	ReadOnlyMemory<byte> dataMemory = dataUtf8;
	List<ReadOnlySequence<byte>> sequences = JsonTestHelper.GetSequences(dataMemory);
	for (int i = 0; i < sequences.Count; i++)
	{
	ReadOnlySequence<byte> sequence = sequences[i];
	for (int j = 0; j < dataUtf8.Length; j++)
	{
	var json = new Utf8JsonReader(sequence.Slice(0, j), isFinalBlock: false, default);
	while (json.Read())
	;
	long consumed = json.BytesConsumed;
	JsonReaderState jsonState = json.CurrentState;
	byte[] consumedArray = sequence.Slice(0, consumed).ToArray();
	Assert.Equal(consumedArray, sequence.Slice(0, json.Position).ToArray());
	json = new Utf8JsonReader(sequence.Slice(consumed), isFinalBlock: true, jsonState);
	while (json.Read())
	;
	Assert.Equal(dataUtf8.Length - consumed, json.BytesConsumed);
	}
	}
	}

[devsko]

how about continuation where the token being split isn't null but other types, like whitespace (\r\n), or true/false boolean, or some large string token, or a number?

All tests here split the payload once on every single character. Thus all tokens are tested how they work when split into 2 chunks including all mentioned examples except whitespaces. I will add them by enabling WriteIndented

[devsko]

I'd say that's it. Thanks for your suggestions, help and reviews. Really appreciated. Feel free to change whatever you want - or wait 2 weeks. See you - peace

[stephentoub]

@devsko, thanks for your efforts here. Are you still working on this?

[layomia]

LGTM - @devsko we can merge this once conflicts and #42393 (comment) are resolved.

[layomia]

I pushed a commit to finish this PR.