Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create class for reading Json files in chunks #5530

Merged
merged 30 commits into from
Dec 12, 2023
Merged

Create class for reading Json files in chunks #5530

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
d1e4d8a
Moved files over and addressed some PR comments
jgonz120 Dec 5, 2023
9ec0869
added comment
jgonz120 Dec 5, 2023
cfa2169
switched to true and false strings
jgonz120 Dec 5, 2023
c1753f9
Added ctr to specify buffer for testing purposes.
jgonz120 Dec 5, 2023
cc2ad30
remove commented code
jgonz120 Dec 6, 2023
32ec713
switch to use Utf8 preamble for BOM
jgonz120 Dec 6, 2023
a9940e9
Create method for checking complete
jgonz120 Dec 6, 2023
6f87583
combined code for ReadStringArray
jgonz120 Dec 7, 2023
0f75860
Updated buffer size to match STJ's default buffer size
jgonz120 Dec 7, 2023
5c4269a
Switch Utf8JsonStreamReader to be disposable.
jgonz120 Dec 7, 2023
c469899
Switch to read the value for numbers into a string directly
jgonz120 Dec 7, 2023
d0f9f5e
revert back to using private var for utf8Bom
jgonz120 Dec 7, 2023
d3e6ab8
Remove ReadStringArrayAsList
jgonz120 Dec 7, 2023
2d7cba8
Avoid referencing buffer after returning
jgonz120 Dec 7, 2023
4192d9b
Actually avoid referencing _buffer after returning
jgonz120 Dec 7, 2023
f67239b
Update how buffers are fed into Utf8JsonReader to avoid feeding extra…
jgonz120 Dec 8, 2023
ca6e1d7
remove extra line
jgonz120 Dec 8, 2023
fa9639d
Reverted back to using try get int for ReadTokenAsString
jgonz120 Dec 11, 2023
997f199
Update src/NuGet.Core/NuGet.ProjectModel/Utf8JsonStreamReader.cs
jgonz120 Dec 11, 2023
3e4146c
Remove ValueTextEquals taking in string
jgonz120 Dec 11, 2023
b403ed8
Switched to Skip instead of TrySkip
jgonz120 Dec 11, 2023
a1c4844
Update src/NuGet.Core/NuGet.ProjectModel/Utf8JsonStreamReader.cs
jgonz120 Dec 11, 2023
4ff0f7e
Added some unit tests
jgonz120 Dec 11, 2023
a9884ec
merge
jgonz120 Dec 11, 2023
7a467d5
fix Bom
jgonz120 Dec 11, 2023
86d3524
Switched to using Moq
jgonz120 Dec 11, 2023
c559e69
Update src/NuGet.Core/NuGet.ProjectModel/Utf8JsonStreamReader.cs
jgonz120 Dec 11, 2023
74b2e54
loop through stream when reading to ensure reading full bytes or to t…
jgonz120 Dec 11, 2023
0c05eb8
update signature comment
jgonz120 Dec 12, 2023
a233b40
Switched stream back to field and supress warning
jgonz120 Dec 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
232 changes: 232 additions & 0 deletions src/NuGet.Core/NuGet.ProjectModel/Utf8JsonReaderExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,232 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Text.Json;

namespace NuGet.ProjectModel
{
internal static class Utf8JsonReaderExtensions
{
//private static readonly char[] DelimitedStringDelimiters = new char[] { ' ', ',' };
jgonz120 marked this conversation as resolved.
Show resolved Hide resolved
//internal static readonly Utf8JsonStreamLockFileConverter LockFileConverter = new Utf8JsonStreamLockFileConverter();
//internal static readonly Utf8JsonStreamLockFileItemConverter<LockFileItem> LockFileItemConverter = new Utf8JsonStreamLockFileItemConverter<LockFileItem>();
//internal static readonly Utf8JsonStreamLockFileItemConverter<LockFileContentFile> LockFileContentFileConverter = new Utf8JsonStreamLockFileItemConverter<LockFileContentFile>();
//internal static readonly Utf8JsonStreamLockFileItemConverter<LockFileRuntimeTarget> LockFileRuntimeTargetConverter = new Utf8JsonStreamLockFileItemConverter<LockFileRuntimeTarget>();
//internal static readonly Utf8JsonStreamLockFileTargetLibraryConverter LockFileTargetLibraryConverter = new Utf8JsonStreamLockFileTargetLibraryConverter();
//internal static readonly Utf8JsonStreamLockFileLibraryConverter LockFileLibraryConverter = new Utf8JsonStreamLockFileLibraryConverter();
//internal static readonly Utf8JsonStreamLockFileTargetConverter LockFileTargetConverter = new Utf8JsonStreamLockFileTargetConverter();
//internal static readonly Utf8JsonStreamProjectFileDependencyGroupConverter ProjectFileDepencencyGroupConverter = new Utf8JsonStreamProjectFileDependencyGroupConverter();
//internal static readonly Utf8JsonStreamAssetsLogMessageConverter AssetsLogMessageConverter = new Utf8JsonStreamAssetsLogMessageConverter();

//internal static IReadOnlyList<string> ReadDelimitedString(this ref Utf8JsonReader reader)
//{
// if (ReadNextToken(ref reader))
// {
// switch (reader.TokenType)
// {
// case JsonTokenType.String:
// var value = reader.GetString();

// return value.Split(DelimitedStringDelimiters, StringSplitOptions.RemoveEmptyEntries);

// default:
// var invalidCastException = new InvalidCastException();
// throw new JsonException(invalidCastException.Message, invalidCastException);
// }
// }

// return null;
//}

//internal static bool ReadNextToken(this ref Utf8JsonReader reader)
//{
// bool wasRead;

// while ((wasRead = reader.Read()) && reader.TokenType == JsonTokenType.Comment)
// {
// }

// return wasRead;
//}

//internal static bool ReadNextTokenAsBoolOrFalse(this ref Utf8JsonReader reader)
//{
// if (reader.ReadNextToken() && (reader.TokenType == JsonTokenType.False || reader.TokenType == JsonTokenType.True))
// {
// return reader.GetBoolean();
// }
// return false;
//}

//internal static string ReadNextTokenAsString(this ref Utf8JsonReader reader)
//{
// if (ReadNextToken(ref reader))
// {
// return ReadTokenAsString(ref reader);
// }

// return null;
//}

//internal static IList<T> ReadObjectAsList<T>(this ref Utf8JsonReader reader, JsonSerializerOptions options)
//{
// if (reader.TokenType != JsonTokenType.StartObject)
// {
// return new List<T>(0);

// }
// //We use JsonObjects for the arrays so we advance to the first property in the object which is the name/ver of the first library
// reader.ReadNextToken();

// if (reader.TokenType == JsonTokenType.EndObject)
// {
// return new List<T>(0);
// }

// var objectConverter = (JsonConverter<T>)options.GetConverter(typeof(T));
// var listObjects = new List<T>();
// do
// {
// listObjects.Add(objectConverter.Read(ref reader, typeof(T), options));
// //At this point we're looking at the EndObject token for the object, need to advance.
// reader.ReadNextToken();
// }
// while (reader.TokenType != JsonTokenType.EndObject);
// return listObjects;
//}

//internal static List<string> ReadNextStringArrayAsList(this ref Utf8JsonReader reader, List<string> strings = null)
//{
// ReadNextToken(ref reader);
// return ReadStringArrayAsList(ref reader, strings);
//}

//internal static List<string> ReadStringArrayAsList(this ref Utf8JsonReader reader, List<string> strings = null)
//{
// if (reader.TokenType == JsonTokenType.StartArray)
// {
// while (ReadNextToken(ref reader) && reader.TokenType != JsonTokenType.EndArray)
// {
// string value = ReadTokenAsString(ref reader);

// strings = strings ?? new List<string>();

// strings.Add(value);
// }
// }

// return strings;
//}


//internal static IList<string> ReadStringArrayAsIList(this ref Utf8JsonReader reader, IList<string> strings = null)
jgonz120 marked this conversation as resolved.
Show resolved Hide resolved
//{
// if (reader.TokenType == JsonTokenType.StartArray)
// {
// while (ReadNextToken(ref reader) && reader.TokenType != JsonTokenType.EndArray)
// {
// string value = ReadTokenAsString(ref reader);

// strings = strings ?? new List<string>();

// strings.Add(value);
// }
// }

// return strings;
//}

//internal static void ReadArrayOfObjects<T1, T2>(this ref Utf8JsonReader reader, JsonSerializerOptions options, IList<T2> objectList) where T1 : T2
//{
// if (objectList is null)
// {
// return;
// }

// var type = typeof(T1);
// var objectConverter = (JsonConverter<T1>)options.GetConverter(type);

// if (ReadNextToken(ref reader) && reader.TokenType == JsonTokenType.StartArray)
// {
// while (ReadNextToken(ref reader) && reader.TokenType != JsonTokenType.EndArray)
// {
// var convertedObject = objectConverter.Read(ref reader, type, options);
// if (convertedObject != null)
// {
// objectList.Add(convertedObject);
// }
// }
// }

//}

//internal static IReadOnlyList<string> ReadNextStringOrArrayOfStringsAsReadOnlyList(this ref Utf8JsonReader reader)
//{
// if (ReadNextToken(ref reader))
// {
// switch (reader.TokenType)
// {
// case JsonTokenType.String:
// return new[] { (string)reader.GetString() };

// case JsonTokenType.StartArray:
// return ReadStringArrayAsReadOnlyListFromArrayStart(ref reader);

// case JsonTokenType.StartObject:
// return null;
// }
// }

// return null;
//}

//internal static IReadOnlyList<string> ReadStringArrayAsReadOnlyListFromArrayStart(this ref Utf8JsonReader reader)
//{
// List<string> strings = null;

// while (ReadNextToken(ref reader) && reader.TokenType != JsonTokenType.EndArray)
// {
// string value = ReadTokenAsString(ref reader);

// strings = strings ?? new List<string>();

// strings.Add(value);
// }

// return (IReadOnlyList<string>)strings ?? Array.Empty<string>();
//}

internal static string ReadTokenAsString(this ref Utf8JsonReader reader)
{
switch (reader.TokenType)
{
case JsonTokenType.True:
return bool.TrueString;
case JsonTokenType.False:
return bool.FalseString;
case JsonTokenType.Number:
if (reader.TryGetInt16(out short shortValue))
{
return shortValue.ToString();
}
if (reader.TryGetInt32(out int intValue))
{
return intValue.ToString();
}
else if (reader.TryGetInt64(out long longValue))
{
return longValue.ToString();
}
jgonz120 marked this conversation as resolved.
Show resolved Hide resolved
return reader.GetDouble().ToString();
case JsonTokenType.String:
return reader.GetString();
case JsonTokenType.None:
case JsonTokenType.Null:
return null;
default:
throw new InvalidCastException();
}
}
}
}
Loading