From 5ff024451cd60c0ba374f31db315bf1f27abb7bc Mon Sep 17 00:00:00 2001 From: Maurycy Markowski Date: Mon, 10 Jul 2023 17:44:20 -0700 Subject: [PATCH] Fix to #30604 - Implement JSON serialization/deserialization via Utf8JsonReader/Utf8JsonWriter (#31160) Using Utf8JsonReader to read JSON data rather than caching it using DOM. This should reduce allocations significantly. Tricky part is that entity materializers are build in a way that assumes we have random access to all the data we need. This is not the case here. We read JSON data sequentially and can only do it once, and we don't know the order in which we get the data. This is somewhat problematic in case where entity takes argument in the constructor. Those could be at the very end of the JSON string, so we must read all the data before we can instantiate the object, and populate it's properties and do navigation fixup. This requires us reading all the JSON data, store them in local variables, and only when we are done reading we instantiate the entity and populate all the properties with data stored in those variables. This adds some allocations (specifically navigations). We also have to disable de-duplication logic - we can't always safely re-read the JSON string, and definitely can't start reading it from arbitrary position, so now we have to add JSON string for every aggregate projected, even if we already project it's parent. Serialization implementation (i.e. Utf8JsonWriter) is pretty straighforward. Also fix to #30993 - Query/Json: data corruption for tracking queries with nested json entities, then updating nested entities outside EF and re-querying Fix is to recognize and modify shaper in case of tracking query, so that nav expansions are not skipped when parent entity is found in Change Tracker. This is necessary to fix alongside streaming, because now we throw exception from reader (unexpected token) if we don't process the entire stream correctly. Before it would be silently ignored apart from the edge case described in the bug. Fixes #30604 Fixes #30993 --- .../Properties/RelationalStrings.Designer.cs | 8 + .../Properties/RelationalStrings.resx | 3 + .../Query/Internal/JsonProjectionInfo.cs | 16 +- ...ionalProjectionBindingExpressionVisitor.cs | 10 +- ...ocessingExpressionVisitor.ClientMethods.cs | 212 ++- ...sitor.ShaperProcessingExpressionVisitor.cs | 1277 ++++++++++++----- .../Query/SqlExpressions/SelectExpression.cs | 113 +- .../Update/ModificationCommand.cs | 96 +- .../Internal/SqlServerJsonTypeMapping.cs | 22 +- .../Storage/Internal/SqliteJsonTypeMapping.cs | 22 +- src/EFCore/Query/ExpressionPrinter.cs | 57 +- .../Internal/EntityMaterializerSource.cs | 1 + .../CSharpRuntimeModelCodeGeneratorTest.cs | 2 +- .../Query/JsonQueryAdHocTestBase.cs | 298 +++- .../Query/JsonQueryFixtureBase.cs | 26 + .../Query/JsonQueryTestBase.cs | 75 + .../Update/JsonUpdateTestBase.cs | 107 ++ .../Query/JsonQueryAdHocSqlServerTest.cs | 23 + .../Query/JsonQuerySqlServerTest.cs | 118 +- .../Update/JsonUpdateSqlServerTest.cs | 59 +- .../Query/JsonQueryAdHocSqliteTest.cs | 23 + .../Update/JsonUpdateSqliteTest.cs | 52 +- 22 files changed, 1966 insertions(+), 654 deletions(-) diff --git a/src/EFCore.Relational/Properties/RelationalStrings.Designer.cs b/src/EFCore.Relational/Properties/RelationalStrings.Designer.cs index 977720170fc..258248c6404 100644 --- a/src/EFCore.Relational/Properties/RelationalStrings.Designer.cs +++ b/src/EFCore.Relational/Properties/RelationalStrings.Designer.cs @@ -1139,6 +1139,14 @@ public static string JsonNodeMustBeHandledByProviderSpecificVisitor public static string JsonPropertyNameShouldBeConfiguredOnNestedNavigation => GetString("JsonPropertyNameShouldBeConfiguredOnNestedNavigation"); + /// + /// Invalid token type: '{tokenType}'. + /// + public static string JsonReaderInvalidTokenType(object? tokenType) + => string.Format( + GetString("JsonReaderInvalidTokenType", nameof(tokenType)), + tokenType); + /// /// Entity {entity} is required but the JSON element containing it is null. /// diff --git a/src/EFCore.Relational/Properties/RelationalStrings.resx b/src/EFCore.Relational/Properties/RelationalStrings.resx index 00203a4f7eb..36222b76063 100644 --- a/src/EFCore.Relational/Properties/RelationalStrings.resx +++ b/src/EFCore.Relational/Properties/RelationalStrings.resx @@ -547,6 +547,9 @@ The JSON property name should only be configured on nested owned navigations. + + Invalid token type: '{tokenType}'. + Entity {entity} is required but the JSON element containing it is null. diff --git a/src/EFCore.Relational/Query/Internal/JsonProjectionInfo.cs b/src/EFCore.Relational/Query/Internal/JsonProjectionInfo.cs index ca5f0555fac..96cdc3da0ff 100644 --- a/src/EFCore.Relational/Query/Internal/JsonProjectionInfo.cs +++ b/src/EFCore.Relational/Query/Internal/JsonProjectionInfo.cs @@ -19,12 +19,10 @@ public readonly struct JsonProjectionInfo /// public JsonProjectionInfo( int jsonColumnIndex, - List<(IProperty?, int?, int?)> keyAccessInfo, - (string?, int?, int?)[] additionalPath) + List<(IProperty?, int?, int?)> keyAccessInfo) { JsonColumnIndex = jsonColumnIndex; KeyAccessInfo = keyAccessInfo; - AdditionalPath = additionalPath; } /// @@ -55,16 +53,4 @@ public JsonProjectionInfo( /// doing so can result in application failures when updating to a new Entity Framework Core release. /// public List<(IProperty? KeyProperty, int? ConstantKeyValue, int? KeyProjectionIndex)> KeyAccessInfo { get; } - - /// - /// List of additional path elements, only one of the values in the tuple is non-null - /// this information is used to access the correct sub-element of a JsonElement that we materialized - /// - /// - /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to - /// the same compatibility standards as public APIs. It may be changed or removed without notice in - /// any release. You should only use it directly in your code with extreme caution and knowing that - /// doing so can result in application failures when updating to a new Entity Framework Core release. - /// - public (string? JsonPropertyName, int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath { get; } } diff --git a/src/EFCore.Relational/Query/Internal/RelationalProjectionBindingExpressionVisitor.cs b/src/EFCore.Relational/Query/Internal/RelationalProjectionBindingExpressionVisitor.cs index c310bf18f89..98bc2ab6a4f 100644 --- a/src/EFCore.Relational/Query/Internal/RelationalProjectionBindingExpressionVisitor.cs +++ b/src/EFCore.Relational/Query/Internal/RelationalProjectionBindingExpressionVisitor.cs @@ -25,7 +25,6 @@ private static readonly MethodInfo GetParameterValueMethodInfo private bool _indexBasedBinding; private Dictionary? _entityProjectionCache; - private Dictionary? _jsonQueryCache; private List? _clientProjections; private readonly Dictionary _projectionMapping = new(); @@ -66,7 +65,6 @@ public virtual Expression Translate(SelectExpression selectExpression, Expressio { _indexBasedBinding = true; _entityProjectionCache = new Dictionary(); - _jsonQueryCache = new Dictionary(); _projectionMapping.Clear(); _clientProjections = new List(); @@ -307,11 +305,8 @@ protected override Expression VisitExtension(Expression extensionExpression) { if (_indexBasedBinding) { - if (!_jsonQueryCache!.TryGetValue(jsonQueryExpression, out var jsonProjectionBinding)) - { - jsonProjectionBinding = AddClientProjection(jsonQueryExpression, typeof(ValueBuffer)); - _jsonQueryCache[jsonQueryExpression] = jsonProjectionBinding; - } + _clientProjections!.Add(jsonQueryExpression); + var jsonProjectionBinding = new ProjectionBindingExpression(_selectExpression, _clientProjections.Count - 1, typeof(ValueBuffer)); return entityShaperExpression.Update(jsonProjectionBinding); } @@ -654,7 +649,6 @@ private ProjectionBindingExpression AddClientProjection(Expression expression, T return new ProjectionBindingExpression(_selectExpression, existingIndex, type); } -#pragma warning disable IDE0052 // Remove unread private members private static T GetParameterValue(QueryContext queryContext, string parameterName) #pragma warning restore IDE0052 // Remove unread private members => (T)queryContext.ParameterValues[parameterName]!; diff --git a/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.ClientMethods.cs b/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.ClientMethods.cs index ae0d31e4760..9205bbab608 100644 --- a/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.ClientMethods.cs +++ b/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.ClientMethods.cs @@ -5,6 +5,7 @@ using System.Text.Json; using Microsoft.EntityFrameworkCore.Internal; using Microsoft.EntityFrameworkCore.Query.Internal; +using Microsoft.EntityFrameworkCore.Storage.Json; namespace Microsoft.EntityFrameworkCore.Query; @@ -67,8 +68,8 @@ private static readonly MethodInfo MaterializeJsonEntityMethodInfo private static readonly MethodInfo MaterializeJsonEntityCollectionMethodInfo = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(MaterializeJsonEntityCollection))!; - private static readonly MethodInfo ExtractJsonPropertyMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(ExtractJsonProperty))!; + private static readonly MethodInfo InverseCollectionFixupMethod + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(InverseCollectionFixup))!; [MethodImpl(MethodImplOptions.AggressiveInlining)] private static TValue ThrowReadValueException( @@ -123,13 +124,6 @@ private static TValue ThrowExtractJsonPropertyException( exception); } - private static T? ExtractJsonProperty(JsonElement element, string propertyName, bool nullable) - => nullable - ? element.TryGetProperty(propertyName, out var jsonValue) - ? jsonValue.Deserialize() - : default - : element.GetProperty(propertyName).Deserialize(); - private static void IncludeReference( QueryContext queryContext, TEntity entity, @@ -869,105 +863,189 @@ static async Task InitializeReaderAsync( dataReaderContext.HasNext = false; } - private static void IncludeJsonEntityReference( + private static TEntity? MaterializeJsonEntity( QueryContext queryContext, - JsonElement? jsonElement, object[] keyPropertyValues, - TIncludingEntity entity, - Func innerShaper, - Action fixup) - where TIncludingEntity : class - where TIncludedEntity : class + JsonReaderData? jsonReaderData, + bool nullable, + Func shaper) + where TEntity : class { - if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) + if (jsonReaderData == null) { - var included = innerShaper(queryContext, keyPropertyValues, jsonElement.Value); - fixup(entity, included); + return nullable + ? null + : throw new InvalidOperationException( + RelationalStrings.JsonRequiredEntityWithNullJson(typeof(TEntity).Name)); + } + + var manager = new Utf8JsonReaderManager(jsonReaderData); + var tokenType = manager.CurrentReader.TokenType; + + if (tokenType == JsonTokenType.Null) + { + return nullable + ? null + : throw new InvalidOperationException( + RelationalStrings.JsonRequiredEntityWithNullJson(typeof(TEntity).Name)); + } + + if (tokenType != JsonTokenType.StartObject) + { + throw new InvalidOperationException( + RelationalStrings.JsonReaderInvalidTokenType(tokenType.ToString())); } + + manager.CaptureState(); + var result = shaper(queryContext, keyPropertyValues, jsonReaderData); + + return result; } - private static void IncludeJsonEntityCollection( + private static TResult? MaterializeJsonEntityCollection( QueryContext queryContext, - JsonElement? jsonElement, object[] keyPropertyValues, - TIncludingEntity entity, - Func innerShaper, - Action fixup) - where TIncludingEntity : class - where TIncludedCollectionElement : class + JsonReaderData? jsonReaderData, + INavigationBase navigation, + Func innerShaper) + where TEntity : class + where TResult : ICollection { - if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) + if (jsonReaderData == null) { - var newKeyPropertyValues = new object[keyPropertyValues.Length + 1]; - Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length); + return default; + } + + var manager = new Utf8JsonReaderManager(jsonReaderData); + var tokenType = manager.CurrentReader.TokenType; + + if (tokenType == JsonTokenType.Null) + { + return default; + } - var i = 0; - foreach (var jsonArrayElement in jsonElement.Value.EnumerateArray()) + if (tokenType != JsonTokenType.StartArray) + { + throw new InvalidOperationException( + RelationalStrings.JsonReaderInvalidTokenType(tokenType.ToString())); + } + + var collectionAccessor = navigation.GetCollectionAccessor(); + var result = (TResult)collectionAccessor!.Create(); + + var newKeyPropertyValues = new object[keyPropertyValues.Length + 1]; + Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length); + + tokenType = manager.MoveNext(); + + var i = 0; + while (tokenType != JsonTokenType.EndArray) + { + newKeyPropertyValues[^1] = ++i; + + if (tokenType == JsonTokenType.StartObject) { - newKeyPropertyValues[^1] = ++i; + manager.CaptureState(); + var entity = innerShaper(queryContext, newKeyPropertyValues, jsonReaderData); + result.Add(entity); + manager = new Utf8JsonReaderManager(manager.Data); - var resultElement = innerShaper(queryContext, newKeyPropertyValues, jsonArrayElement); + if (manager.CurrentReader.TokenType != JsonTokenType.EndObject) + { + throw new InvalidOperationException( + RelationalStrings.JsonReaderInvalidTokenType(tokenType.ToString())); + } - fixup(entity, resultElement); + tokenType = manager.MoveNext(); } } + + manager.CaptureState(); + + return result; } - private static TEntity? MaterializeJsonEntity( + private static void IncludeJsonEntityReference( QueryContext queryContext, - JsonElement? jsonElement, object[] keyPropertyValues, - bool nullable, - Func shaper) - where TEntity : class + JsonReaderData? jsonReaderData, + TIncludingEntity entity, + Func innerShaper, + Action fixup, + bool trackingQuery) + where TIncludingEntity : class + where TIncludedEntity : class { - if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) + if (jsonReaderData == null) { - var result = shaper(queryContext, keyPropertyValues, jsonElement.Value); - - return result; + return; } - if (nullable) + var included = innerShaper(queryContext, keyPropertyValues, jsonReaderData); + + if (!trackingQuery) { - return default; + fixup(entity, included); } - - throw new InvalidOperationException( - RelationalStrings.JsonRequiredEntityWithNullJson(typeof(TEntity).Name)); } - private static TResult? MaterializeJsonEntityCollection( + private static void IncludeJsonEntityCollection( QueryContext queryContext, - JsonElement? jsonElement, object[] keyPropertyValues, - INavigationBase navigation, - Func innerShaper) - where TEntity : class - where TResult : ICollection + JsonReaderData? jsonReaderData, + TIncludingEntity entity, + Func innerShaper, + Action fixup, + bool trackingQuery) + where TIncludingEntity : class + where TIncludedCollectionElement : class { - if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) + if (jsonReaderData == null) { - var collectionAccessor = navigation.GetCollectionAccessor(); - var result = (TResult)collectionAccessor!.Create(); + return; + } + + var manager = new Utf8JsonReaderManager(jsonReaderData); + var tokenType = manager.CurrentReader.TokenType; + + if (tokenType != JsonTokenType.StartArray) + { + throw new InvalidOperationException( + RelationalStrings.JsonReaderInvalidTokenType(tokenType.ToString())); + } - var newKeyPropertyValues = new object[keyPropertyValues.Length + 1]; - Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length); + var newKeyPropertyValues = new object[keyPropertyValues.Length + 1]; + Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length); - var i = 0; - foreach (var jsonArrayElement in jsonElement.Value.EnumerateArray()) + tokenType = manager.MoveNext(); + + var i = 0; + while (tokenType != JsonTokenType.EndArray) + { + newKeyPropertyValues[^1] = ++i; + + if (tokenType == JsonTokenType.StartObject) { - newKeyPropertyValues[^1] = ++i; + manager.CaptureState(); + var resultElement = innerShaper(queryContext, newKeyPropertyValues, jsonReaderData); + + if (!trackingQuery) + { + fixup(entity, resultElement); + } - var resultElement = innerShaper(queryContext, newKeyPropertyValues, jsonArrayElement); + manager = new Utf8JsonReaderManager(manager.Data); + if (manager.CurrentReader.TokenType != JsonTokenType.EndObject) + { + throw new InvalidOperationException( + RelationalStrings.JsonReaderInvalidTokenType(tokenType.ToString())); + } - result.Add(resultElement); + tokenType = manager.MoveNext(); } - - return result; } - return default; + manager.CaptureState(); } private static async Task TaskAwaiter(Func[] taskFactories) diff --git a/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.cs b/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.cs index 3f1855a1f3d..c8316bcb6c1 100644 --- a/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.cs +++ b/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.cs @@ -2,10 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics.CodeAnalysis; +using System.Text; using System.Text.Json; +using Microsoft.EntityFrameworkCore.ChangeTracking.Internal; using Microsoft.EntityFrameworkCore.Metadata.Internal; using Microsoft.EntityFrameworkCore.Query.Internal; using Microsoft.EntityFrameworkCore.Query.SqlExpressions; +using Microsoft.EntityFrameworkCore.Storage.Json; namespace Microsoft.EntityFrameworkCore.Query; @@ -34,20 +37,41 @@ private static readonly MemberInfo SingleQueryResultCoordinatorResultReadyMember private static readonly MethodInfo CollectionAccessorAddMethodInfo = typeof(IClrCollectionAccessor).GetTypeInfo().GetDeclaredMethod(nameof(IClrCollectionAccessor.Add))!; - private static readonly MethodInfo JsonElementTryGetPropertyMethod - = typeof(JsonElement).GetMethod(nameof(JsonElement.TryGetProperty), new[] { typeof(string), typeof(JsonElement).MakeByRefType() })!; - - private static readonly MethodInfo JsonElementGetItemMethodInfo - = typeof(JsonElement).GetMethod("get_Item", new[] { typeof(int) })!; - private static readonly PropertyInfo ObjectArrayIndexerPropertyInfo = typeof(object[]).GetProperty("Item")!; - private static readonly PropertyInfo NullableJsonElementHasValuePropertyInfo - = typeof(JsonElement?).GetProperty(nameof(Nullable.HasValue))!; + private static readonly PropertyInfo UTF8Property + = typeof(Encoding).GetProperty(nameof(Encoding.UTF8))!; + + private static readonly MethodInfo EncodingGetBytesMethod + = typeof(Encoding).GetMethod(nameof(Encoding.GetBytes), new[] { typeof(string) })!; + + private static readonly ConstructorInfo MemoryStreamConstructor + = typeof(MemoryStream).GetConstructor(new[] { typeof(byte[]) })!; + + private static readonly ConstructorInfo JsonReaderDataConstructor + = typeof(JsonReaderData).GetConstructor(new Type[] { typeof(Stream) })!; + + private static readonly ConstructorInfo JsonReaderManagerConstructor + = typeof(Utf8JsonReaderManager).GetConstructor(new Type[] { typeof(JsonReaderData) })!; + + private static readonly MethodInfo Utf8JsonReaderManagerMoveNextMethod + = typeof(Utf8JsonReaderManager).GetMethod(nameof(Utf8JsonReaderManager.MoveNext), new Type[] { })!; + + private static readonly MethodInfo Utf8JsonReaderManagerCaptureStateMethod + = typeof(Utf8JsonReaderManager).GetMethod(nameof(Utf8JsonReaderManager.CaptureState), new Type[] { })!; - private static readonly PropertyInfo NullableJsonElementValuePropertyInfo - = typeof(JsonElement?).GetProperty(nameof(Nullable.Value))!; + private static readonly FieldInfo Utf8JsonReaderManagerCurrentReaderField + = typeof(Utf8JsonReaderManager).GetField(nameof(Utf8JsonReaderManager.CurrentReader))!; + + private static readonly MethodInfo Utf8JsonReaderValueTextEqualsMethod + = typeof(Utf8JsonReader).GetMethod(nameof(Utf8JsonReader.ValueTextEquals), new Type[] { typeof(ReadOnlySpan) })!; + + private static readonly MethodInfo Utf8JsonReaderTrySkipMethod + = typeof(Utf8JsonReader).GetMethod(nameof(Utf8JsonReader.TrySkip), new Type[] { })!; + + private static readonly PropertyInfo Utf8JsonReaderTokenTypeProperty + = typeof(Utf8JsonReader).GetProperty(nameof(Utf8JsonReader.TokenType))!; private static readonly MethodInfo ArrayCopyMethodInfo = typeof(Array).GetMethod(nameof(Array.Copy), new[] { typeof(Array), typeof(Array), typeof(int) })!; @@ -116,22 +140,13 @@ private static readonly MethodInfo ArrayCopyMethodInfo private readonly Dictionary _singleEntityTypeDiscriminatorValues = new(); private readonly Dictionary - _jsonValueBufferParameterMapping = new(); + _jsonValueBufferToJsonReaderDataAndKeyValuesParameterMapping = new(); private readonly Dictionary - _jsonMaterializationContextParameterMapping = new(); - - /// - /// Cache for the JsonElement values we have generated - storing variables that the JsonElements are assigned to - /// - private readonly Dictionary<(int JsonColumnIndex, (string? JsonPropertyName, int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath), ParameterExpression> _existingJsonElementMap - = new(new ExistingJsonElementMapKeyComparer()); + _jsonMaterializationContextToJsonReaderDataAndKeyValuesParameterMapping = new(); - /// - /// Cache for the key values we have generated - storing variables that the keys are assigned to - /// - private readonly Dictionary<(int JsonColumnIndex, (int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath), ParameterExpression> _existingKeyValuesMap - = new(new ExistingJsonKeyValuesMapKeyComparer()); + private readonly Dictionary + _jsonReaderDataToJsonReaderManagerParameterMapping = new(); /// /// Map between index of the non-constant json array element access @@ -428,10 +443,10 @@ protected override Expression VisitBinary(BinaryExpression binaryExpression) } if (newExpression.Arguments[0] is ParameterExpression valueBufferParameter - && _jsonValueBufferParameterMapping.ContainsKey(valueBufferParameter)) + && _jsonValueBufferToJsonReaderDataAndKeyValuesParameterMapping.ContainsKey(valueBufferParameter)) { - _jsonMaterializationContextParameterMapping[parameterExpression] = - _jsonValueBufferParameterMapping[valueBufferParameter]; + _jsonMaterializationContextToJsonReaderDataAndKeyValuesParameterMapping[parameterExpression] = + _jsonValueBufferToJsonReaderDataAndKeyValuesParameterMapping[valueBufferParameter]; var updatedExpression = newExpression.Update( new[] { Expression.Constant(ValueBuffer.Empty), newExpression.Arguments[1] }); @@ -449,6 +464,16 @@ protected override Expression VisitBinary(BinaryExpression binaryExpression) return memberExpression.Assign(Visit(binaryExpression.Right)); } + // we only have mapping between MaterializationContext and JsonReaderData, but we use JsonReaderManager to extract JSON values + // so we need to add mapping between JsonReaderData and JsonReaderManager parameter, so we know which parameter to use + // when generating actual Get* method + if (binaryExpression is { NodeType: ExpressionType.Assign, Left: ParameterExpression jsonReaderManagerParameter } + && jsonReaderManagerParameter.Type == typeof(Utf8JsonReaderManager)) + { + var jsonReaderDataParameter = (ParameterExpression)((NewExpression)binaryExpression.Right).Arguments[0]; + _jsonReaderDataToJsonReaderManagerParameterMapping[jsonReaderDataParameter] = jsonReaderManagerParameter; + } + return base.VisitBinary(binaryExpression); } @@ -459,20 +484,24 @@ protected override Expression VisitExtension(Expression extensionExpression) case RelationalEntityShaperExpression entityShaperExpression when !_inline && entityShaperExpression.ValueBufferExpression is ProjectionBindingExpression projectionBindingExpression: { - if (!_variableShaperMapping.TryGetValue(entityShaperExpression.ValueBufferExpression, out var accessor)) + // we can't cache ProjectionBindingExpression results for non-tracking queries + // JSON entities must be read and re-shaped every time (streaming) + // as part of the process we do fixup to the parents, so those JSON entities would be potentially fixed up multiple times + // it's ok for references (overwrite) but for collections they would be added multiple times if we were to cache the parent + // by creating every entity every time we guarantee this doesn't happen + if (!_isTracking || !_variableShaperMapping.TryGetValue(entityShaperExpression.ValueBufferExpression, out var accessor)) { if (GetProjectionIndex(projectionBindingExpression) is JsonProjectionInfo jsonProjectionInfo) { // json entity at the root - var (jsonElementParameter, keyValuesParameter) = JsonShapingPreProcess( + var (jsonReaderDataVariable, keyValuesParameter) = JsonShapingPreProcess( jsonProjectionInfo, entityShaperExpression.EntityType); var shaperResult = CreateJsonShapers( entityShaperExpression.EntityType, entityShaperExpression.IsNullable, - collection: false, - jsonElementParameter, + jsonReaderDataVariable, keyValuesParameter, parentEntityExpression: null, navigation: null); @@ -482,7 +511,6 @@ protected override Expression VisitExtension(Expression extensionExpression) _variables.Add(visitedShaperResultParameter); _jsonEntityExpressions.Add(Expression.Assign(visitedShaperResultParameter, visitedShaperResult)); - accessor = CompensateForCollectionMaterialization( visitedShaperResultParameter, entityShaperExpression.Type); @@ -515,7 +543,10 @@ protected override Expression VisitExtension(Expression extensionExpression) entityShaperExpression.Type); } - _variableShaperMapping[entityShaperExpression.ValueBufferExpression] = accessor; + if (_isTracking) + { + _variableShaperMapping[entityShaperExpression.ValueBufferExpression] = accessor; + } } return accessor; @@ -549,18 +580,17 @@ when GetProjectionIndex(collectionResultExpression.ProjectionBindingExpression) is JsonProjectionInfo jsonProjectionInfo: { // json entity collection at the root - var (jsonElementParameter, keyValuesParameter) = JsonShapingPreProcess( + var (jsonReaderDataVariable, keyValuesParameter) = JsonShapingPreProcess( jsonProjectionInfo, navigation.TargetEntityType); var shaperResult = CreateJsonShapers( navigation.TargetEntityType, nullable: true, - collection: true, - jsonElementParameter, + jsonReaderDataVariable, keyValuesParameter, parentEntityExpression: null, - navigation); + navigation: navigation); var visitedShaperResult = Visit(shaperResult); @@ -825,15 +855,14 @@ when GetProjectionIndex(collectionResultExpression.ProjectionBindingExpression) if (projectionBindingExpression != null && GetProjectionIndex(projectionBindingExpression) is JsonProjectionInfo jsonProjectionInfo) { - var (jsonElementParameter, keyValuesParameter) = JsonShapingPreProcess( + var (jsonReaderDataVariable, keyValuesParameter) = JsonShapingPreProcess( jsonProjectionInfo, includeExpression.Navigation.TargetEntityType); var shaperResult = CreateJsonShapers( includeExpression.Navigation.TargetEntityType, nullable: true, - collection: includeExpression.NavigationExpression is CollectionResultExpression, - jsonElementParameter, + jsonReaderDataVariable, keyValuesParameter, parentEntityExpression: entity, navigation: (INavigation)includeExpression.Navigation); @@ -1082,16 +1111,27 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp var property = methodCallExpression.Arguments[2].GetConstantValue(); var mappingParameter = (ParameterExpression)((MethodCallExpression)methodCallExpression.Arguments[0]).Object!; - if (_jsonMaterializationContextParameterMapping.ContainsKey(mappingParameter)) + if (_jsonMaterializationContextToJsonReaderDataAndKeyValuesParameterMapping.ContainsKey(mappingParameter)) { - var (jsonElementParameter, keyPropertyValuesParameter) = _jsonMaterializationContextParameterMapping[mappingParameter]; + var (jsonReaderDataParameter, keyPropertyValuesParameter) = _jsonMaterializationContextToJsonReaderDataAndKeyValuesParameterMapping[mappingParameter]; - return property!.IsPrimaryKey() - ? Expression.MakeIndex( + if (property!.IsPrimaryKey()) + { + return Expression.MakeIndex( keyPropertyValuesParameter, ObjectArrayIndexerPropertyInfo, - new[] { Expression.Constant(index) }) - : CreateExtractJsonPropertyExpression(jsonElementParameter, property); + new[] { Expression.Constant(index) }); + } + else + { + var jsonReaderManagerParameter = _jsonReaderDataToJsonReaderManagerParameterMapping[jsonReaderDataParameter]; + + var jsonReadPropertyValueExpression = CreateReadJsonPropertyValueExpression(jsonReaderManagerParameter, property); + + return methodCallExpression.Type != jsonReadPropertyValueExpression.Type + ? Expression.Convert(jsonReadPropertyValueExpression, methodCallExpression.Type) + : jsonReadPropertyValueExpression; + } } int projectionIndex; @@ -1135,20 +1175,19 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp private Expression CreateJsonShapers( IEntityType entityType, bool nullable, - bool collection, - ParameterExpression jsonElementParameter, + ParameterExpression jsonReaderDataParameter, ParameterExpression keyValuesParameter, Expression? parentEntityExpression, INavigation? navigation) { - var jsonElementShaperLambdaParameter = Expression.Parameter(typeof(JsonElement)); + var jsonReaderDataShaperLambdaParameter = Expression.Parameter(typeof(JsonReaderData)); var keyValuesShaperLambdaParameter = Expression.Parameter(typeof(object[])); var shaperBlockVariables = new List(); var shaperBlockExpressions = new List(); var valueBufferParameter = Expression.Parameter(typeof(ValueBuffer)); - _jsonValueBufferParameterMapping[valueBufferParameter] = (jsonElementShaperLambdaParameter, keyValuesShaperLambdaParameter); + _jsonValueBufferToJsonReaderDataAndKeyValuesParameterMapping[valueBufferParameter] = (jsonReaderDataShaperLambdaParameter, keyValuesShaperLambdaParameter); var entityShaperExpression = new RelationalEntityShaperExpression( entityType, @@ -1156,54 +1195,99 @@ private Expression CreateJsonShapers( nullable); var entityShaperMaterializer = (BlockExpression)_parentVisitor.InjectEntityMaterializers(entityShaperExpression); - var entityShaperMaterializerVariable = Expression.Variable(entityShaperMaterializer.Type); - shaperBlockVariables.Add(entityShaperMaterializerVariable); - shaperBlockExpressions.Add(Expression.Assign(entityShaperMaterializerVariable, entityShaperMaterializer)); + var innerShapersMap = new Dictionary(); + var innerFixupMap = new Dictionary(); foreach (var ownedNavigation in entityType.GetNavigations().Where( - n => n.TargetEntityType.IsMappedToJson() && n.ForeignKey.IsOwnership && n == n.ForeignKey.PrincipalToDependent)) + n => n.TargetEntityType.IsMappedToJson() && n.ForeignKey.IsOwnership && n == n.ForeignKey.PrincipalToDependent)) { - // TODO: use caching like we do in pre-process, there's chance we already have this json element - var innerJsonElementParameter = Expression.Variable( - typeof(JsonElement?)); - - shaperBlockVariables.Add(innerJsonElementParameter); - - // JsonElement temp; - // JsonElement? innerJsonElement = jsonElement.TryGetProperty("PropertyName", temp) - // ? (JsonElement?)temp - // : null; - var tempParameter = Expression.Variable(typeof(JsonElement)); - shaperBlockVariables.Add(tempParameter); - - var innerJsonElementAssignment = Expression.Assign( - innerJsonElementParameter, - Expression.Condition( - Expression.Call( - jsonElementShaperLambdaParameter, - JsonElementTryGetPropertyMethod, - Expression.Constant(ownedNavigation.TargetEntityType.GetJsonPropertyName()), - tempParameter), - Expression.Convert( - tempParameter, - typeof(JsonElement?)), - Expression.Constant(null, typeof(JsonElement?)))); - - shaperBlockExpressions.Add(innerJsonElementAssignment); - - var innerShaperResult = CreateJsonShapers( + // we need to build entity shapers and fixup separately + // we don't know the order in which data comes, so we need to read through everything + // before we can do fixup safely + var innerShaper = CreateJsonShapers( ownedNavigation.TargetEntityType, nullable || !ownedNavigation.ForeignKey.IsRequired, - ownedNavigation.IsCollection, - innerJsonElementParameter, + jsonReaderDataShaperLambdaParameter, keyValuesShaperLambdaParameter, - entityShaperMaterializerVariable, - ownedNavigation); + parentEntityExpression: null, + navigation: ownedNavigation); - shaperBlockExpressions.Add(innerShaperResult); + var navigationJsonPropertyName = ownedNavigation.TargetEntityType.GetJsonPropertyName()!; + innerShapersMap[navigationJsonPropertyName] = innerShaper; + + if (ownedNavigation.IsCollection) + { + var shaperEntityParameter = Expression.Parameter(ownedNavigation.DeclaringEntityType.ClrType); + var shaperCollectionParameter = Expression.Parameter(ownedNavigation.ClrType); + var expressions = new List(); + + if (!ownedNavigation.IsShadowProperty()) + { + expressions.Add( + shaperEntityParameter.MakeMemberAccess(ownedNavigation.GetMemberInfo(forMaterialization: true, forSet: true)).Assign(shaperCollectionParameter)); + } + + if (ownedNavigation.Inverse is INavigation inverseNavigation + && !inverseNavigation.IsShadowProperty()) + { + //for (var i = 0; i < prm.Count; i++) + //{ + // prm[i].Parent = instance + //} + var innerFixupCollectionElementParameter = Expression.Parameter(inverseNavigation.DeclaringEntityType.ClrType); + var innerFixupParentParameter = Expression.Parameter(inverseNavigation.TargetEntityType.ClrType); + + var elementFixup = Expression.Lambda( + Expression.Block( + typeof(void), + AssignReferenceNavigation( + innerFixupCollectionElementParameter, + innerFixupParentParameter, + inverseNavigation)), + innerFixupCollectionElementParameter, + innerFixupParentParameter); + + expressions.Add( + Expression.Call( + InverseCollectionFixupMethod.MakeGenericMethod( + inverseNavigation.DeclaringEntityType.ClrType, + inverseNavigation.TargetEntityType.ClrType), + shaperCollectionParameter, + shaperEntityParameter, + elementFixup)); + } + + var fixup = Expression.Lambda( + Expression.Block(typeof(void), expressions), + shaperEntityParameter, + shaperCollectionParameter); + + innerFixupMap[navigationJsonPropertyName] = fixup; + } + else + { + var fixup = GenerateReferenceFixupForJson( + ownedNavigation.DeclaringEntityType.ClrType, + ownedNavigation.TargetEntityType.ClrType, + ownedNavigation, + ownedNavigation.Inverse); + + innerFixupMap[navigationJsonPropertyName] = fixup; + } } - shaperBlockExpressions.Add(entityShaperMaterializerVariable); + var rewrittenEntityShaperMaterializer = new JsonEntityMaterializerRewriter( + entityShaperExpression.EntityType, + _isTracking, + jsonReaderDataShaperLambdaParameter, + innerShapersMap, innerFixupMap).Rewrite(entityShaperMaterializer); + + var entityShaperMaterializerVariable = Expression.Variable( + entityShaperMaterializer.Type, + "entityShaperMaterializer"); + + shaperBlockVariables.Add(entityShaperMaterializerVariable); + shaperBlockExpressions.Add(Expression.Assign(entityShaperMaterializerVariable, rewrittenEntityShaperMaterializer)); var shaperBlock = Expression.Block( shaperBlockVariables, @@ -1213,10 +1297,14 @@ private Expression CreateJsonShapers( shaperBlock, QueryCompilationContext.QueryContextParameter, keyValuesShaperLambdaParameter, - jsonElementShaperLambdaParameter); + jsonReaderDataShaperLambdaParameter); if (parentEntityExpression != null) { + // this happens only on top level when we project owner entity in this case we can do fixup as part of generating materializer + // (since we are guaranteed that the parent already exists) - for nested JSON materialization we need to do fixup at the end + // because we are streaming the data and don't know if we get the parent json object before the child + // (in case parent ctor takes some parameters and they are read as last thing in the JSON) Check.DebugAssert(navigation != null, "Navigation shouldn't be null when including."); var fixup = GenerateFixup( @@ -1238,11 +1326,12 @@ private Expression CreateJsonShapers( navigation.DeclaringEntityType.ClrType, navigation.TargetEntityType.ClrType), QueryCompilationContext.QueryContextParameter, - jsonElementParameter, keyValuesParameter, + jsonReaderDataParameter, includingEntityExpression, shaperLambda, - fixup); + fixup, + Expression.Constant(_isTracking)); return navigation.DeclaringEntityType.ClrType.IsAssignableFrom(parentEntityExpression.Type) ? includeJsonEntityCollectionMethodCall @@ -1259,11 +1348,12 @@ private Expression CreateJsonShapers( navigation.DeclaringEntityType.ClrType, navigation.TargetEntityType.ClrType), QueryCompilationContext.QueryContextParameter, - jsonElementParameter, keyValuesParameter, + jsonReaderDataParameter, includingEntityExpression, shaperLambda, - fixup); + fixup, + Expression.Constant(_isTracking)); return navigation.DeclaringEntityType.ClrType.IsAssignableFrom(parentEntityExpression.Type) ? includeJsonEntityReferenceMethodCall @@ -1274,274 +1364,651 @@ private Expression CreateJsonShapers( includeJsonEntityReferenceMethodCall); } - if (collection) + if (navigation is { IsCollection: true }) { - Check.DebugAssert(navigation != null, "navigation shouldn't be null when materializing collection."); - - var materializeJsonEntityCollection = Expression.Call( - MaterializeJsonEntityCollectionMethodInfo.MakeGenericMethod( - entityType.ClrType, - navigation.ClrType), - QueryCompilationContext.QueryContextParameter, - jsonElementParameter, - keyValuesParameter, - Expression.Constant(navigation), - shaperLambda); + var materializeJsonEntityCollectionMethodCall = + Expression.Call( + MaterializeJsonEntityCollectionMethodInfo.MakeGenericMethod( + navigation.TargetEntityType.ClrType, + navigation.ClrType), + QueryCompilationContext.QueryContextParameter, + keyValuesParameter, + jsonReaderDataParameter, + Expression.Constant(navigation), + shaperLambda); - return materializeJsonEntityCollection; + return materializeJsonEntityCollectionMethodCall; } var materializedRootJsonEntity = Expression.Call( MaterializeJsonEntityMethodInfo.MakeGenericMethod(entityType.ClrType), QueryCompilationContext.QueryContextParameter, - jsonElementParameter, keyValuesParameter, + jsonReaderDataParameter, Expression.Constant(nullable), shaperLambda); return materializedRootJsonEntity; } - private (ParameterExpression, ParameterExpression) JsonShapingPreProcess( - JsonProjectionInfo jsonProjectionInfo, - IEntityType entityType) + private sealed class JsonEntityMaterializerRewriter : ExpressionVisitor { - if (_existingJsonElementMap.TryGetValue( - (jsonProjectionInfo.JsonColumnIndex, jsonProjectionInfo.AdditionalPath), - out var finalJsonElementVariable)) + private readonly IEntityType _entityType; + private readonly bool _isTracking; + private readonly ParameterExpression _jsonReaderDataParameter; + private readonly IDictionary _innerShapersMap; + private readonly IDictionary _innerFixupMap; + + private static readonly PropertyInfo JsonEncodedTextEncodedUtf8BytesProperty + = typeof(JsonEncodedText).GetProperty(nameof(JsonEncodedText.EncodedUtf8Bytes))!; + + // keep track which variable corresponds to which navigation - we need that info for fixup + // which happens at the end (after we read everything to guarantee that we can instantiate the entity + private readonly Dictionary _navigationVariableMap = new(); + + public JsonEntityMaterializerRewriter( + IEntityType entityType, + bool isTracking, + ParameterExpression jsonReaderDataParameter, + IDictionary innerShapersMap, + IDictionary innerFixupMap) { - // if we already cached JsonElement then key values are guaranteed to have been cached also, as they go in tandem - var fullPathCacheKey = jsonProjectionInfo.AdditionalPath.Select(x => (x.ConstantArrayIndex, x.NonConstantArrayIndex)).ToArray(); - var finalKeyValuesVariable = _existingKeyValuesMap[(jsonProjectionInfo.JsonColumnIndex, fullPathCacheKey)]; - - // if the JsonElement variable for the full path is present in the cache, - // it means we already went through this process before - // and have already generated all the steps leading to the result - // i.e. we can safely return from the pre process - return (finalJsonElementVariable, finalKeyValuesVariable); + _entityType = entityType; + _isTracking = isTracking; + _jsonReaderDataParameter = jsonReaderDataParameter; + _innerShapersMap = innerShapersMap; + _innerFixupMap = innerFixupMap; } - var currentJsonElementVariable = default(ParameterExpression); - var currentKeyValuesVariable = default(ParameterExpression); - var additionalKeyGeneratedCount = 0; + public BlockExpression Rewrite(BlockExpression jsonEntityShaperMaterializer) + => (BlockExpression)VisitBlock(jsonEntityShaperMaterializer); - // go through each segment in the additional path and generate JsonElement and key values - // store them in variables and cache them, so we can re-use it later if needed - // JsonElement needs to be generated for every path segment, as they are always different - // key values only changes if we access element of the array (as opposed to JSON property access) - for (var index = 0; index <= jsonProjectionInfo.AdditionalPath.Length; index++) + protected override Expression VisitSwitch(SwitchExpression switchExpression) { - var jsonElementCacheKey = jsonProjectionInfo.AdditionalPath[..index]; - var keyValuesCacheKey = jsonProjectionInfo.AdditionalPath[..index].Select(x => (x.ConstantArrayIndex, x.NonConstantArrayIndex)).ToArray(); - - if (_existingJsonElementMap.TryGetValue( - (jsonProjectionInfo.JsonColumnIndex, jsonElementCacheKey), - out var existingJsonElementVariable)) + if (switchExpression.SwitchValue.Type == typeof(IEntityType) + && switchExpression is + { + Cases: [{ TestValues: [ConstantExpression onlyValue], Body: BlockExpression body }] + } + && onlyValue.Value == _entityType + && body.Expressions.Count > 0) { - currentJsonElementVariable = existingJsonElementVariable; - currentKeyValuesVariable = _existingKeyValuesMap[(jsonProjectionInfo.JsonColumnIndex, keyValuesCacheKey)]; + var valueBufferTryReadValueMethodsToProcess = new ValueBufferTryReadValueMethodsFinder(_entityType).FindValueBufferTryReadValueMethods(body); - continue; - } - - if (index == 0) - { - var jsonColumnName = entityType.GetContainerColumnName()!; - var jsonColumnTypeMapping = (entityType.GetViewOrTableMappings().SingleOrDefault()?.Table - ?? entityType.GetDefaultMappings().Single().Table) - .FindColumn(jsonColumnName)!.StoreTypeMapping; + BlockExpression jsonEntityTypeInitializerBlock; + //sometimes we have shadow value buffer and sometimes not, but type initializer always comes last + switch (body.Expressions[^1]) + { + case BlockExpression b: + jsonEntityTypeInitializerBlock = b; + break; + // case where we don't use block but rather return construction directly, as in: + // return new MyEntity(...) + // + // rather than: + // return + // { + // MyEntity instance; + // instance = new MyEntity(...) + // (...) + // } + // we normalize this into block, since we are going to be adding extra statements (i.e. loop extracting JSON + // property values) there anyway + case NewExpression jsonEntityTypeInitializerCtor: + var newInstanceVariable = Expression.Variable(jsonEntityTypeInitializerCtor.Type, "instance"); + jsonEntityTypeInitializerBlock = Expression.Block( + new[] { newInstanceVariable }, + Expression.Assign(newInstanceVariable, jsonEntityTypeInitializerCtor), + newInstanceVariable); + break; + default: + throw new InvalidOperationException("IMPOSSIBLE"); + } - // create the JsonElement for the initial entity - var jsonElementValueExpression = CreateGetValueExpression( - _dataReaderParameter, - jsonProjectionInfo.JsonColumnIndex, - nullable: true, - jsonColumnTypeMapping, - typeof(JsonElement?), - property: null); + var managerVariable = Expression.Variable(typeof(Utf8JsonReaderManager), "jsonReaderManager"); + var tokenTypeVariable = Expression.Variable(typeof(JsonTokenType), "tokenType"); + var jsonEntityTypeVariable = (ParameterExpression)jsonEntityTypeInitializerBlock.Expressions[^1]; - currentJsonElementVariable = Expression.Variable( - typeof(JsonElement?)); + Debug.Assert(jsonEntityTypeVariable.Type == _entityType.ClrType); - var jsonElementAssignment = Expression.Assign( - currentJsonElementVariable, - jsonElementValueExpression); + var finalBlockVariables = new List + { + managerVariable, tokenTypeVariable, + }; - _variables.Add(currentJsonElementVariable); - _expressions.Add(jsonElementAssignment); + finalBlockVariables.AddRange(jsonEntityTypeInitializerBlock.Variables); - var keyValues = new Expression[jsonProjectionInfo.KeyAccessInfo.Count]; - for (var i = 0; i < jsonProjectionInfo.KeyAccessInfo.Count; i++) - { - if (jsonProjectionInfo.KeyAccessInfo[i].ConstantKeyValue is int constant) + var finalBlockExpressions = new List { - // if key access was a constant (and we have the actual value) add it directly to key values array - // adding 1 to the value as we start keys from 1 and the array starts at 0 - keyValues[i] = Expression.Convert( - Expression.Constant(constant + 1), - typeof(object)); - } - else if (jsonProjectionInfo.KeyAccessInfo[i].KeyProperty is IProperty keyProperty) + // jsonReaderManager = new Utf8JsonReaderManager(jsonReaderData)) + Expression.Assign( + managerVariable, + Expression.New( + JsonReaderManagerConstructor, + _jsonReaderDataParameter)), + // tokenType = jsonReaderManager.CurrentReader.TokenType + Expression.Assign( + tokenTypeVariable, + Expression.Property( + Expression.Field( + managerVariable, + Utf8JsonReaderManagerCurrentReaderField), + Utf8JsonReaderTokenTypeProperty)), + }; + + var (loop, propertyAssignmentMap) = GenerateJsonPropertyReadLoop( + managerVariable, + tokenTypeVariable, + finalBlockVariables, + valueBufferTryReadValueMethodsToProcess); + + finalBlockExpressions.Add(loop); + + var finalCaptureState = Expression.Call(managerVariable, Utf8JsonReaderManagerCaptureStateMethod); + finalBlockExpressions.Add(finalCaptureState); + + // we have the loop, now we can add code that generate the entity instance + // will have to replace ValueBufferTryReadValue method calls with the parameters that store the value + // we can't use simple ExpressionReplacingVisitor, because there could be multiple instances of MethodCallExpression for given property + // using dedicated mini-visitor that looks for MCEs with a given shape and compare the IProperty inside + // order is: + // - shadow value buffer (if there was one) + // - entity construction / property assignments + // - navigation fixups + // - entity instance variable that is returned as end result + var propertyAssignmentReplacer = new ValueBufferTryReadValueMethodsReplacer(propertyAssignmentMap); + + if (body.Expressions[0] is BinaryExpression { - // if key value has IProperty, it must be a PK of the owner - var projection = _selectExpression.Projection[jsonProjectionInfo.KeyAccessInfo[i].KeyProjectionIndex!.Value]; - keyValues[i] = Expression.Convert( - CreateGetValueExpression( - _dataReaderParameter, - jsonProjectionInfo.KeyAccessInfo[i].KeyProjectionIndex!.Value, - IsNullableProjection(projection), - projection.Expression.TypeMapping!, - keyProperty.ClrType, - keyProperty), - typeof(object)); - } - else + NodeType: ExpressionType.Assign, + Right: NewExpression + { + Arguments: [NewArrayExpression] + } + } shadowValueBufferAssignment + && shadowValueBufferAssignment.Type == typeof(ValueBuffer)) + { + finalBlockExpressions.Add(propertyAssignmentReplacer.Visit(shadowValueBufferAssignment)); + } + + foreach (var jsonEntityTypeInitializerBlockExpression in jsonEntityTypeInitializerBlock.Expressions.ToArray()[..^1]) + { + finalBlockExpressions.Add(propertyAssignmentReplacer.Visit(jsonEntityTypeInitializerBlockExpression)); + } + + // fixup is only needed for non-tracking queries, in case of tracking - ChangeTracker does the job + if (!_isTracking) + { + foreach (var fixup in _innerFixupMap) { - // otherwise it must be non-constant array access and we stored it's projection index - // extract the value from the projection (or the cache if we used it before) - var collectionElementAccessParameter = ExtractAndCacheNonConstantJsonArrayElementAccessValue( - jsonProjectionInfo.KeyAccessInfo[i].KeyProjectionIndex!.Value); - - keyValues[i] = Expression.Convert( - Expression.Add(collectionElementAccessParameter, Expression.Constant(1, typeof(int?))), - typeof(object)); + var navigationEntityParameter = _navigationVariableMap[fixup.Key]; + + // we need to add null checks before we run fixup logic. For regular entities, whose fixup is done as part of the "Materialize*" method + // the checks are done there (same will be done for the "optimized" scenario, where we populate properties directly rather than store in variables) + // but in this case fixups are standalone, so the null safety must be added by us directly + finalBlockExpressions.Add( + Expression.IfThen( + Expression.AndAlso( + Expression.NotEqual( + jsonEntityTypeVariable, + Expression.Constant(null, jsonEntityTypeVariable.Type)), + Expression.NotEqual( + navigationEntityParameter, + Expression.Constant(null, navigationEntityParameter.Type))), + Expression.Invoke( + fixup.Value, + jsonEntityTypeVariable, + _navigationVariableMap[fixup.Key]))); } } - // create key values for initial entity - currentKeyValuesVariable = Expression.Parameter(typeof(object[])); - var keyValuesAssignment = Expression.Assign( - currentKeyValuesVariable, - Expression.NewArrayInit(typeof(object), keyValues)); + finalBlockExpressions.Add(jsonEntityTypeVariable); + + return Expression.Block( + finalBlockVariables, + finalBlockExpressions); + } + + return base.VisitSwitch(switchExpression); + + // builds a loop that extracts values of JSON properties and assigns them into variables + // also injects entity shapers (generated earlier) for child navigations + // returns the loop expression and mappings for properties (so we know which calls to replace with variables) + (LoopExpression, Dictionary) GenerateJsonPropertyReadLoop( + ParameterExpression managerVariable, + ParameterExpression tokenTypeVariable, + List finalBlockVariables, + List valueBufferTryReadValueMethodsToProcess) + { + var breakLabel = Expression.Label("done"); + var testExpressions = new List(); + var readExpressions = new List(); + var propertyAssignmentMap = new Dictionary(); + + foreach (var valueBufferTryReadValueMethodToProcess in valueBufferTryReadValueMethodsToProcess) + { + var property = (IProperty)((ConstantExpression)valueBufferTryReadValueMethodToProcess.Arguments[2]).Value!; + + testExpressions.Add( + Expression.Call( + Expression.Field( + managerVariable, + Utf8JsonReaderManagerCurrentReaderField), + Utf8JsonReaderValueTextEqualsMethod, + Expression.Property( + Expression.Constant(JsonEncodedText.Encode(property.GetJsonPropertyName()!)), + JsonEncodedTextEncodedUtf8BytesProperty))); + + var propertyVariable = Expression.Variable(valueBufferTryReadValueMethodToProcess.Type); + + finalBlockVariables.Add(propertyVariable); + + var moveNext = Expression.Call( + managerVariable, + Utf8JsonReaderManagerMoveNextMethod); + + var assignment = Expression.Assign( + propertyVariable, + valueBufferTryReadValueMethodToProcess); + + readExpressions.Add( + Expression.Block( + moveNext, + assignment, + Expression.Empty())); + + propertyAssignmentMap[property] = propertyVariable; + } + + foreach (var innerShaperMapElement in _innerShapersMap) + { + testExpressions.Add( + Expression.Call( + Expression.Field( + managerVariable, + Utf8JsonReaderManagerCurrentReaderField), + Utf8JsonReaderValueTextEqualsMethod, + Expression.Property( + Expression.Constant(JsonEncodedText.Encode(innerShaperMapElement.Key)), + JsonEncodedTextEncodedUtf8BytesProperty))); + + var propertyVariable = Expression.Variable(innerShaperMapElement.Value.Type); + finalBlockVariables.Add(propertyVariable); + + _navigationVariableMap[innerShaperMapElement.Key] = propertyVariable; + + var moveNext = Expression.Call( + managerVariable, + Utf8JsonReaderManagerMoveNextMethod); + + var captureState = Expression.Call( + managerVariable, + Utf8JsonReaderManagerCaptureStateMethod); + + var assignment = Expression.Assign( + propertyVariable, + innerShaperMapElement.Value); + + var managerRecreation = Expression.Assign( + managerVariable, + Expression.New(JsonReaderManagerConstructor, _jsonReaderDataParameter)); + + readExpressions.Add( + Expression.Block( + moveNext, + captureState, + assignment, + managerRecreation, + Expression.Empty())); + } - _variables.Add(currentKeyValuesVariable); - _expressions.Add(keyValuesAssignment); + var testsCount = testExpressions.Count; + var testExpression = Expression.IfThen( + testExpressions[testsCount - 1], + readExpressions[testsCount - 1]); - _existingJsonElementMap[(jsonProjectionInfo.JsonColumnIndex, jsonElementCacheKey)] = currentJsonElementVariable; - _existingKeyValuesMap[(jsonProjectionInfo.JsonColumnIndex, keyValuesCacheKey)] = currentKeyValuesVariable; + for (var i = testsCount - 2; i >= 0; i--) + { + testExpression = Expression.IfThenElse( + testExpressions[i], + readExpressions[i], + testExpression); + } + + var loopBody = Expression.Block( + Expression.Assign( + tokenTypeVariable, + Expression.Call( + managerVariable, + Utf8JsonReaderManagerMoveNextMethod)), + Expression.Switch( + tokenTypeVariable, + Expression.Block( + Expression.Call( + Expression.Field( + managerVariable, + Utf8JsonReaderManagerCurrentReaderField), + Utf8JsonReaderTrySkipMethod), + Expression.Default(typeof(void))), + new SwitchCase[] + { + Expression.SwitchCase( + testExpression, + Expression.Constant(JsonTokenType.PropertyName)), + Expression.SwitchCase( + Expression.Break(breakLabel), + Expression.Constant(JsonTokenType.EndObject)), + })); + + return (Expression.Loop(loopBody, breakLabel), propertyAssignmentMap); } - else + } + + protected override Expression VisitConditional(ConditionalExpression conditionalExpression) + { + var visited = base.VisitConditional(conditionalExpression); + + // this code compensates for differences between regular entities and JSON entities for tracking queries + // for regular entities we preserve all the includes, so shaper for each entity is visited regardless + // because of that, the original entity materializer code short-circuits if we find entity in change tracker + // + // for JSON entities that is incorrect, because all includes are part of the parent's shaper + // so if we short circuit the parent, we never process the children + // this is a problem when someone modifies child entity in the database directly - we would never pick up those changes + // if we are tracking the parent + // the code here re-arranges the existing materializer so that even if we find parent in the change tracker + // we still process all the child navigations, it's just that we use the parent instance from change tracker, rather than create new one +#pragma warning disable EF1001 // Internal EF Core API usage. + if (_isTracking + && visited is ConditionalExpression + { + Test: BinaryExpression { NodeType: ExpressionType.NotEqual, Left: ParameterExpression leftPrm, Right: DefaultExpression rightDefault } testBinaryExpression, + IfTrue: BlockExpression ifTrueBlock, + IfFalse: BlockExpression ifFalseBlock + } resultConditional + && rightDefault.Type == typeof(InternalEntityEntry)) { - // create JsonElement for the additional path segment - var currentPath = jsonProjectionInfo.AdditionalPath[index - 1]; + var entityAlreadyTrackedVariable = Expression.Variable(typeof(bool), "entityAlreadyTracked"); - if (currentPath.JsonPropertyName is string stringPath) + var resultBlockVariables = new List { entityAlreadyTrackedVariable }; + var resultBlockExpressions = new List { - // JsonElement? jsonElement = (...) <- this is the previous one - // JsonElement temp; - // JsonElement? newJsonElement = jsonElement.HasValue && jsonElement.Value.TryGetProperty("PropertyName", temp) - // ? (JsonElement?)temp - // : null; - var tempParameter = Expression.Variable(typeof(JsonElement)); - _variables.Add(tempParameter); - - var tryGetPropertyCall = Expression.Call( - Expression.MakeMemberAccess( - currentJsonElementVariable!, - NullableJsonElementValuePropertyInfo), - JsonElementTryGetPropertyMethod, - Expression.Constant(stringPath), - tempParameter); - - var newJsonElementVariable = Expression.Variable( - typeof(JsonElement?)); - - var newJsonElementAssignment = Expression.Assign( - newJsonElementVariable, - Expression.Condition( - Expression.AndAlso( - Expression.MakeMemberAccess( - currentJsonElementVariable!, - NullableJsonElementHasValuePropertyInfo), - tryGetPropertyCall), - Expression.Convert(tempParameter, typeof(JsonElement?)), - Expression.Constant(null, typeof(JsonElement?)))); - - _variables.Add(newJsonElementVariable); - _expressions.Add(newJsonElementAssignment); - - currentJsonElementVariable = newJsonElementVariable; + Expression.Assign(entityAlreadyTrackedVariable, Expression.Constant(false)), + + // shadowValueBuffer = ValueBuffer; + ifFalseBlock.Expressions[0], + + // entityType = EntityType; + ifFalseBlock.Expressions[1], + + Expression.IfThen( + testBinaryExpression, + Expression.Block( + ifTrueBlock.Variables, + ifTrueBlock.Expressions.Concat(new Expression[] + { + Expression.Assign(entityAlreadyTrackedVariable, Expression.Constant(true)), + Expression.Default(typeof(void)) + }))) + }; + + resultBlockVariables.AddRange(ifFalseBlock.Variables.ToList()); + + var instanceAssignment = ifFalseBlock.Expressions.OfType().Single(e => e is { NodeType: ExpressionType.Assign, Left: ParameterExpression instance, Right: BlockExpression } && instance.Type == _entityType.ClrType); + var instanceAssignmentBody = (BlockExpression)instanceAssignment.Right; + + var newInstanceAssignmentVariables = instanceAssignmentBody.Variables.ToList(); + var newInstanceAssignmentExpressions = new List(); + + // we only need to generate shadowValueBuffer if the entity isn't already tracked + // shadow value buffer can be generated early in the block (default) + // or after we read all the values from JSON (case when the entity has some shadow properties) + // so we loop through the existing expressions and add the condition to value buffer assignment when we find it + // expressions processed here: + // shadowValueBuffer = new ValueBuffer(...) + // jsonManagerPrm = new Utf8JsonReaderManager(jsonReaderDataPrm); + // tokenType = jsonManagerPrm.TokenType; + // property_reading_loop(...) + // jsonManagerPrm.CaptureState(); + for (var i = 0; i < 5; i++) + { + newInstanceAssignmentExpressions.Add( + instanceAssignmentBody.Expressions[i].Type == typeof(ValueBuffer) + ? Expression.IfThen( + Expression.Not(entityAlreadyTrackedVariable), + instanceAssignmentBody.Expressions[i]) + : instanceAssignmentBody.Expressions[i]); } - else + + // from now on we have entity construction and property assignments + // then navigation fixup and then returning the final product + // entity construction could vary in length (e.g. when we have custom materializer) + // but we know how many navigation fixups there are and that instance is returned as last statement + var innerInstanceVariable = instanceAssignmentBody.Expressions[^1]; + + var createAndPopulateInstanceIfTrueBlock = Expression.Block( + Expression.Assign(innerInstanceVariable, instanceAssignment.Left), + Expression.Default(typeof(void))); + + // all expressions except first 5 (that we already added) + // final variable being returned is also omitted but we generate Express.Default(typeof(void)) instead + var createAndPopulateInstanceIfFalseBlockExpressionsCount = instanceAssignmentBody.Expressions.Count - 5; + var createAndPopulateInstanceIfFalseBlockExpressions = new Expression[createAndPopulateInstanceIfFalseBlockExpressionsCount]; + + Array.Copy( + instanceAssignmentBody.Expressions.ToArray()[5..^1], + createAndPopulateInstanceIfFalseBlockExpressions, + createAndPopulateInstanceIfFalseBlockExpressionsCount - 1); + + createAndPopulateInstanceIfFalseBlockExpressions[^1] = Expression.Default(typeof(void)); + + var createAndPopulateInstanceExpression = Expression.IfThenElse( + entityAlreadyTrackedVariable, + createAndPopulateInstanceIfTrueBlock, + Expression.Block(createAndPopulateInstanceIfFalseBlockExpressions)); + + newInstanceAssignmentExpressions.Add(createAndPopulateInstanceExpression); + newInstanceAssignmentExpressions.Add(innerInstanceVariable); + + var newInstanceAssignmentBlock = Expression.Block(newInstanceAssignmentVariables, newInstanceAssignmentExpressions); + + resultBlockExpressions.Add( + Expression.Assign( + instanceAssignment.Left, + newInstanceAssignmentBlock)); + + var startTrackingAssignment = ifFalseBlock.Expressions + .OfType() + .Single(e => e is { NodeType: ExpressionType.Assign, Left: ParameterExpression instance, Right: ConditionalExpression } && instance.Type == typeof(InternalEntityEntry)); + + var startTrackingExpression = Expression.IfThen( + Expression.Not( + Expression.OrElse( + entityAlreadyTrackedVariable, + ((ConditionalExpression)startTrackingAssignment.Right).Test)), + Expression.Block( + ((ConditionalExpression)startTrackingAssignment.Right).IfFalse, + Expression.Default(typeof(void)))); + + resultBlockExpressions.Add(startTrackingExpression); + resultBlockExpressions.Add(Expression.Default(typeof(void))); + var resultBlock = Expression.Block(resultBlockVariables, resultBlockExpressions); + + return resultBlock; + } +#pragma warning restore EF1001 // Internal EF Core API usage. + + return visited; + } + + private sealed class ValueBufferTryReadValueMethodsFinder : ExpressionVisitor + { + private readonly List _nonKeyProperties; + private readonly List _valueBufferTryReadValueMethods = new(); + + public ValueBufferTryReadValueMethodsFinder(IEntityType entityType) + { + _nonKeyProperties = entityType.GetProperties().Where(p => !p.IsPrimaryKey()).ToList(); + } + + public List FindValueBufferTryReadValueMethods(Expression expression) + { + _valueBufferTryReadValueMethods.Clear(); + + Visit(expression); + + return _valueBufferTryReadValueMethods; + } + + protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression) + { + if (methodCallExpression.Method.IsGenericMethod + && methodCallExpression.Method.GetGenericMethodDefinition() == Infrastructure.ExpressionExtensions.ValueBufferTryReadValueMethod + && ((ConstantExpression)methodCallExpression.Arguments[2]).Value is IProperty property + && _nonKeyProperties.Contains(property)) { - var elementAccessExpression = currentPath.ConstantArrayIndex is int constantElementAccess - ? (Expression)Expression.Constant(constantElementAccess) - : Expression.Convert( - ExtractAndCacheNonConstantJsonArrayElementAccessValue(currentPath.NonConstantArrayIndex!.Value), - typeof(int)); - - Expression jsonElementAccessExpressionFragment = Expression.Call( - Expression.MakeMemberAccess( - currentJsonElementVariable!, - NullableJsonElementValuePropertyInfo), - JsonElementGetItemMethodInfo, - elementAccessExpression); - - additionalKeyGeneratedCount++; - if (_existingKeyValuesMap.TryGetValue((jsonProjectionInfo.JsonColumnIndex, keyValuesCacheKey), out var existingKeyValuesVariable)) - { - currentKeyValuesVariable = existingKeyValuesVariable; - } - else - { - // create new array of size 1 more than current array (as we will be adding the extra key value) - // copy values from current array and set the last remaining value - var previousKeyValuesVariable = currentKeyValuesVariable; - currentKeyValuesVariable = Expression.Parameter(typeof(object[])); - - var currentKeyValuesCount = jsonProjectionInfo.KeyAccessInfo.Count - + additionalKeyGeneratedCount; - - var currentKeyValuesArrayInitAssignment = Expression.Assign( - currentKeyValuesVariable, - Expression.NewArrayBounds( - typeof(object), - Expression.Constant(currentKeyValuesCount))); - - var keyValuesArrayCopyFromPrevious = Expression.Call( - ArrayCopyMethodInfo, - previousKeyValuesVariable!, - currentKeyValuesVariable, - Expression.Constant(currentKeyValuesCount - 1)); - - var missingKeyValueAssignment = Expression.Assign( - Expression.MakeIndex( - currentKeyValuesVariable, - ObjectArrayIndexerPropertyInfo, - new[] { Expression.Constant(currentKeyValuesCount - 1) }), - Expression.Convert( - Expression.Add(elementAccessExpression, Expression.Constant(1)), - typeof(object))); - - _variables.Add(currentKeyValuesVariable); - _expressions.Add(currentKeyValuesArrayInitAssignment); - _expressions.Add(keyValuesArrayCopyFromPrevious); - _expressions.Add(missingKeyValueAssignment); - } + _valueBufferTryReadValueMethods.Add(methodCallExpression); + _nonKeyProperties.Remove(property); + + return methodCallExpression; + } - var jsonElementValueExpression = Expression.Condition( - Expression.MakeMemberAccess( - currentJsonElementVariable, - NullableJsonElementHasValuePropertyInfo), - Expression.Convert( - jsonElementAccessExpressionFragment, - currentJsonElementVariable!.Type), - Expression.Default(currentJsonElementVariable.Type)); + return base.VisitMethodCall(methodCallExpression); + } + } - currentJsonElementVariable = Expression.Variable( - typeof(JsonElement?)); + private sealed class ValueBufferTryReadValueMethodsReplacer : ExpressionVisitor + { + private readonly Dictionary _propertyAssignmentMap; - var jsonElementAssignment = Expression.Assign( - currentJsonElementVariable, - jsonElementValueExpression); + public ValueBufferTryReadValueMethodsReplacer(Dictionary propertyAssignmentMap) + { + _propertyAssignmentMap = propertyAssignmentMap; + } - _variables.Add(currentJsonElementVariable); - _expressions.Add(jsonElementAssignment); + protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression) + { + if (methodCallExpression.Method.IsGenericMethod + && methodCallExpression.Method.GetGenericMethodDefinition() == Infrastructure.ExpressionExtensions.ValueBufferTryReadValueMethod + && ((ConstantExpression)methodCallExpression.Arguments[2]).Value is IProperty property + && _propertyAssignmentMap.TryGetValue(property, out var parameter)) + { + return parameter; } + + return base.VisitMethodCall(methodCallExpression); + } + } + } + + private (ParameterExpression, ParameterExpression) JsonShapingPreProcess( + JsonProjectionInfo jsonProjectionInfo, + IEntityType entityType) + { + var jsonColumnName = entityType.GetContainerColumnName()!; + var jsonColumnTypeMapping = (entityType.GetViewOrTableMappings().SingleOrDefault()?.Table + ?? entityType.GetDefaultMappings().Single().Table) + .FindColumn(jsonColumnName)!.StoreTypeMapping; + + var jsonStreamVariable = Expression.Variable(typeof(Stream), "jsonStream"); + var jsonReaderDataVariable = Expression.Variable(typeof(JsonReaderData), "jsonReader"); + var jsonReaderManagerVariable = Expression.Variable(typeof(Utf8JsonReaderManager), "jsonReaderManager"); + + var jsonStreamAssignment = Expression.Assign( + jsonStreamVariable, + CreateGetValueExpression( + _dataReaderParameter, + jsonProjectionInfo.JsonColumnIndex, + nullable: true, + jsonColumnTypeMapping, + typeof(MemoryStream), + property: null)); + + var jsonReaderDataAssignment = Expression.Assign( + jsonReaderDataVariable, + Expression.Condition( + Expression.Equal( + jsonStreamVariable, + Expression.Default(typeof(MemoryStream))), + Expression.Default(typeof(JsonReaderData)), + Expression.New(JsonReaderDataConstructor, jsonStreamVariable))); + + // if (jsonReaderData) != default + // { + // var jsonReaderManager = new Utf8JsonReaderManager(jsonReaderData); + // jsonReaderManager.MoveNext(); + // jsonReaderManager.CaptureState(); + // } + var jsonReaderManagerBlock = Expression.IfThen( + Expression.NotEqual( + jsonReaderDataVariable, + Expression.Default(typeof(JsonReaderData))), + Expression.Block( + Expression.Assign( + jsonReaderManagerVariable, + Expression.New(JsonReaderManagerConstructor, jsonReaderDataVariable)), + Expression.Call( + jsonReaderManagerVariable, + Utf8JsonReaderManagerMoveNextMethod), + Expression.Call( + jsonReaderManagerVariable, + Utf8JsonReaderManagerCaptureStateMethod))); + + _variables.Add(jsonStreamVariable); + _variables.Add(jsonReaderDataVariable); + _variables.Add(jsonReaderManagerVariable); + _expressions.Add(jsonStreamAssignment); + _expressions.Add(jsonReaderDataAssignment); + _expressions.Add(jsonReaderManagerBlock); + + var keyValues = new Expression[jsonProjectionInfo.KeyAccessInfo.Count]; + for (var i = 0; i < jsonProjectionInfo.KeyAccessInfo.Count; i++) + { + var keyAccessInfo = jsonProjectionInfo.KeyAccessInfo[i]; + switch (keyAccessInfo) + { + case { ConstantKeyValue: int constant }: + // if key access was a constant (and we have the actual value) add it directly to key values array + // adding 1 to the value as we start keys from 1 and the array starts at 0 + keyValues[i] = Expression.Convert( + Expression.Constant(constant + 1), + typeof(object)); + break; + case { KeyProperty: IProperty keyProperty }: + // if key value has IProperty, it must be a PK of the owner + var projection = _selectExpression.Projection[keyAccessInfo.KeyProjectionIndex!.Value]; + keyValues[i] = Expression.Convert( + CreateGetValueExpression( + _dataReaderParameter, + keyAccessInfo.KeyProjectionIndex!.Value, + IsNullableProjection(projection), + projection.Expression.TypeMapping!, + keyProperty.ClrType, + keyProperty), + typeof(object)); + break; + default: + // otherwise it must be non-constant array access and we stored its projection index + // extract the value from the projection (or the cache if we used it before) + var collectionElementAccessParameter = ExtractAndCacheNonConstantJsonArrayElementAccessValue( + keyAccessInfo.KeyProjectionIndex!.Value); + keyValues[i] = Expression.Convert( + Expression.Add(collectionElementAccessParameter, Expression.Constant(1, typeof(int?))), + typeof(object)); + break; } } - return (currentJsonElementVariable!, currentKeyValuesVariable!); + // create key values for initial entity + var currentKeyValuesVariable = Expression.Variable(typeof(object[]), "currentKeyValues"); + var keyValuesAssignment = Expression.Assign( + currentKeyValuesVariable, + Expression.NewArrayInit(typeof(object), keyValues)); + + _variables.Add(currentKeyValuesVariable); + _expressions.Add(keyValuesAssignment); + + return (jsonReaderDataVariable, currentKeyValuesVariable); ParameterExpression ExtractAndCacheNonConstantJsonArrayElementAccessValue(int index) { @@ -1602,6 +2069,49 @@ private static LambdaExpression GenerateFixup( return Expression.Lambda(Expression.Block(typeof(void), expressions), entityParameter, relatedEntityParameter); } + private static LambdaExpression GenerateReferenceFixupForJson( + Type entityType, + Type relatedEntityType, + INavigationBase navigation, + INavigationBase? inverseNavigation) + { + var entityParameter = Expression.Parameter(entityType); + var relatedEntityParameter = Expression.Parameter(relatedEntityType); + var expressions = new List(); + + if (!navigation.IsShadowProperty()) + { + expressions.Add( + AssignReferenceNavigation( + entityParameter, + relatedEntityParameter, + navigation)); + } + + if (inverseNavigation != null + && !inverseNavigation.IsShadowProperty()) + { + expressions.Add( + AssignReferenceNavigation( + relatedEntityParameter, + entityParameter, + inverseNavigation)); + } + + return Expression.Lambda(Expression.Block(typeof(void), expressions), entityParameter, relatedEntityParameter); + } + + private static void InverseCollectionFixup( + ICollection collection, + TEntity entity, + Action elementFixup) + { + foreach (var collectionElement in collection) + { + elementFixup(collectionElement, entity); + } + } + private static Expression AssignReferenceNavigation( ParameterExpression entity, ParameterExpression relatedEntity, @@ -1760,22 +2270,46 @@ Expression valueExpression return valueExpression; } - private Expression CreateExtractJsonPropertyExpression( - ParameterExpression jsonElementParameter, + private Expression CreateReadJsonPropertyValueExpression( + ParameterExpression jsonReaderManagerParameter, IProperty property) { var nullable = property.IsNullable; - Expression resultExpression; + var typeMapping = property.GetTypeMapping(); + var providerClrType = (typeMapping.Converter?.ProviderClrType ?? typeMapping.ClrType).UnwrapNullableType(); + + var jsonReaderWriterExpression = Expression.Constant(property.GetJsonValueReaderWriter()!); + + var fromJsonMethod = jsonReaderWriterExpression.Type.GetMethod( + nameof(JsonValueReaderWriter.FromJson), new[] { typeof(Utf8JsonReaderManager).MakeByRefType() })!; + + Expression resultExpression = Expression.Convert( + Expression.Call(jsonReaderWriterExpression, fromJsonMethod, jsonReaderManagerParameter), + providerClrType); + if (property.GetTypeMapping().Converter is ValueConverter converter) { - var providerClrType = converter.ProviderClrType.MakeNullable(nullable); if (!property.IsNullable || converter.ConvertsNulls) { - resultExpression = Expression.Call( - ExtractJsonPropertyMethodInfo.MakeGenericMethod(providerClrType), - jsonElementParameter, - Expression.Constant(property.GetJsonPropertyName()), - Expression.Constant(nullable)); + // in case of null value we can't just use the JsonReader method, but rather check the current token type + // if it's JsonTokenType.Null means value is null, only if it's not we are safe to read the value + if (nullable) + { + resultExpression = Expression.Condition( + Expression.Equal( + Expression.Property( + Expression.Field( + jsonReaderManagerParameter, + Utf8JsonReaderManagerCurrentReaderField), + Utf8JsonReaderTokenTypeProperty), + Expression.Constant(JsonTokenType.Null)), + Expression.Default(providerClrType), + resultExpression); + } + + resultExpression = Expression.Convert( + Expression.Call(jsonReaderWriterExpression, fromJsonMethod, jsonReaderManagerParameter), + providerClrType); resultExpression = ReplacingExpressionVisitor.Replace( converter.ConvertFromProviderExpression.Parameters.Single(), @@ -1791,52 +2325,55 @@ private Expression CreateExtractJsonPropertyExpression( { // property is nullable and the converter can't handle nulls // we need to peek into the JSON value and only pass it thru converter if it's not null - var jsonPropertyCall = Expression.Call( - ExtractJsonPropertyMethodInfo.MakeGenericMethod(providerClrType), - jsonElementParameter, - Expression.Constant(property.GetJsonPropertyName()), - Expression.Constant(nullable)); - - var jsonPropertyVariable = Expression.Variable(providerClrType); - var jsonPropertyAssignment = Expression.Assign(jsonPropertyVariable, jsonPropertyCall); - - var testExpression = Expression.NotEqual( - jsonPropertyVariable, - Expression.Default(providerClrType)); + resultExpression = Expression.Convert( + Expression.Call(jsonReaderWriterExpression, fromJsonMethod, jsonReaderManagerParameter), + providerClrType); - var ifTrueExpression = (Expression)jsonPropertyVariable; - if (ifTrueExpression.Type != converter.ProviderClrType) - { - ifTrueExpression = Expression.Convert(ifTrueExpression, converter.ProviderClrType); - } - - ifTrueExpression = ReplacingExpressionVisitor.Replace( + resultExpression = ReplacingExpressionVisitor.Replace( converter.ConvertFromProviderExpression.Parameters.Single(), - ifTrueExpression, + resultExpression, converter.ConvertFromProviderExpression.Body); - if (ifTrueExpression.Type != property.ClrType) + if (resultExpression.Type != property.ClrType) { - ifTrueExpression = Expression.Convert(ifTrueExpression, property.ClrType); + resultExpression = Expression.Convert(resultExpression, property.ClrType); } - var condition = Expression.Condition( - testExpression, - ifTrueExpression, - Expression.Default(property.ClrType)); - - resultExpression = Expression.Block( - new ParameterExpression[] { jsonPropertyVariable }, - new Expression[] { jsonPropertyAssignment, condition }); + resultExpression = Expression.Condition( + Expression.Equal( + Expression.Property( + Expression.Field( + jsonReaderManagerParameter, + Utf8JsonReaderManagerCurrentReaderField), + Utf8JsonReaderTokenTypeProperty), + Expression.Constant(JsonTokenType.Null)), + Expression.Default(property.ClrType), + resultExpression); } } else { - resultExpression = Expression.Call( - ExtractJsonPropertyMethodInfo.MakeGenericMethod(property.ClrType), - jsonElementParameter, - Expression.Constant(property.GetJsonPropertyName()), - Expression.Constant(nullable)); + if (nullable) + { + // in case of null value we can't just use the JsonReader method, but rather check the current token type + // if it's JsonTokenType.Null means value is null, only if it's not we are safe to read the value + + if (resultExpression.Type != property.ClrType) + { + resultExpression = Expression.Convert(resultExpression, property.ClrType); + } + + resultExpression = Expression.Condition( + Expression.Equal( + Expression.Property( + Expression.Field( + jsonReaderManagerParameter, + Utf8JsonReaderManagerCurrentReaderField), + Utf8JsonReaderTokenTypeProperty), + Expression.Constant(JsonTokenType.Null)), + Expression.Default(property.ClrType), + resultExpression); + } } if (_detailedErrorsEnabled) @@ -1886,33 +2423,5 @@ public bool ContainsCollectionMaterialization(Expression expression) return base.Visit(expression); } } - - private sealed class ExistingJsonElementMapKeyComparer - : IEqualityComparer<(int JsonColumnIndex, (string? JsonPropertyName, int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath)> - { - public bool Equals( - (int JsonColumnIndex, (string? JsonPropertyName, int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath) x, - (int JsonColumnIndex, (string? JsonPropertyName, int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath) y) - => x.JsonColumnIndex == y.JsonColumnIndex - && x.AdditionalPath.Length == y.AdditionalPath.Length - && x.AdditionalPath.SequenceEqual(y.AdditionalPath); - - public int GetHashCode([DisallowNull] (int JsonColumnIndex, (string? JsonPropertyName, int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath) obj) - => HashCode.Combine(obj.JsonColumnIndex, obj.AdditionalPath?.Length); - } - - private sealed class ExistingJsonKeyValuesMapKeyComparer - : IEqualityComparer<(int JsonColumnIndex, (int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath)> - { - public bool Equals( - (int JsonColumnIndex, (int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath) x, - (int JsonColumnIndex, (int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath) y) - => x.JsonColumnIndex == y.JsonColumnIndex - && x.AdditionalPath.Length == y.AdditionalPath.Length - && x.AdditionalPath.SequenceEqual(y.AdditionalPath); - - public int GetHashCode([DisallowNull] (int JsonColumnIndex, (int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath) obj) - => HashCode.Combine(obj.JsonColumnIndex, obj.AdditionalPath?.Length); - } } } diff --git a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs index adf28f083d5..092e1b225da 100644 --- a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs +++ b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs @@ -878,7 +878,6 @@ public Expression ApplyProjection( cloningExpressionVisitor = new CloningExpressionVisitor(); } - var jsonClientProjectionDeduplicationMap = BuildJsonProjectionDeduplicationMap(_clientProjections.OfType()); var earlierClientProjectionCount = _clientProjections.Count; var newClientProjections = new List(); var clientProjectionIndexMap = new List(); @@ -1065,10 +1064,6 @@ static void UpdateLimit(SelectExpression selectExpression) baseSelectExpression._mutable = true; baseSelectExpression._projection.Clear(); } - - //since we updated the client projections, we also need updated deduplication map - jsonClientProjectionDeduplicationMap = BuildJsonProjectionDeduplicationMap( - _clientProjections.Skip(i).OfType()); } var value = _clientProjections[i]; @@ -1087,7 +1082,12 @@ static void UpdateLimit(SelectExpression selectExpression) { var jsonProjectionResult = AddJsonProjection( jsonQueryExpression, - jsonScalarToAdd: jsonClientProjectionDeduplicationMap[jsonQueryExpression]); + jsonScalarToAdd: new JsonScalarExpression( + jsonQueryExpression.JsonColumn, + jsonQueryExpression.Path, + jsonQueryExpression.JsonColumn.Type, + jsonQueryExpression.JsonColumn.TypeMapping!, + jsonQueryExpression.IsNullable)); newClientProjections.Add(jsonProjectionResult); clientProjectionIndexMap.Add(newClientProjections.Count - 1); @@ -1546,8 +1546,7 @@ Expression CopyProjectionToOuter(SelectExpression innerSelectExpression, Express remappedConstant = Constant( new JsonProjectionInfo( projectionIndexMap[jsonProjectionInfo.JsonColumnIndex], - newKeyAccessInfo, - jsonProjectionInfo.AdditionalPath)); + newKeyAccessInfo)); } else { @@ -1569,9 +1568,6 @@ Expression CopyProjectionToOuter(SelectExpression innerSelectExpression, Express } else { - var jsonProjectionDeduplicationMap = BuildJsonProjectionDeduplicationMap( - _projectionMapping.Select(x => x.Value).OfType()); - var result = new Dictionary(_projectionMapping.Count); foreach (var (projectionMember, expression) in _projectionMapping) @@ -1580,7 +1576,13 @@ Expression CopyProjectionToOuter(SelectExpression innerSelectExpression, Express { EntityProjectionExpression entityProjection => AddEntityProjection(entityProjection), JsonQueryExpression jsonQueryExpression => AddJsonProjection( - jsonQueryExpression, jsonProjectionDeduplicationMap[jsonQueryExpression]), + jsonQueryExpression, + new JsonScalarExpression( + jsonQueryExpression.JsonColumn, + jsonQueryExpression.Path, + jsonQueryExpression.JsonColumn.Type, + jsonQueryExpression.JsonColumn.TypeMapping!, + jsonQueryExpression.IsNullable)), _ => Constant(AddToProjection((SqlExpression)expression, projectionMember.Last?.Name)) }; } @@ -1591,46 +1593,6 @@ Expression CopyProjectionToOuter(SelectExpression innerSelectExpression, Express return shaperExpression; } - static Dictionary BuildJsonProjectionDeduplicationMap( - IEnumerable projections) - { - // force reference comparison for this one, even if we implement custom equality for JsonQueryExpression in the future - var deduplicationMap = new Dictionary(ReferenceEqualityComparer.Instance); - if (projections.Any()) - { - var ordered = projections - .OrderBy(x => $"{x.JsonColumn.TableAlias}.{x.JsonColumn.Name}") - .ThenBy(x => x.Path.Count) - .ThenBy(x => x.Path.Count > 0 && x.Path[^1].ArrayIndex != null); - - var needed = new List(); - foreach (var orderedElement in ordered) - { - var match = needed.FirstOrDefault(x => JsonEntityContainedIn(x, orderedElement)); - JsonScalarExpression jsonScalarExpression; - if (match == null) - { - jsonScalarExpression = new JsonScalarExpression( - orderedElement.JsonColumn, - orderedElement.Path, - orderedElement.JsonColumn.Type, - orderedElement.JsonColumn.TypeMapping!, - orderedElement.IsNullable); - - needed.Add(jsonScalarExpression); - } - else - { - jsonScalarExpression = match; - } - - deduplicationMap[orderedElement] = jsonScalarExpression; - } - } - - return deduplicationMap; - } - ConstantExpression AddEntityProjection(EntityProjectionExpression entityProjectionExpression) { var dictionary = new Dictionary(); @@ -1649,12 +1611,9 @@ ConstantExpression AddEntityProjection(EntityProjectionExpression entityProjecti ConstantExpression AddJsonProjection(JsonQueryExpression jsonQueryExpression, JsonScalarExpression jsonScalarToAdd) { - var additionalPath = jsonQueryExpression.Path - .Skip(jsonScalarToAdd.Path.Count) - .ToArray(); - - var jsonColumnIndex = AddToProjection(jsonScalarToAdd); - + var sqlExpression = AssignUniqueAliases(jsonScalarToAdd); + _projection.Add(new ProjectionExpression(sqlExpression, "")); + var jsonColumnIndex = _projection.Count - 1; var keyAccessInfo = new List<(IProperty?, int?, int?)>(); var keyProperties = GetMappedKeyProperties(jsonQueryExpression.EntityType.FindPrimaryKey()!); foreach (var keyProperty in keyProperties) @@ -1675,28 +1634,10 @@ ConstantExpression AddJsonProjection(JsonQueryExpression jsonQueryExpression, Js } } - var additionalPathList = new List<(string?, int?, int?)>(); - foreach (var additionalPathSegment in additionalPath) - { - if (additionalPathSegment.PropertyName is not null) - { - additionalPathList.Add((additionalPathSegment.PropertyName, null, null)); - } - else if (additionalPathSegment.ArrayIndex is SqlConstantExpression { Value: int intValue }) - { - additionalPathList.Add((null, intValue, null)); - } - else - { - additionalPathList.Add((null, null, AddToProjection(additionalPathSegment.ArrayIndex!))); - } - } - return Constant( new JsonProjectionInfo( jsonColumnIndex, - keyAccessInfo, - additionalPathList.ToArray())); + keyAccessInfo)); } static IReadOnlyList GetMappedKeyProperties(IKey key) @@ -1721,24 +1662,6 @@ static IReadOnlyList GetMappedKeyProperties(IKey key) return key.Properties.Take(count).ToList(); } - - static bool JsonEntityContainedIn(JsonScalarExpression sourceExpression, JsonQueryExpression targetExpression) - { - if (sourceExpression.Json != targetExpression.JsonColumn) - { - return false; - } - - var sourcePath = sourceExpression.Path; - var targetPath = targetExpression.Path; - - if (targetPath.Count < sourcePath.Count) - { - return false; - } - - return sourcePath.SequenceEqual(targetPath.Take(sourcePath.Count)); - } } /// diff --git a/src/EFCore.Relational/Update/ModificationCommand.cs b/src/EFCore.Relational/Update/ModificationCommand.cs index 5ef656d7d1b..03263e0fdc2 100644 --- a/src/EFCore.Relational/Update/ModificationCommand.cs +++ b/src/EFCore.Relational/Update/ModificationCommand.cs @@ -3,6 +3,8 @@ using System.Collections; using System.Data; +using System.Text; +using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.EntityFrameworkCore.ChangeTracking.Internal; using Microsoft.EntityFrameworkCore.Internal; @@ -322,19 +324,17 @@ private List GenerateColumnModifications() var navigation = finalUpdatePathElement.Navigation; var jsonColumnTypeMapping = jsonColumn.StoreTypeMapping; var navigationValue = finalUpdatePathElement.ParentEntry.GetCurrentValue(navigation); - - var json = default(JsonNode?); - var jsonPathString = string.Join( - ".", updateInfo.Path.Select(x => x.PropertyName + (x.Ordinal != null ? "[" + x.Ordinal + "]" : ""))); - - object? singlePropertyValue = default; - if (updateInfo.Property != null) + var jsonPathString = string.Join(".", updateInfo.Path.Select(x => x.PropertyName + (x.Ordinal != null ? "[" + x.Ordinal + "]" : ""))); + object? value; + if (updateInfo.Property is not null) { - singlePropertyValue = GenerateValueForSinglePropertyUpdate(updateInfo.Property, updateInfo.PropertyValue); + value = GenerateValueForSinglePropertyUpdate(updateInfo.Property, updateInfo.PropertyValue); jsonPathString = jsonPathString + "." + updateInfo.Property.GetJsonPropertyName(); } else { + var stream = new MemoryStream(); + var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = false }); if (finalUpdatePathElement.Ordinal != null && navigationValue != null) { var i = 0; @@ -342,12 +342,14 @@ private List GenerateColumnModifications() { if (i == finalUpdatePathElement.Ordinal) { - json = CreateJson( + WriteJson( + writer, navigationValueElement, finalUpdatePathElement.ParentEntry, navigation.TargetEntityType, ordinal: null, - isCollection: false); + isCollection: false, + isTopLevel: true); break; } @@ -357,18 +359,26 @@ private List GenerateColumnModifications() } else { - json = CreateJson( + WriteJson( + writer, navigationValue, finalUpdatePathElement.ParentEntry, navigation.TargetEntityType, ordinal: null, - isCollection: navigation.IsCollection); + isCollection: navigation.IsCollection, + isTopLevel: true); } + + writer.Flush(); + + value = writer.BytesCommitted > 0 + ? Encoding.UTF8.GetString(stream.ToArray()) + : null; } var columnModificationParameters = new ColumnModificationParameters( jsonColumn.Name, - value: updateInfo.Property != null ? singlePropertyValue : json?.ToJsonString(), + value: value, property: updateInfo.Property, columnType: jsonColumnTypeMapping.StoreType, jsonColumnTypeMapping, @@ -377,7 +387,8 @@ private List GenerateColumnModifications() write: true, key: false, condition: false, - _sensitiveLoggingEnabled) { GenerateParameterName = _generateParameterName, }; + _sensitiveLoggingEnabled) + { GenerateParameterName = _generateParameterName, }; columnModifications.Add(new ColumnModification(columnModificationParameters)); } @@ -718,32 +729,50 @@ static JsonPartialUpdateInfo FindCommonJsonPartialUpdateInfo( : propertyValue; } - private JsonNode? CreateJson(object? navigationValue, IUpdateEntry parentEntry, IEntityType entityType, int? ordinal, bool isCollection) + private void WriteJson( + Utf8JsonWriter writer, + object? navigationValue, + IUpdateEntry parentEntry, + IEntityType entityType, + int? ordinal, + bool isCollection, + bool isTopLevel) { if (navigationValue == null) { - return isCollection ? new JsonArray() : null; + if (!isTopLevel) + { + writer.WriteNullValue(); + } + + return; } if (isCollection) { var i = 1; - var jsonNodes = new List(); + writer.WriteStartArray(); foreach (var collectionElement in (IEnumerable)navigationValue) { - // TODO: should we ever expect null entities inside a collection? - var collectionElementJson = CreateJson(collectionElement, parentEntry, entityType, i++, isCollection: false); - jsonNodes.Add(collectionElementJson); + WriteJson( + writer, + collectionElement, + parentEntry, + entityType, + i++, + isCollection: false, + isTopLevel: false); } - return new JsonArray(jsonNodes.ToArray()); + writer.WriteEndArray(); + return; } #pragma warning disable EF1001 // Internal EF Core API usage. var entry = (IUpdateEntry)((InternalEntityEntry)parentEntry).StateManager.TryGetEntry(navigationValue, entityType)!; #pragma warning restore EF1001 // Internal EF Core API usage. - var jsonNode = new JsonObject(); + writer.WriteStartObject(); foreach (var property in entityType.GetProperties()) { if (property.IsKey()) @@ -759,7 +788,16 @@ static JsonPartialUpdateInfo FindCommonJsonPartialUpdateInfo( // jsonPropertyName can only be null for key properties var jsonPropertyName = property.GetJsonPropertyName()!; var value = entry.GetCurrentProviderValue(property); - jsonNode[jsonPropertyName] = JsonValue.Create(value); + writer.WritePropertyName(jsonPropertyName); + + if (value is not null) + { + property.GetJsonValueReaderWriter()!.ToJson(writer, value); + } + else + { + writer.WriteNullValue(); + } } foreach (var navigation in entityType.GetNavigations()) @@ -772,17 +810,19 @@ static JsonPartialUpdateInfo FindCommonJsonPartialUpdateInfo( var jsonPropertyName = navigation.TargetEntityType.GetJsonPropertyName()!; var ownedNavigationValue = entry.GetCurrentValue(navigation)!; - var navigationJson = CreateJson( + + writer.WritePropertyName(jsonPropertyName); + WriteJson( + writer, ownedNavigationValue, entry, navigation.TargetEntityType, ordinal: null, - isCollection: navigation.IsCollection); - - jsonNode[jsonPropertyName] = navigationJson; + isCollection: navigation.IsCollection, + isTopLevel: false); } - return jsonNode; + writer.WriteEndObject(); } private ITableMapping? GetTableMapping(IEntityType entityType) diff --git a/src/EFCore.SqlServer/Storage/Internal/SqlServerJsonTypeMapping.cs b/src/EFCore.SqlServer/Storage/Internal/SqlServerJsonTypeMapping.cs index 2b8e7da9f01..3ba48d4eb9a 100644 --- a/src/EFCore.SqlServer/Storage/Internal/SqlServerJsonTypeMapping.cs +++ b/src/EFCore.SqlServer/Storage/Internal/SqlServerJsonTypeMapping.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text; using System.Text.Json; namespace Microsoft.EntityFrameworkCore.SqlServer.Storage.Internal; @@ -16,11 +17,14 @@ public class SqlServerJsonTypeMapping : JsonTypeMapping private static readonly MethodInfo GetStringMethod = typeof(DbDataReader).GetRuntimeMethod(nameof(DbDataReader.GetString), new[] { typeof(int) })!; - private static readonly MethodInfo JsonDocumentParseMethod - = typeof(JsonDocument).GetRuntimeMethod(nameof(JsonDocument.Parse), new[] { typeof(string), typeof(JsonDocumentOptions) })!; + private static readonly PropertyInfo UTF8Property + = typeof(Encoding).GetProperty(nameof(Encoding.UTF8))!; - private static readonly MemberInfo JsonDocumentRootElementMember - = typeof(JsonDocument).GetRuntimeProperty(nameof(JsonDocument.RootElement))!; + private static readonly MethodInfo EncodingGetBytesMethod + = typeof(Encoding).GetMethod(nameof(Encoding.GetBytes), new[] { typeof(string) })!; + + private static readonly ConstructorInfo MemoryStreamConstructor + = typeof(MemoryStream).GetConstructor(new[] { typeof(byte[]) })!; /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -49,12 +53,12 @@ public override MethodInfo GetDataReaderMethod() /// doing so can result in application failures when updating to a new Entity Framework Core release. /// public override Expression CustomizeDataReaderExpression(Expression expression) - => Expression.MakeMemberAccess( + => Expression.New( + MemoryStreamConstructor, Expression.Call( - JsonDocumentParseMethod, - expression, - Expression.Default(typeof(JsonDocumentOptions))), - JsonDocumentRootElementMember); + Expression.Property(null, UTF8Property), + EncodingGetBytesMethod, + expression)); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to diff --git a/src/EFCore.Sqlite.Core/Storage/Internal/SqliteJsonTypeMapping.cs b/src/EFCore.Sqlite.Core/Storage/Internal/SqliteJsonTypeMapping.cs index 372e1e06fed..74c875bdd28 100644 --- a/src/EFCore.Sqlite.Core/Storage/Internal/SqliteJsonTypeMapping.cs +++ b/src/EFCore.Sqlite.Core/Storage/Internal/SqliteJsonTypeMapping.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text; using System.Text.Json; namespace Microsoft.EntityFrameworkCore.Sqlite.Storage.Internal; @@ -16,11 +17,14 @@ public class SqliteJsonTypeMapping : JsonTypeMapping private static readonly MethodInfo GetStringMethod = typeof(DbDataReader).GetRuntimeMethod(nameof(DbDataReader.GetString), new[] { typeof(int) })!; - private static readonly MethodInfo JsonDocumentParseMethod - = typeof(JsonDocument).GetRuntimeMethod(nameof(JsonDocument.Parse), new[] { typeof(string), typeof(JsonDocumentOptions) })!; + private static readonly PropertyInfo UTF8Property + = typeof(Encoding).GetProperty(nameof(Encoding.UTF8))!; - private static readonly MemberInfo JsonDocumentRootElementMember - = typeof(JsonDocument).GetRuntimeProperty(nameof(JsonDocument.RootElement))!; + private static readonly MethodInfo EncodingGetBytesMethod + = typeof(Encoding).GetMethod(nameof(Encoding.GetBytes), new[] { typeof(string) })!; + + private static readonly ConstructorInfo MemoryStreamConstructor + = typeof(MemoryStream).GetConstructor(new[] { typeof(byte[]) })!; /// /// Initializes a new instance of the class. @@ -58,12 +62,12 @@ public override MethodInfo GetDataReaderMethod() /// doing so can result in application failures when updating to a new Entity Framework Core release. /// public override Expression CustomizeDataReaderExpression(Expression expression) - => Expression.MakeMemberAccess( + => Expression.New( + MemoryStreamConstructor, Expression.Call( - JsonDocumentParseMethod, - expression, - Expression.Default(typeof(JsonDocumentOptions))), - JsonDocumentRootElementMember); + Expression.Property(null, UTF8Property), + EncodingGetBytesMethod, + expression)); /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to diff --git a/src/EFCore/Query/ExpressionPrinter.cs b/src/EFCore/Query/ExpressionPrinter.cs index 5c51f71449c..541a85c90fa 100644 --- a/src/EFCore/Query/ExpressionPrinter.cs +++ b/src/EFCore/Query/ExpressionPrinter.cs @@ -3,6 +3,7 @@ using System.Collections; using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; using System.Runtime.CompilerServices; namespace Microsoft.EntityFrameworkCore.Query; @@ -338,6 +339,10 @@ public virtual void VisitCollection(IReadOnlyCollection items, Action 0) @@ -417,8 +426,15 @@ protected override Expression VisitBlock(BlockExpression blockExpression) Append("return "); } - Visit(blockExpression.Result); - AppendLine(";"); + if (blockExpression.Result is not DefaultExpression) + { + Visit(blockExpression.Result); + + if (blockExpression.Result is not (BlockExpression or LoopExpression or SwitchExpression)) + { + AppendLine(";"); + } + } } } @@ -500,13 +516,24 @@ void PrintValue(object? value) /// protected override Expression VisitGoto(GotoExpression gotoExpression) { - AppendLine("return (" + gotoExpression.Target.Type.ShortDisplayName() + ")" + gotoExpression.Target + " {"); - using (_stringBuilder.Indent()) + Append("Goto(" + gotoExpression.Kind.ToString().ToLower() + " "); + + if (gotoExpression.Kind == GotoExpressionKind.Break) { - Visit(gotoExpression.Value); + Append(gotoExpression.Target.Name!); + } + else + { + AppendLine("(" + gotoExpression.Target.Type.ShortDisplayName() + ")" + gotoExpression.Target + " {"); + using (_stringBuilder.Indent()) + { + Visit(gotoExpression.Value); + } + + _stringBuilder.Append("}"); } - _stringBuilder.Append("}"); + AppendLine(")"); return gotoExpression; } @@ -1045,6 +1072,22 @@ protected override Expression VisitInvocation(InvocationExpression invocationExp return invocationExpression; } + /// + protected override Expression VisitLoop(LoopExpression loopExpression) + { + _stringBuilder.AppendLine($"Loop(Break: {loopExpression.BreakLabel?.Name} Continue: {loopExpression.ContinueLabel?.Name})"); + _stringBuilder.AppendLine("{"); + + using (_stringBuilder.Indent()) + { + Visit(loopExpression.Body); + } + + _stringBuilder.AppendLine("}"); + + return loopExpression; + } + /// protected override Expression VisitExtension(Expression extensionExpression) { diff --git a/src/EFCore/Query/Internal/EntityMaterializerSource.cs b/src/EFCore/Query/Internal/EntityMaterializerSource.cs index 4bb189c4350..a6f4fc00357 100644 --- a/src/EFCore/Query/Internal/EntityMaterializerSource.cs +++ b/src/EFCore/Query/Internal/EntityMaterializerSource.cs @@ -332,6 +332,7 @@ private static Expression CreateInterceptionMaterializeExpression( materializationDataVariable, instanceVariable), instanceVariable.Type))); + blockExpressions.Add(instanceVariable); return Expression.Block( bindingInfo.ServiceInstances.Concat(new[] { accessorDictionaryVariable, materializationDataVariable, creatingResultVariable }), diff --git a/test/EFCore.Design.Tests/Scaffolding/Internal/CSharpRuntimeModelCodeGeneratorTest.cs b/test/EFCore.Design.Tests/Scaffolding/Internal/CSharpRuntimeModelCodeGeneratorTest.cs index e541ad867a0..c00f01ecc37 100644 --- a/test/EFCore.Design.Tests/Scaffolding/Internal/CSharpRuntimeModelCodeGeneratorTest.cs +++ b/test/EFCore.Design.Tests/Scaffolding/Internal/CSharpRuntimeModelCodeGeneratorTest.cs @@ -2898,7 +2898,7 @@ public static void CreateAnnotations(RuntimeEntityType runtimeEntityType) c.SaveChanges(); }); - [ConditionalFact] + [ConditionalFact(Skip = "issue #31201")] [SqlServerConfiguredCondition] public void BigModel_with_JSON_columns() => Test( diff --git a/test/EFCore.Relational.Specification.Tests/Query/JsonQueryAdHocTestBase.cs b/test/EFCore.Relational.Specification.Tests/Query/JsonQueryAdHocTestBase.cs index 166ced63dea..0e8cc27363a 100644 --- a/test/EFCore.Relational.Specification.Tests/Query/JsonQueryAdHocTestBase.cs +++ b/test/EFCore.Relational.Specification.Tests/Query/JsonQueryAdHocTestBase.cs @@ -352,7 +352,6 @@ public virtual async Task Predicate_based_on_element_of_json_array_of_primitives } } - protected abstract void SeedArrayOfPrimitives(MyContextArrayOfPrimitives ctx); protected class MyContextArrayOfPrimitives : DbContext @@ -411,6 +410,303 @@ public class MyJsonEntityArrayOfPrimitives #endregion + #region JunkInJson + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual async Task Junk_in_json_basic_tracking(bool async) + { + var contextFactory = await InitializeAsync( + seed: SeedJunkInJson); + + using (var context = contextFactory.CreateContext()) + { + var query = context.Entities; + + var result = async + ? await query.ToListAsync() + : query.ToList(); + + Assert.Equal(1, result.Count); + Assert.Equal(2, result[0].Collection.Count); + Assert.Equal(2, result[0].CollectionWithCtor.Count); + Assert.Equal(2, result[0].Reference.NestedCollection.Count); + Assert.NotNull(result[0].Reference.NestedReference); + Assert.Equal(2, result[0].ReferenceWithCtor.NestedCollection.Count); + Assert.NotNull(result[0].ReferenceWithCtor.NestedReference); + } + } + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual async Task Junk_in_json_basic_no_tracking(bool async) + { + var contextFactory = await InitializeAsync( + seed: SeedJunkInJson); + + using (var context = contextFactory.CreateContext()) + { + var query = context.Entities.AsNoTracking(); + + var result = async + ? await query.ToListAsync() + : query.ToList(); + + Assert.Equal(1, result.Count); + Assert.Equal(2, result[0].Collection.Count); + Assert.Equal(2, result[0].CollectionWithCtor.Count); + Assert.Equal(2, result[0].Reference.NestedCollection.Count); + Assert.NotNull(result[0].Reference.NestedReference); + Assert.Equal(2, result[0].ReferenceWithCtor.NestedCollection.Count); + Assert.NotNull(result[0].ReferenceWithCtor.NestedReference); + } + } + + protected abstract void SeedJunkInJson(MyContextJunkInJson ctx); + + protected class MyContextJunkInJson : DbContext + { + public MyContextJunkInJson(DbContextOptions options) + : base(options) + { + } + + public DbSet Entities { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().Property(x => x.Id).ValueGeneratedNever(); + modelBuilder.Entity().OwnsOne(x => x.Reference, b => + { + b.ToJson(); + b.OwnsOne(x => x.NestedReference); + b.OwnsMany(x => x.NestedCollection); + }); + modelBuilder.Entity().OwnsOne(x => x.ReferenceWithCtor, b => + { + b.ToJson(); + b.OwnsOne(x => x.NestedReference); + b.OwnsMany(x => x.NestedCollection); + }); + modelBuilder.Entity().OwnsMany(x => x.Collection, b => + { + b.ToJson(); + b.OwnsOne(x => x.NestedReference); + b.OwnsMany(x => x.NestedCollection); + }); + modelBuilder.Entity().OwnsMany(x => x.CollectionWithCtor, b => + { + b.ToJson(); + b.OwnsOne(x => x.NestedReference); + b.OwnsMany(x => x.NestedCollection); + }); + } + } + + public class MyEntityJunkInJson + { + public int Id { get; set; } + public MyJsonEntityJunkInJson Reference { get; set; } + public MyJsonEntityJunkInJsonWithCtor ReferenceWithCtor { get; set; } + public List Collection { get; set; } + public List CollectionWithCtor { get; set; } + } + + public class MyJsonEntityJunkInJson + { + public string Name { get; set; } + public double Number { get; set; } + + public MyJsonEntityJunkInJsonNested NestedReference { get; set; } + public List NestedCollection { get; set; } + } + + public class MyJsonEntityJunkInJsonNested + { + public DateTime DoB { get; set; } + } + + public class MyJsonEntityJunkInJsonWithCtor + { + public MyJsonEntityJunkInJsonWithCtor(bool myBool, string name) + { + MyBool = myBool; + Name = name; + } + + public bool MyBool { get; set; } + public string Name { get; set; } + + public MyJsonEntityJunkInJsonWithCtorNested NestedReference { get; set; } + public List NestedCollection { get; set; } + } + + public class MyJsonEntityJunkInJsonWithCtorNested + { + public MyJsonEntityJunkInJsonWithCtorNested(DateTime doB) + { + DoB = doB; + } + + public DateTime DoB { get; set; } + } + + #endregion + + #region ShadowProperties + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual async Task Shadow_properties_basic_tracking(bool async) + { + var contextFactory = await InitializeAsync( + seed: SeedShadowProperties); + + using (var context = contextFactory.CreateContext()) + { + var query = context.Entities; + + var result = async + ? await query.ToListAsync() + : query.ToList(); + + Assert.Equal(1, result.Count); + Assert.Equal(2, result[0].Collection.Count); + Assert.Equal(2, result[0].CollectionWithCtor.Count); + Assert.NotNull(result[0].Reference); + Assert.NotNull(result[0].ReferenceWithCtor); + + var referenceEntry = context.ChangeTracker.Entries().Single(x => x.Entity == result[0].Reference); + Assert.Equal("Foo", referenceEntry.Property("ShadowString").CurrentValue); + + var referenceCtorEntry = context.ChangeTracker.Entries().Single(x => x.Entity == result[0].ReferenceWithCtor); + Assert.Equal(143, referenceCtorEntry.Property("Shadow_Int").CurrentValue); + + var collectionEntry1 = context.ChangeTracker.Entries().Single(x => x.Entity == result[0].Collection[0]); + var collectionEntry2 = context.ChangeTracker.Entries().Single(x => x.Entity == result[0].Collection[1]); + Assert.Equal(5.5, collectionEntry1.Property("ShadowDouble").CurrentValue); + Assert.Equal(20.5, collectionEntry2.Property("ShadowDouble").CurrentValue); + + var collectionCtorEntry1 = context.ChangeTracker.Entries().Single(x => x.Entity == result[0].CollectionWithCtor[0]); + var collectionCtorEntry2 = context.ChangeTracker.Entries().Single(x => x.Entity == result[0].CollectionWithCtor[1]); + Assert.Equal((byte)6, collectionCtorEntry1.Property("ShadowNullableByte").CurrentValue); + Assert.Null(collectionCtorEntry2.Property("ShadowNullableByte").CurrentValue); + } + } + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual async Task Shadow_properties_basic_no_tracking(bool async) + { + var contextFactory = await InitializeAsync( + seed: SeedShadowProperties); + + using (var context = contextFactory.CreateContext()) + { + var query = context.Entities.AsNoTracking(); + + var result = async + ? await query.ToListAsync() + : query.ToList(); + + Assert.Equal(1, result.Count); + Assert.Equal(2, result[0].Collection.Count); + Assert.Equal(2, result[0].CollectionWithCtor.Count); + Assert.NotNull(result[0].Reference); + Assert.NotNull(result[0].ReferenceWithCtor); + } + } + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual async Task Project_shadow_properties_from_json_entity(bool async) + { + var contextFactory = await InitializeAsync( + seed: SeedShadowProperties); + + using (var context = contextFactory.CreateContext()) + { + var query = context.Entities.Select(x => new + { + ShadowString = EF.Property(x.Reference, "ShadowString"), + ShadowInt = EF.Property(x.ReferenceWithCtor, "Shadow_Int"), + }); + + var result = async + ? await query.ToListAsync() + : query.ToList(); + + Assert.Equal(1, result.Count); + Assert.Equal("Foo", result[0].ShadowString); + Assert.Equal(143, result[0].ShadowInt); + } + } + + protected abstract void SeedShadowProperties(MyContextShadowProperties ctx); + + protected class MyContextShadowProperties : DbContext + { + public MyContextShadowProperties(DbContextOptions options) + : base(options) + { + } + + public DbSet Entities { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().Property(x => x.Id).ValueGeneratedNever(); + modelBuilder.Entity().OwnsOne(x => x.Reference, b => + { + b.ToJson(); + b.Property("ShadowString"); + }); + modelBuilder.Entity().OwnsOne(x => x.ReferenceWithCtor, b => + { + b.ToJson(); + b.Property("Shadow_Int").HasJsonPropertyName("ShadowInt"); + }); + modelBuilder.Entity().OwnsMany(x => x.Collection, b => + { + b.ToJson(); + b.Property("ShadowDouble"); + }); + modelBuilder.Entity().OwnsMany(x => x.CollectionWithCtor, b => + { + b.ToJson(); + b.Property("ShadowNullableByte"); + }); + } + } + + public class MyEntityShadowProperties + { + public int Id { get; set; } + public string Name { get; set; } + + public MyJsonEntityShadowProperties Reference { get; set; } + public List Collection { get; set; } + public MyJsonEntityShadowPropertiesWithCtor ReferenceWithCtor { get; set; } + public List CollectionWithCtor { get; set; } + } + + public class MyJsonEntityShadowProperties + { + public string Name { get; set; } + } + + public class MyJsonEntityShadowPropertiesWithCtor + { + public MyJsonEntityShadowPropertiesWithCtor(string name) + { + Name = name; + } + + public string Name { get; set; } + } + + #endregion + protected TestSqlLoggerFactory TestSqlLoggerFactory => (TestSqlLoggerFactory)ListLoggerFactory; } diff --git a/test/EFCore.Relational.Specification.Tests/Query/JsonQueryFixtureBase.cs b/test/EFCore.Relational.Specification.Tests/Query/JsonQueryFixtureBase.cs index bc11470d327..712eae55d2b 100644 --- a/test/EFCore.Relational.Specification.Tests/Query/JsonQueryFixtureBase.cs +++ b/test/EFCore.Relational.Specification.Tests/Query/JsonQueryFixtureBase.cs @@ -277,6 +277,19 @@ public virtual ISetSource GetExpectedData() } } }, + { + typeof(JsonOwnedAllTypes), (e, a) => + { + Assert.Equal(e == null, a == null); + if (a != null) + { + var ee = (JsonOwnedAllTypes)e; + var aa = (JsonOwnedAllTypes)a; + + AssertAllTypes(ee, aa); + } + } + }, { typeof(JsonEntityConverters), (e, a) => { @@ -292,6 +305,19 @@ public virtual ISetSource GetExpectedData() } } }, + { + typeof(JsonOwnedConverters), (e, a) => + { + Assert.Equal(e == null, a == null); + if (a != null) + { + var ee = (JsonOwnedConverters)e; + var aa = (JsonOwnedConverters)a; + + AssertConverters(ee, aa); + } + } + }, }.ToDictionary(e => e.Key, e => (object)e.Value); private static void AssertOwnedRoot(JsonOwnedRoot expected, JsonOwnedRoot actual) diff --git a/test/EFCore.Relational.Specification.Tests/Query/JsonQueryTestBase.cs b/test/EFCore.Relational.Specification.Tests/Query/JsonQueryTestBase.cs index 20796e4ddc2..5db41d80422 100644 --- a/test/EFCore.Relational.Specification.Tests/Query/JsonQueryTestBase.cs +++ b/test/EFCore.Relational.Specification.Tests/Query/JsonQueryTestBase.cs @@ -21,6 +21,40 @@ public virtual Task Basic_json_projection_owner_entity(bool async) ss => ss.Set(), entryCount: 40); + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Basic_json_projection_owner_entity_NoTracking(bool async) + => AssertQuery( + async, + ss => ss.Set().AsNoTracking()); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Basic_json_projection_owner_entity_duplicated(bool async) + => AssertQuery( + async, + ss => ss.Set().Select(x => new { First = x, Second = x }), + elementSorter: e => e.First.Id, + elementAsserter: (e, a) => + { + AssertEqual(e.First, a.First); + AssertEqual(e.Second, a.Second); + }, + entryCount: 40); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Basic_json_projection_owner_entity_duplicated_NoTracking(bool async) + => AssertQuery( + async, + ss => ss.Set().Select(x => new { First = x, Second = x }).AsNoTracking(), + elementSorter: e => e.First.Id, + elementAsserter: (e, a) => + { + AssertEqual(e.First, a.First); + AssertEqual(e.Second, a.Second); + }); + [ConditionalTheory] [MemberData(nameof(IsAsyncData))] public virtual Task Basic_json_projection_owned_reference_root(bool async) @@ -28,6 +62,30 @@ public virtual Task Basic_json_projection_owned_reference_root(bool async) async, ss => ss.Set().Select(x => x.OwnedReferenceRoot).AsNoTracking()); + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Basic_json_projection_owned_reference_duplicated2(bool async) + => AssertQuery( + async, + ss => ss.Set() + .OrderBy(x => x.Id) + .Select( + x => new + { + Root1 = x.OwnedReferenceRoot, + Leaf1 = x.OwnedReferenceRoot.OwnedReferenceBranch.OwnedReferenceLeaf, + Root2 = x.OwnedReferenceRoot, + Leaf2 = x.OwnedReferenceRoot.OwnedReferenceBranch.OwnedReferenceLeaf, + }).AsNoTracking(), + assertOrder: true, + elementAsserter: (e, a) => + { + AssertEqual(e.Root1, a.Root1); + AssertEqual(e.Root2, a.Root2); + AssertEqual(e.Leaf1, a.Leaf1); + AssertEqual(e.Leaf2, a.Leaf2); + }); + [ConditionalTheory] [MemberData(nameof(IsAsyncData))] public virtual Task Basic_json_projection_owned_reference_duplicated(bool async) @@ -787,6 +845,14 @@ public virtual Task Json_collection_element_access_outside_bounds(bool async) ss => ss.Set().Select(x => x.OwnedCollectionRoot[25]).AsNoTracking(), ss => ss.Set().Select(x => (JsonOwnedRoot)null)); + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Json_collection_element_access_outside_bounds2(bool async) + => AssertQuery( + async, + ss => ss.Set().Select(x => x.OwnedReferenceRoot.OwnedReferenceBranch.OwnedCollectionLeaf[25]).AsNoTracking(), + ss => ss.Set().Select(x => (JsonOwnedLeaf)null)); + [ConditionalTheory] [MemberData(nameof(IsAsyncData))] public virtual Task Json_collection_element_access_outside_bounds_with_property_access(bool async) @@ -1536,6 +1602,15 @@ public virtual Task Json_all_types_entity_projection(bool async) ss => ss.Set(), entryCount: 6); + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Json_all_types_projection_from_owned_entity_reference(bool async) + => AssertQuery( + async, + ss => ss.Set().Select(x => x.Reference).AsNoTracking(), + elementSorter: e => e.TestInt32, + elementAsserter: (e, a) => AssertEqual(e, a)); + [ConditionalTheory] [MemberData(nameof(IsAsyncData))] public virtual Task Json_all_types_projection_individual_properties(bool async) diff --git a/test/EFCore.Relational.Specification.Tests/Update/JsonUpdateTestBase.cs b/test/EFCore.Relational.Specification.Tests/Update/JsonUpdateTestBase.cs index 8a753ae706f..74ba93bd0c1 100644 --- a/test/EFCore.Relational.Specification.Tests/Update/JsonUpdateTestBase.cs +++ b/test/EFCore.Relational.Specification.Tests/Update/JsonUpdateTestBase.cs @@ -60,9 +60,11 @@ public virtual Task Add_entity_with_json() var newEntity = query.Where(e => e.Id == 2).Single(); Assert.Equal("NewEntity", newEntity.Name); + // TODO: #29348 - collection should be empty here Assert.Null(newEntity.OwnedCollectionRoot); Assert.Equal("RootName", newEntity.OwnedReferenceRoot.Name); Assert.Equal(42, newEntity.OwnedReferenceRoot.Number); + // TODO: #29348 - collection should be empty here Assert.Null(newEntity.OwnedReferenceRoot.OwnedCollectionBranch); Assert.Equal(new DateTime(2010, 10, 10), newEntity.OwnedReferenceRoot.OwnedReferenceBranch.Date); Assert.Equal(JsonEnum.Three, newEntity.OwnedReferenceRoot.OwnedReferenceBranch.Enum); @@ -77,6 +79,66 @@ public virtual Task Add_entity_with_json() Assert.Equal("ss2", collectionLeaf[1].SomethingSomething); }); + [ConditionalFact] + public virtual Task Add_entity_with_json_null_navigations() + => TestHelpers.ExecuteWithStrategyInTransactionAsync( + CreateContext, + UseTransaction, + async context => + { + var newEntity = new JsonEntityBasic + { + Id = 2, + Name = "NewEntity", + OwnedCollectionRoot = null, + OwnedReferenceRoot = new JsonOwnedRoot + { + Name = "RootName", + Number = 42, + //OwnedCollectionBranch missing on purpose + OwnedReferenceBranch = new JsonOwnedBranch + { + Date = new DateTime(2010, 10, 10), + Enum = JsonEnum.Three, + Fraction = 42.42m, + OwnedCollectionLeaf = new List + { + new() { SomethingSomething = "ss1" }, new() { SomethingSomething = "ss2" }, + }, + OwnedReferenceLeaf = null, + } + }, + }; + + context.Set().Add(newEntity); + ClearLog(); + await context.SaveChangesAsync(); + }, + async context => + { + var query = await context.JsonEntitiesBasic.ToListAsync(); + Assert.Equal(2, query.Count); + + var newEntity = query.Where(e => e.Id == 2).Single(); + Assert.Equal("NewEntity", newEntity.Name); + Assert.Null(newEntity.OwnedCollectionRoot); + Assert.Equal("RootName", newEntity.OwnedReferenceRoot.Name); + Assert.Equal(42, newEntity.OwnedReferenceRoot.Number); + Assert.Null(newEntity.OwnedReferenceRoot.OwnedCollectionBranch); + Assert.Equal(new DateTime(2010, 10, 10), newEntity.OwnedReferenceRoot.OwnedReferenceBranch.Date); + Assert.Equal(JsonEnum.Three, newEntity.OwnedReferenceRoot.OwnedReferenceBranch.Enum); + Assert.Equal(42.42m, newEntity.OwnedReferenceRoot.OwnedReferenceBranch.Fraction); + + Assert.Equal(42.42m, newEntity.OwnedReferenceRoot.OwnedReferenceBranch.Fraction); + Assert.Null(newEntity.OwnedReferenceRoot.OwnedReferenceBranch.OwnedReferenceLeaf); + + var collectionLeaf = newEntity.OwnedReferenceRoot.OwnedReferenceBranch.OwnedCollectionLeaf; + Assert.Equal(2, collectionLeaf.Count); + Assert.Equal("ss1", collectionLeaf[0].SomethingSomething); + Assert.Equal("ss2", collectionLeaf[1].SomethingSomething); + }); + + [ConditionalFact] public virtual Task Add_json_reference_root() => TestHelpers.ExecuteWithStrategyInTransactionAsync( @@ -121,6 +183,7 @@ public virtual Task Add_json_reference_root() var updatedReference = updatedEntity.OwnedReferenceRoot; Assert.Equal("RootName", updatedReference.Name); Assert.Equal(42, updatedReference.Number); + // TODO: #29348 - collection should be empty here Assert.Null(updatedReference.OwnedCollectionBranch); Assert.Equal(new DateTime(2010, 10, 10), updatedReference.OwnedReferenceBranch.Date); Assert.Equal(JsonEnum.Three, updatedReference.OwnedReferenceBranch.Enum); @@ -202,6 +265,7 @@ public virtual Task Add_element_to_json_collection_root() Assert.Equal(3, updatedCollection.Count); Assert.Equal("new Name", updatedCollection[2].Name); Assert.Equal(142, updatedCollection[2].Number); + // TODO: #29348 - collection should be empty here Assert.Null(updatedCollection[2].OwnedCollectionBranch); Assert.Equal(new DateTime(2010, 10, 10), updatedCollection[2].OwnedReferenceBranch.Date); Assert.Equal(JsonEnum.Three, updatedCollection[2].OwnedReferenceBranch.Enum); @@ -213,6 +277,49 @@ public virtual Task Add_element_to_json_collection_root() Assert.Equal("ss2", collectionLeaf[1].SomethingSomething); }); + [ConditionalFact] + public virtual Task Add_element_to_json_collection_root_null_navigations() + => TestHelpers.ExecuteWithStrategyInTransactionAsync( + CreateContext, + UseTransaction, + async context => + { + var query = await context.JsonEntitiesBasic.ToListAsync(); + var entity = query.Single(); + + var newRoot = new JsonOwnedRoot + { + Name = "new Name", + Number = 142, + OwnedCollectionBranch = null, + OwnedReferenceBranch = new JsonOwnedBranch + { + Date = new DateTime(2010, 10, 10), + Enum = JsonEnum.Three, + Fraction = 42.42m, + OwnedReferenceLeaf = null + } + }; + + entity.OwnedCollectionRoot.Add(newRoot); + ClearLog(); + await context.SaveChangesAsync(); + }, + async context => + { + var updatedEntity = await context.JsonEntitiesBasic.SingleAsync(); + var updatedCollection = updatedEntity.OwnedCollectionRoot; + Assert.Equal(3, updatedCollection.Count); + Assert.Equal("new Name", updatedCollection[2].Name); + Assert.Equal(142, updatedCollection[2].Number); + Assert.Null(updatedCollection[2].OwnedCollectionBranch); + Assert.Equal(new DateTime(2010, 10, 10), updatedCollection[2].OwnedReferenceBranch.Date); + Assert.Equal(JsonEnum.Three, updatedCollection[2].OwnedReferenceBranch.Enum); + Assert.Equal(42.42m, updatedCollection[2].OwnedReferenceBranch.Fraction); + Assert.Null(updatedCollection[2].OwnedReferenceBranch.OwnedReferenceLeaf); + Assert.Null(updatedCollection[2].OwnedReferenceBranch.OwnedCollectionLeaf); + }); + [ConditionalFact] public virtual Task Add_element_to_json_collection_branch() => TestHelpers.ExecuteWithStrategyInTransactionAsync( diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/JsonQueryAdHocSqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/JsonQueryAdHocSqlServerTest.cs index 643d048cd11..31e265bcff9 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/JsonQueryAdHocSqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/JsonQueryAdHocSqlServerTest.cs @@ -117,4 +117,27 @@ protected override void SeedArrayOfPrimitives(MyContextArrayOfPrimitives ctx) ctx.Entities.AddRange(entity1, entity2); ctx.SaveChanges(); } + + protected override void SeedJunkInJson(MyContextJunkInJson ctx) + { + ctx.Database.ExecuteSqlRaw(@"INSERT INTO [Entities] ([Collection], [CollectionWithCtor], [Reference], [ReferenceWithCtor], [Id]) +VALUES( +N'[{{""JunkReference"":{{""Something"":""SomeValue"" }},""Name"":""c11"",""JunkProperty1"":50,""Number"":11.5,""JunkCollection1"":[],""JunkCollection2"":[{{""Foo"":""junk value""}}],""NestedCollection"":[{{""DoB"":""2002-04-01T00:00:00"",""DummyProp"":""Dummy value""}},{{""DoB"":""2002-04-02T00:00:00"",""DummyReference"":{{""Foo"":5}}}}],""NestedReference"":{{""DoB"":""2002-03-01T00:00:00""}}}},{{""Name"":""c12"",""Number"":12.5,""NestedCollection"":[{{""DoB"":""2002-06-01T00:00:00""}},{{""DoB"":""2002-06-02T00:00:00""}}],""NestedDummy"":59,""NestedReference"":{{""DoB"":""2002-05-01T00:00:00""}}}}]', +N'[{{""MyBool"":true,""Name"":""c11 ctor"",""JunkReference"":{{""Something"":""SomeValue"",""JunkCollection"":[{{""Foo"":""junk value""}}]}},""NestedCollection"":[{{""DoB"":""2002-08-01T00:00:00""}},{{""DoB"":""2002-08-02T00:00:00""}}],""NestedReference"":{{""DoB"":""2002-07-01T00:00:00""}}}},{{""MyBool"":false,""Name"":""c12 ctor"",""NestedCollection"":[{{""DoB"":""2002-10-01T00:00:00""}},{{""DoB"":""2002-10-02T00:00:00""}}],""JunkCollection"":[{{""Foo"":""junk value""}}],""NestedReference"":{{""DoB"":""2002-09-01T00:00:00""}}}}]', +N'{{""Name"":""r1"",""JunkCollection"":[{{""Foo"":""junk value""}}],""JunkReference"":{{""Something"":""SomeValue"" }},""Number"":1.5,""NestedCollection"":[{{""DoB"":""2000-02-01T00:00:00"",""JunkReference"":{{""Something"":""SomeValue""}}}},{{""DoB"":""2000-02-02T00:00:00""}}],""NestedReference"":{{""DoB"":""2000-01-01T00:00:00""}}}}', +N'{{""MyBool"":true,""JunkCollection"":[{{""Foo"":""junk value""}}],""Name"":""r1 ctor"",""JunkReference"":{{""Something"":""SomeValue"" }},""NestedCollection"":[{{""DoB"":""2001-02-01T00:00:00""}},{{""DoB"":""2001-02-02T00:00:00""}}],""NestedReference"":{{""JunkCollection"":[{{""Foo"":""junk value""}}],""DoB"":""2001-01-01T00:00:00""}}}}', +1)"); + } + + protected override void SeedShadowProperties(MyContextShadowProperties ctx) + { + ctx.Database.ExecuteSqlRaw(@"INSERT INTO [Entities] ([Collection], [CollectionWithCtor], [Reference], [ReferenceWithCtor], [Id], [Name]) +VALUES( +N'[{{""Name"":""e1_c1"",""ShadowDouble"":5.5}},{{""ShadowDouble"":20.5,""Name"":""e1_c2""}}]', +N'[{{""Name"":""e1_c1 ctor"",""ShadowNullableByte"":6}},{{""ShadowNullableByte"":null,""Name"":""e1_c2 ctor""}}]', +N'{{""Name"":""e1_r"", ""ShadowString"":""Foo""}}', +N'{{""ShadowInt"":143,""Name"":""e1_r ctor""}}', +1, +N'e1')"); + } } diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/JsonQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/JsonQuerySqlServerTest.cs index f057572bd58..ec09065229e 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/JsonQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/JsonQuerySqlServerTest.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json; using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore.TestModels.JsonQuery; @@ -26,6 +27,39 @@ FROM [JsonEntitiesBasic] AS [j] """); } + public override async Task Basic_json_projection_owner_entity_NoTracking(bool async) + { + await base.Basic_json_projection_owner_entity_NoTracking(async); + + AssertSql( +""" +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot] +FROM [JsonEntitiesBasic] AS [j] +"""); + } + + public override async Task Basic_json_projection_owner_entity_duplicated(bool async) + { + await base.Basic_json_projection_owner_entity_duplicated(async); + + AssertSql( +""" +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot] +FROM [JsonEntitiesBasic] AS [j] +"""); + } + + public override async Task Basic_json_projection_owner_entity_duplicated_NoTracking(bool async) + { + await base.Basic_json_projection_owner_entity_duplicated_NoTracking(async); + + AssertSql( +""" +SELECT [j].[Id], [j].[Name], [j].[OwnedCollection], [j].[OwnedCollection] +FROM [JsonEntitiesSingleOwned] AS [j] +"""); + } + public override async Task Basic_json_projection_owned_reference_root(bool async) { await base.Basic_json_projection_owned_reference_root(async); @@ -37,13 +71,25 @@ FROM [JsonEntitiesBasic] AS [j] """); } + public override async Task Basic_json_projection_owned_reference_duplicated2(bool async) + { + await base.Basic_json_projection_owned_reference_duplicated2(async); + + AssertSql( +""" +SELECT [j].[OwnedReferenceRoot], [j].[Id], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf'), [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf') +FROM [JsonEntitiesBasic] AS [j] +ORDER BY [j].[Id] +"""); + } + public override async Task Basic_json_projection_owned_reference_duplicated(bool async) { await base.Basic_json_projection_owned_reference_duplicated(async); AssertSql( """ -SELECT [j].[OwnedReferenceRoot], [j].[Id] +SELECT [j].[OwnedReferenceRoot], [j].[Id], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch'), [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch') FROM [JsonEntitiesBasic] AS [j] ORDER BY [j].[Id] """); @@ -155,7 +201,7 @@ public override async Task Json_projection_with_deduplication(bool async) AssertSql( """ -SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], JSON_VALUE([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf.SomethingSomething') +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch'), JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf'), JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedCollectionLeaf'), JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedCollectionBranch'), JSON_VALUE([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf.SomethingSomething') FROM [JsonEntitiesBasic] AS [j] """); } @@ -166,7 +212,7 @@ public override async Task Json_projection_with_deduplication_reverse_order(bool AssertSql( """ -SELECT [j].[OwnedReferenceRoot], [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot] +SELECT JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf'), [j].[Id], [j].[OwnedReferenceRoot], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot] FROM [JsonEntitiesBasic] AS [j] """); } @@ -395,7 +441,7 @@ public override async Task Custom_naming_projection_everything(bool async) AssertSql( """ -SELECT [j].[Id], [j].[Title], [j].[json_collection_custom_naming], [j].[json_reference_custom_naming], JSON_VALUE([j].[json_reference_custom_naming], '$.CustomName'), CAST(JSON_VALUE([j].[json_reference_custom_naming], '$.CustomOwnedReferenceBranch.CustomFraction') AS float) +SELECT [j].[Id], [j].[Title], [j].[json_collection_custom_naming], [j].[json_reference_custom_naming], [j].[json_reference_custom_naming], JSON_QUERY([j].[json_reference_custom_naming], '$.CustomOwnedReferenceBranch'), [j].[json_collection_custom_naming], JSON_QUERY([j].[json_reference_custom_naming], '$.CustomOwnedCollectionBranch'), JSON_VALUE([j].[json_reference_custom_naming], '$.CustomName'), CAST(JSON_VALUE([j].[json_reference_custom_naming], '$.CustomOwnedReferenceBranch.CustomFraction') AS float) FROM [JsonEntitiesCustomNaming] AS [j] """); } @@ -429,7 +475,7 @@ public override async Task Left_join_json_entities_complex_projection(bool async AssertSql( """ -SELECT [j].[Id], [j0].[Id], [j0].[EntityBasicId], [j0].[Name], [j0].[OwnedCollectionRoot], [j0].[OwnedReferenceRoot] +SELECT [j].[Id], [j0].[Id], [j0].[EntityBasicId], [j0].[Name], [j0].[OwnedCollectionRoot], [j0].[OwnedReferenceRoot], [j0].[OwnedReferenceRoot], JSON_QUERY([j0].[OwnedReferenceRoot], '$.OwnedReferenceBranch'), JSON_QUERY([j0].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf'), JSON_QUERY([j0].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedCollectionLeaf') FROM [JsonEntitiesSingleOwned] AS [j] LEFT JOIN [JsonEntitiesBasic] AS [j0] ON [j].[Id] = [j0].[Id] """); @@ -453,7 +499,7 @@ public override async Task Left_join_json_entities_complex_projection_json_being AssertSql( """ -SELECT [j].[Id], [j0].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], [j0].[Name], [j0].[OwnedCollection] +SELECT [j].[Id], [j0].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch'), JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf'), JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedCollectionLeaf'), [j0].[Name], [j0].[OwnedCollection] FROM [JsonEntitiesBasic] AS [j] LEFT JOIN [JsonEntitiesSingleOwned] AS [j0] ON [j].[Id] = [j0].[Id] """); @@ -590,7 +636,7 @@ public override async Task Json_entity_with_inheritance_project_navigations_on_d AssertSql( """ -SELECT [j].[Id], [j].[Discriminator], [j].[Name], [j].[Fraction], [j].[CollectionOnBase], [j].[ReferenceOnBase], [j].[CollectionOnDerived], [j].[ReferenceOnDerived] +SELECT [j].[Id], [j].[Discriminator], [j].[Name], [j].[Fraction], [j].[CollectionOnBase], [j].[ReferenceOnBase], [j].[CollectionOnDerived], [j].[ReferenceOnDerived], [j].[ReferenceOnBase], [j].[ReferenceOnDerived], [j].[CollectionOnBase], [j].[CollectionOnDerived] FROM [JsonEntitiesInheritance] AS [j] WHERE [j].[Discriminator] = N'JsonEntityInheritanceDerived' """); @@ -721,6 +767,17 @@ FROM [JsonEntitiesBasic] AS [j] """); } + public override async Task Json_collection_element_access_outside_bounds2(bool async) + { + await base.Json_collection_element_access_outside_bounds2(async); + + AssertSql( +""" +SELECT JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedCollectionLeaf[25]'), [j].[Id] +FROM [JsonEntitiesBasic] AS [j] +"""); + } + public override async Task Json_collection_element_access_outside_bounds_with_property_access(bool async) { await base.Json_collection_element_access_outside_bounds_with_property_access(async); @@ -961,7 +1018,7 @@ public override async Task Json_projection_deduplication_with_collection_indexer AssertSql( """ -SELECT [j].[Id], JSON_QUERY([j].[OwnedCollectionRoot], '$[0]') +SELECT [j].[Id], JSON_QUERY([j].[OwnedCollectionRoot], '$[0].OwnedReferenceBranch'), JSON_QUERY([j].[OwnedCollectionRoot], '$[0]'), JSON_QUERY([j].[OwnedCollectionRoot], '$[0].OwnedReferenceBranch.OwnedCollectionLeaf') FROM [JsonEntitiesBasic] AS [j] """); } @@ -975,11 +1032,12 @@ public override async Task Json_projection_deduplication_with_collection_indexer """ @__prm_0='1' -SELECT [j].[Id], [j].[OwnedReferenceRoot], @__prm_0 +SELECT [j].[Id], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedCollectionBranch[1]'), [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedCollectionLeaf[' + CAST(@__prm_0 AS nvarchar(max)) + ']'), @__prm_0 FROM [JsonEntitiesBasic] AS [j] """); } + [SqlServerCondition(SqlServerCondition.SupportsJsonPathExpressions)] public override async Task Json_projection_deduplication_with_collection_in_original_and_collection_indexer_in_target(bool async) { await base.Json_projection_deduplication_with_collection_in_original_and_collection_indexer_in_target(async); @@ -988,7 +1046,7 @@ public override async Task Json_projection_deduplication_with_collection_in_orig """ @__prm_0='1' -SELECT JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedCollectionBranch'), [j].[Id], @__prm_0 +SELECT JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedCollectionBranch[0].OwnedCollectionLeaf[' + CAST(@__prm_0 AS nvarchar(max)) + ']'), [j].[Id], @__prm_0, JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedCollectionBranch[' + CAST(@__prm_0 AS nvarchar(max)) + ']'), JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedCollectionBranch'), JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedCollectionBranch[0]') FROM [JsonEntitiesBasic] AS [j] """); } @@ -999,11 +1057,12 @@ public override async Task Json_collection_element_access_in_projection_using_co AssertSql( """ -SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot] +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedCollectionRoot], '$[1]') FROM [JsonEntitiesBasic] AS [j] """); } + [SqlServerCondition(SqlServerCondition.SupportsJsonPathExpressions)] public override async Task Json_collection_element_access_in_projection_using_parameter_when_owner_is_present(bool async) { await base.Json_collection_element_access_in_projection_using_parameter_when_owner_is_present(async); @@ -1012,7 +1071,7 @@ public override async Task Json_collection_element_access_in_projection_using_pa """ @__prm_0='1' -SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], @__prm_0 +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedCollectionRoot], '$[' + CAST(@__prm_0 AS nvarchar(max)) + ']'), @__prm_0 FROM [JsonEntitiesBasic] AS [j] """); } @@ -1023,11 +1082,12 @@ public override async Task Json_collection_after_collection_element_access_in_pr AssertSql( """ -SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot] +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedCollectionRoot], '$[1].OwnedCollectionBranch') FROM [JsonEntitiesBasic] AS [j] """); } + [SqlServerCondition(SqlServerCondition.SupportsJsonPathExpressions)] public override async Task Json_collection_after_collection_element_access_in_projection_using_parameter_when_owner_is_present(bool async) { await base.Json_collection_after_collection_element_access_in_projection_using_parameter_when_owner_is_present(async); @@ -1036,11 +1096,12 @@ public override async Task Json_collection_after_collection_element_access_in_pr """ @__prm_0='1' -SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], @__prm_0 +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedCollectionRoot], '$[' + CAST(@__prm_0 AS nvarchar(max)) + '].OwnedCollectionBranch'), @__prm_0 FROM [JsonEntitiesBasic] AS [j] """); } + [SqlServerCondition(SqlServerCondition.SupportsJsonPathExpressions)] public override async Task Json_collection_element_access_in_projection_when_owner_is_present_misc1(bool async) { await base.Json_collection_element_access_in_projection_when_owner_is_present_misc1(async); @@ -1049,7 +1110,7 @@ public override async Task Json_collection_element_access_in_projection_when_own """ @__prm_0='1' -SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], @__prm_0 +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedCollectionRoot], '$[1].OwnedCollectionBranch[' + CAST(@__prm_0 AS nvarchar(max)) + ']'), @__prm_0 FROM [JsonEntitiesBasic] AS [j] """); } @@ -1060,11 +1121,12 @@ public override async Task Json_collection_element_access_in_projection_when_own AssertSql( """ -SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot] +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedCollectionLeaf[1]') FROM [JsonEntitiesBasic] AS [j] """); } + [SqlServerCondition(SqlServerCondition.SupportsJsonPathExpressions)] public override async Task Json_collection_element_access_in_projection_when_owner_is_present_multiple(bool async) { await base.Json_collection_element_access_in_projection_when_owner_is_present_multiple(async); @@ -1073,7 +1135,7 @@ public override async Task Json_collection_element_access_in_projection_when_own """ @__prm_0='1' -SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], @__prm_0 +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], JSON_QUERY([j].[OwnedCollectionRoot], '$[' + CAST(@__prm_0 AS nvarchar(max)) + '].OwnedCollectionBranch[1]'), @__prm_0, JSON_QUERY([j].[OwnedCollectionRoot], '$[1].OwnedCollectionBranch[1].OwnedReferenceLeaf'), JSON_QUERY([j].[OwnedCollectionRoot], '$[1].OwnedReferenceBranch'), JSON_QUERY([j].[OwnedCollectionRoot], '$[' + CAST(@__prm_0 AS nvarchar(max)) + '].OwnedReferenceBranch'), JSON_QUERY([j].[OwnedCollectionRoot], '$[' + CAST(@__prm_0 AS nvarchar(max)) + '].OwnedCollectionBranch[' + CAST([j].[Id] AS nvarchar(max)) + ']'), JSON_QUERY([j].[OwnedCollectionRoot], '$[' + CAST([j].[Id] AS nvarchar(max)) + '].OwnedCollectionBranch[1].OwnedReferenceLeaf'), JSON_QUERY([j].[OwnedCollectionRoot], '$[1].OwnedReferenceBranch'), JSON_QUERY([j].[OwnedCollectionRoot], '$[' + CAST([j].[Id] AS nvarchar(max)) + '].OwnedReferenceBranch'), JSON_QUERY([j].[OwnedCollectionRoot], '$[' + CAST([j].[Id] AS nvarchar(max)) + '].OwnedCollectionBranch[' + CAST([j].[Id] AS nvarchar(max)) + ']') FROM [JsonEntitiesBasic] AS [j] """); } @@ -1397,12 +1459,21 @@ public override async Task Json_with_projection_of_mix_of_json_collections_json_ AssertSql( """ -SELECT JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedCollectionLeaf'), [j].[Id], [j0].[Id], [j0].[Name], [j0].[ParentId], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf'), [j1].[Id], [j1].[Name], [j1].[ParentId], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedCollectionBranch'), [j].[OwnedCollectionRoot] +SELECT JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedCollectionLeaf'), [j].[Id], [j0].[Id], [j0].[Name], [j0].[ParentId], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf'), [j1].[Id], [j1].[Name], [j1].[ParentId], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedCollectionLeaf[0]'), JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedCollectionBranch'), [j].[OwnedCollectionRoot], JSON_QUERY([j].[OwnedCollectionRoot], '$[0].OwnedReferenceBranch'), JSON_QUERY([j].[OwnedCollectionRoot], '$[0].OwnedCollectionBranch') FROM [JsonEntitiesBasic] AS [j] LEFT JOIN [JsonEntitiesBasicForReference] AS [j0] ON [j].[Id] = [j0].[ParentId] LEFT JOIN [JsonEntitiesBasicForCollection] AS [j1] ON [j].[Id] = [j1].[ParentId] ORDER BY [j].[Id], [j0].[Id] """); + +// AssertSql( +//""" +//SELECT JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedCollectionLeaf'), [j].[Id], [j0].[Id], [j0].[Name], [j0].[ParentId], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedReferenceBranch.OwnedReferenceLeaf'), [j1].[Id], [j1].[Name], [j1].[ParentId], JSON_QUERY([j].[OwnedReferenceRoot], '$.OwnedCollectionBranch'), [j].[OwnedCollectionRoot] +//FROM [JsonEntitiesBasic] AS [j] +//LEFT JOIN [JsonEntitiesBasicForReference] AS [j0] ON [j].[Id] = [j0].[ParentId] +//LEFT JOIN [JsonEntitiesBasicForCollection] AS [j1] ON [j].[Id] = [j1].[ParentId] +//ORDER BY [j].[Id], [j0].[Id] +//"""); } public override async Task Json_all_types_entity_projection(bool async) @@ -1416,6 +1487,17 @@ FROM [JsonEntitiesAllTypes] AS [j] """); } + public override async Task Json_all_types_projection_from_owned_entity_reference(bool async) + { + await base.Json_all_types_projection_from_owned_entity_reference(async); + + AssertSql( +""" +SELECT [j].[Reference], [j].[Id] +FROM [JsonEntitiesAllTypes] AS [j] +"""); + } + public override async Task Json_all_types_projection_individual_properties(bool async) { await base.Json_all_types_projection_individual_properties(async); diff --git a/test/EFCore.SqlServer.FunctionalTests/Update/JsonUpdateSqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Update/JsonUpdateSqlServerTest.cs index fa14c837705..d4a5c348d7c 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Update/JsonUpdateSqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Update/JsonUpdateSqlServerTest.cs @@ -87,6 +87,28 @@ public override async Task Add_element_to_json_collection_root() @p0='[{"Name":"e1_c1","Number":11,"OwnedCollectionBranch":[{"Date":"2111-01-01T00:00:00","Enum":"Two","Fraction":11.1,"NullableEnum":"One","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c1_c1_c1"},{"SomethingSomething":"e1_c1_c1_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c1_c1_r"}},{"Date":"2112-01-01T00:00:00","Enum":"Three","Fraction":11.2,"NullableEnum":"Two","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c1_c2_c1"},{"SomethingSomething":"e1_c1_c2_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c1_c2_r"}}],"OwnedReferenceBranch":{"Date":"2110-01-01T00:00:00","Enum":"One","Fraction":11.0,"NullableEnum":null,"OwnedCollectionLeaf":[{"SomethingSomething":"e1_c1_r_c1"},{"SomethingSomething":"e1_c1_r_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c1_r_r"}}},{"Name":"e1_c2","Number":12,"OwnedCollectionBranch":[{"Date":"2121-01-01T00:00:00","Enum":"Two","Fraction":12.1,"NullableEnum":"One","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c2_c1_c1"},{"SomethingSomething":"e1_c2_c1_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c2_c1_r"}},{"Date":"2122-01-01T00:00:00","Enum":"One","Fraction":12.2,"NullableEnum":null,"OwnedCollectionLeaf":[{"SomethingSomething":"e1_c2_c2_c1"},{"SomethingSomething":"e1_c2_c2_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c2_c2_r"}}],"OwnedReferenceBranch":{"Date":"2120-01-01T00:00:00","Enum":"Three","Fraction":12.0,"NullableEnum":"Two","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c2_r_c1"},{"SomethingSomething":"e1_c2_r_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c2_r_r"}}},{"Name":"new Name","Number":142,"OwnedCollectionBranch":[],"OwnedReferenceBranch":{"Date":"2010-10-10T00:00:00","Enum":"Three","Fraction":42.42,"NullableEnum":null,"OwnedCollectionLeaf":[{"SomethingSomething":"ss1"},{"SomethingSomething":"ss2"}],"OwnedReferenceLeaf":{"SomethingSomething":"ss3"}}}]' (Nullable = false) (Size = 1867) @p1='1' +SET IMPLICIT_TRANSACTIONS OFF; +SET NOCOUNT ON; +UPDATE [JsonEntitiesBasic] SET [OwnedCollectionRoot] = @p0 +OUTPUT 1 +WHERE [Id] = @p1; +""", + // +""" +SELECT TOP(2) [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot] +FROM [JsonEntitiesBasic] AS [j] +"""); + } + + public override async Task Add_element_to_json_collection_root_null_navigations() + { + await base.Add_element_to_json_collection_root_null_navigations(); + + AssertSql( +""" +@p0='[{"Name":"e1_c1","Number":11,"OwnedCollectionBranch":[{"Date":"2111-01-01T00:00:00","Enum":"Two","Fraction":11.1,"NullableEnum":"One","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c1_c1_c1"},{"SomethingSomething":"e1_c1_c1_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c1_c1_r"}},{"Date":"2112-01-01T00:00:00","Enum":"Three","Fraction":11.2,"NullableEnum":"Two","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c1_c2_c1"},{"SomethingSomething":"e1_c1_c2_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c1_c2_r"}}],"OwnedReferenceBranch":{"Date":"2110-01-01T00:00:00","Enum":"One","Fraction":11.0,"NullableEnum":null,"OwnedCollectionLeaf":[{"SomethingSomething":"e1_c1_r_c1"},{"SomethingSomething":"e1_c1_r_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c1_r_r"}}},{"Name":"e1_c2","Number":12,"OwnedCollectionBranch":[{"Date":"2121-01-01T00:00:00","Enum":"Two","Fraction":12.1,"NullableEnum":"One","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c2_c1_c1"},{"SomethingSomething":"e1_c2_c1_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c2_c1_r"}},{"Date":"2122-01-01T00:00:00","Enum":"One","Fraction":12.2,"NullableEnum":null,"OwnedCollectionLeaf":[{"SomethingSomething":"e1_c2_c2_c1"},{"SomethingSomething":"e1_c2_c2_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c2_c2_r"}}],"OwnedReferenceBranch":{"Date":"2120-01-01T00:00:00","Enum":"Three","Fraction":12.0,"NullableEnum":"Two","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c2_r_c1"},{"SomethingSomething":"e1_c2_r_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c2_r_r"}}},{"Name":"new Name","Number":142,"OwnedCollectionBranch":null,"OwnedReferenceBranch":{"Date":"2010-10-10T00:00:00","Enum":"Three","Fraction":42.42,"NullableEnum":null,"OwnedCollectionLeaf":null,"OwnedReferenceLeaf":null}}]' (Nullable = false) (Size = 1790) +@p1='1' + SET IMPLICIT_TRANSACTIONS OFF; SET NOCOUNT ON; UPDATE [JsonEntitiesBasic] SET [OwnedCollectionRoot] = @p0 @@ -111,6 +133,29 @@ public override async Task Add_entity_with_json() @p2=NULL (DbType = Int32) @p3='NewEntity' (Size = 4000) +SET IMPLICIT_TRANSACTIONS OFF; +SET NOCOUNT ON; +INSERT INTO [JsonEntitiesBasic] ([OwnedReferenceRoot], [Id], [EntityBasicId], [Name]) +VALUES (@p0, @p1, @p2, @p3); +""", + // +""" +SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot] +FROM [JsonEntitiesBasic] AS [j] +"""); + } + + public override async Task Add_entity_with_json_null_navigations() + { + await base.Add_entity_with_json_null_navigations(); + + AssertSql( +""" +@p0='{"Name":"RootName","Number":42,"OwnedCollectionBranch":null,"OwnedReferenceBranch":{"Date":"2010-10-10T00:00:00","Enum":"Three","Fraction":42.42,"NullableEnum":null,"OwnedCollectionLeaf":[{"SomethingSomething":"ss1"},{"SomethingSomething":"ss2"}],"OwnedReferenceLeaf":null}}' (Nullable = false) (Size = 274) +@p1='2' +@p2=NULL (DbType = Int32) +@p3='NewEntity' (Size = 4000) + SET IMPLICIT_TRANSACTIONS OFF; SET NOCOUNT ON; INSERT INTO [JsonEntitiesBasic] ([OwnedReferenceRoot], [Id], [EntityBasicId], [Name]) @@ -194,7 +239,7 @@ public override async Task Delete_json_collection_branch() AssertSql( """ -@p0='[]' (Nullable = false) (Size = 2) +@p0=NULL (Nullable = false) @p1='1' SET IMPLICIT_TRANSACTIONS OFF; @@ -216,7 +261,7 @@ public override async Task Delete_json_collection_root() AssertSql( """ -@p0='[]' (Nullable = false) (Size = 2) +@p0=NULL (Nullable = false) @p1='1' SET IMPLICIT_TRANSACTIONS OFF; @@ -371,7 +416,7 @@ public override async Task Edit_element_in_json_branch_collection_and_add_elemen AssertSql( """ -@p0='[{"Date":"2101-01-01T00:00:00","Enum":"Two","Fraction":4321.3,"NullableEnum":"One","OwnedCollectionLeaf":[{"SomethingSomething":"e1_r_c1_c1"},{"SomethingSomething":"e1_r_c1_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_r_c1_r"}},{"Date":"2102-01-01T00:00:00","Enum":"Three","Fraction":10.2,"NullableEnum":"Two","OwnedCollectionLeaf":[{"SomethingSomething":"e1_r_c2_c1"},{"SomethingSomething":"e1_r_c2_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_r_c2_r"}},{"Date":"2222-11-11T00:00:00","Enum":"Three","Fraction":45.32,"NullableEnum":null,"OwnedCollectionLeaf":[],"OwnedReferenceLeaf":{"SomethingSomething":"cc"}}]' (Nullable = false) (Size = 628) +@p0='[{"Date":"2101-01-01T00:00:00","Enum":"Two","Fraction":4321.3,"NullableEnum":"One","OwnedCollectionLeaf":[{"SomethingSomething":"e1_r_c1_c1"},{"SomethingSomething":"e1_r_c1_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_r_c1_r"}},{"Date":"2102-01-01T00:00:00","Enum":"Three","Fraction":10.2,"NullableEnum":"Two","OwnedCollectionLeaf":[{"SomethingSomething":"e1_r_c2_c1"},{"SomethingSomething":"e1_r_c2_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_r_c2_r"}},{"Date":"2222-11-11T00:00:00","Enum":"Three","Fraction":45.32,"NullableEnum":null,"OwnedCollectionLeaf":null,"OwnedReferenceLeaf":{"SomethingSomething":"cc"}}]' (Nullable = false) (Size = 630) @p1='1' SET IMPLICIT_TRANSACTIONS OFF; @@ -1152,8 +1197,8 @@ public override async Task Edit_two_properties_on_same_entity_updates_the_entire AssertSql( """ -@p0='{"TestBoolean":false,"TestByte":25,"TestCharacter":"h","TestDateTime":"2100-11-11T12:34:56","TestDateTimeOffset":"2200-11-11T12:34:56-05:00","TestDecimal":-123450.01,"TestDefaultString":"MyDefaultStringInCollection1","TestDouble":-1.2345,"TestEnum":"One","TestEnumWithIntConverter":1,"TestGuid":"00000000-0000-0000-0000-000000000000","TestInt16":-12,"TestInt32":32,"TestInt64":64,"TestMaxLengthString":"Baz","TestNullableEnum":"One","TestNullableEnumWithConverterThatHandlesNulls":"Two","TestNullableEnumWithIntConverter":2,"TestNullableInt32":90,"TestSignedByte":-18,"TestSingle":-1.4,"TestTimeSpan":"06:05:04.0030000","TestUnsignedInt16":12,"TestUnsignedInt32":12345,"TestUnsignedInt64":1234567867}' (Nullable = false) (Size = 700) -@p1='{"TestBoolean":true,"TestByte":255,"TestCharacter":"a","TestDateTime":"2000-01-01T12:34:56","TestDateTimeOffset":"2000-01-01T12:34:56-08:00","TestDecimal":-1234567890.01,"TestDefaultString":"MyDefaultStringInReference1","TestDouble":-1.23456789,"TestEnum":"One","TestEnumWithIntConverter":1,"TestGuid":"12345678-1234-4321-7777-987654321000","TestInt16":-1234,"TestInt32":32,"TestInt64":64,"TestMaxLengthString":"Foo","TestNullableEnum":"One","TestNullableEnumWithConverterThatHandlesNulls":"Three","TestNullableEnumWithIntConverter":1,"TestNullableInt32":78,"TestSignedByte":-128,"TestSingle":-1.234,"TestTimeSpan":"10:09:08.0070000","TestUnsignedInt16":1234,"TestUnsignedInt32":1234565789,"TestUnsignedInt64":1234567890123456789}' (Nullable = false) (Size = 730) +@p0='{"TestBoolean":false,"TestByte":25,"TestCharacter":"h","TestDateTime":"2100-11-11T12:34:56","TestDateTimeOffset":"2200-11-11T12:34:56-05:00","TestDecimal":-123450.01,"TestDefaultString":"MyDefaultStringInCollection1","TestDouble":-1.2345,"TestEnum":"One","TestEnumWithIntConverter":1,"TestGuid":"00000000-0000-0000-0000-000000000000","TestInt16":-12,"TestInt32":32,"TestInt64":64,"TestMaxLengthString":"Baz","TestNullableEnum":"One","TestNullableEnumWithConverterThatHandlesNulls":"Two","TestNullableEnumWithIntConverter":2,"TestNullableInt32":90,"TestSignedByte":-18,"TestSingle":-1.4,"TestTimeSpan":"6:05:04.003","TestUnsignedInt16":12,"TestUnsignedInt32":12345,"TestUnsignedInt64":1234567867}' (Nullable = false) (Size = 695) +@p1='{"TestBoolean":true,"TestByte":255,"TestCharacter":"a","TestDateTime":"2000-01-01T12:34:56","TestDateTimeOffset":"2000-01-01T12:34:56-08:00","TestDecimal":-1234567890.01,"TestDefaultString":"MyDefaultStringInReference1","TestDouble":-1.23456789,"TestEnum":"One","TestEnumWithIntConverter":1,"TestGuid":"12345678-1234-4321-7777-987654321000","TestInt16":-1234,"TestInt32":32,"TestInt64":64,"TestMaxLengthString":"Foo","TestNullableEnum":"One","TestNullableEnumWithConverterThatHandlesNulls":"Three","TestNullableEnumWithIntConverter":1,"TestNullableInt32":78,"TestSignedByte":-128,"TestSingle":-1.234,"TestTimeSpan":"10:09:08.007","TestUnsignedInt16":1234,"TestUnsignedInt32":1234565789,"TestUnsignedInt64":1234567890123456789}' (Nullable = false) (Size = 726) @p2='1' SET IMPLICIT_TRANSACTIONS OFF; @@ -1162,8 +1207,8 @@ public override async Task Edit_two_properties_on_same_entity_updates_the_entire OUTPUT 1 WHERE [Id] = @p2; """, - // -""" + // + """ SELECT TOP(2) [j].[Id], [j].[Collection], [j].[Reference] FROM [JsonEntitiesAllTypes] AS [j] WHERE [j].[Id] = 1 diff --git a/test/EFCore.Sqlite.FunctionalTests/Query/JsonQueryAdHocSqliteTest.cs b/test/EFCore.Sqlite.FunctionalTests/Query/JsonQueryAdHocSqliteTest.cs index 88033c590dd..aed0c50aa40 100644 --- a/test/EFCore.Sqlite.FunctionalTests/Query/JsonQueryAdHocSqliteTest.cs +++ b/test/EFCore.Sqlite.FunctionalTests/Query/JsonQueryAdHocSqliteTest.cs @@ -117,4 +117,27 @@ protected override void SeedArrayOfPrimitives(MyContextArrayOfPrimitives ctx) ctx.Entities.AddRange(entity1, entity2); ctx.SaveChanges(); } + + protected override void SeedJunkInJson(MyContextJunkInJson ctx) + { + ctx.Database.ExecuteSqlRaw(@"INSERT INTO ""Entities"" (""Collection"", ""CollectionWithCtor"", ""Reference"", ""ReferenceWithCtor"", ""Id"") +VALUES( +'[{{""JunkReference"":{{""Something"":""SomeValue"" }},""Name"":""c11"",""JunkProperty1"":50,""Number"":11.5,""JunkCollection1"":[],""JunkCollection2"":[{{""Foo"":""junk value""}}],""NestedCollection"":[{{""DoB"":""2002-04-01T00:00:00"",""DummyProp"":""Dummy value""}},{{""DoB"":""2002-04-02T00:00:00"",""DummyReference"":{{""Foo"":5}}}}],""NestedReference"":{{""DoB"":""2002-03-01T00:00:00""}}}},{{""Name"":""c12"",""Number"":12.5,""NestedCollection"":[{{""DoB"":""2002-06-01T00:00:00""}},{{""DoB"":""2002-06-02T00:00:00""}}],""NestedDummy"":59,""NestedReference"":{{""DoB"":""2002-05-01T00:00:00""}}}}]', +'[{{""MyBool"":true,""Name"":""c11 ctor"",""JunkReference"":{{""Something"":""SomeValue"",""JunkCollection"":[{{""Foo"":""junk value""}}]}},""NestedCollection"":[{{""DoB"":""2002-08-01T00:00:00""}},{{""DoB"":""2002-08-02T00:00:00""}}],""NestedReference"":{{""DoB"":""2002-07-01T00:00:00""}}}},{{""MyBool"":false,""Name"":""c12 ctor"",""NestedCollection"":[{{""DoB"":""2002-10-01T00:00:00""}},{{""DoB"":""2002-10-02T00:00:00""}}],""JunkCollection"":[{{""Foo"":""junk value""}}],""NestedReference"":{{""DoB"":""2002-09-01T00:00:00""}}}}]', +'{{""Name"":""r1"",""JunkCollection"":[{{""Foo"":""junk value""}}],""JunkReference"":{{""Something"":""SomeValue"" }},""Number"":1.5,""NestedCollection"":[{{""DoB"":""2000-02-01T00:00:00"",""JunkReference"":{{""Something"":""SomeValue""}}}},{{""DoB"":""2000-02-02T00:00:00""}}],""NestedReference"":{{""DoB"":""2000-01-01T00:00:00""}}}}', +'{{""MyBool"":true,""JunkCollection"":[{{""Foo"":""junk value""}}],""Name"":""r1 ctor"",""JunkReference"":{{""Something"":""SomeValue"" }},""NestedCollection"":[{{""DoB"":""2001-02-01T00:00:00""}},{{""DoB"":""2001-02-02T00:00:00""}}],""NestedReference"":{{""JunkCollection"":[{{""Foo"":""junk value""}}],""DoB"":""2001-01-01T00:00:00""}}}}', +1)"); + } + + protected override void SeedShadowProperties(MyContextShadowProperties ctx) + { + ctx.Database.ExecuteSqlRaw(@"INSERT INTO ""Entities"" (""Collection"", ""CollectionWithCtor"", ""Reference"", ""ReferenceWithCtor"", ""Id"", ""Name"") +VALUES( +'[{{""Name"":""e1_c1"",""ShadowDouble"":5.5}},{{""ShadowDouble"":20.5,""Name"":""e1_c2""}}]', +'[{{""Name"":""e1_c1 ctor"",""ShadowNullableByte"":6}},{{""ShadowNullableByte"":null,""Name"":""e1_c2 ctor""}}]', +'{{""Name"":""e1_r"", ""ShadowString"":""Foo""}}', +'{{""ShadowInt"":143,""Name"":""e1_r ctor""}}', +1, +'e1')"); + } } diff --git a/test/EFCore.Sqlite.FunctionalTests/Update/JsonUpdateSqliteTest.cs b/test/EFCore.Sqlite.FunctionalTests/Update/JsonUpdateSqliteTest.cs index aa4e3397ae8..0e3b99fffd9 100644 --- a/test/EFCore.Sqlite.FunctionalTests/Update/JsonUpdateSqliteTest.cs +++ b/test/EFCore.Sqlite.FunctionalTests/Update/JsonUpdateSqliteTest.cs @@ -96,6 +96,27 @@ LIMIT 2 """); } + public override async Task Add_element_to_json_collection_root_null_navigations() + { + await base.Add_element_to_json_collection_root_null_navigations(); + + AssertSql( +""" +@p0='[{"Name":"e1_c1","Number":11,"OwnedCollectionBranch":[{"Date":"2111-01-01T00:00:00","Enum":"Two","Fraction":11.1,"NullableEnum":"One","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c1_c1_c1"},{"SomethingSomething":"e1_c1_c1_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c1_c1_r"}},{"Date":"2112-01-01T00:00:00","Enum":"Three","Fraction":11.2,"NullableEnum":"Two","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c1_c2_c1"},{"SomethingSomething":"e1_c1_c2_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c1_c2_r"}}],"OwnedReferenceBranch":{"Date":"2110-01-01T00:00:00","Enum":"One","Fraction":11.0,"NullableEnum":null,"OwnedCollectionLeaf":[{"SomethingSomething":"e1_c1_r_c1"},{"SomethingSomething":"e1_c1_r_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c1_r_r"}}},{"Name":"e1_c2","Number":12,"OwnedCollectionBranch":[{"Date":"2121-01-01T00:00:00","Enum":"Two","Fraction":12.1,"NullableEnum":"One","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c2_c1_c1"},{"SomethingSomething":"e1_c2_c1_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c2_c1_r"}},{"Date":"2122-01-01T00:00:00","Enum":"One","Fraction":12.2,"NullableEnum":null,"OwnedCollectionLeaf":[{"SomethingSomething":"e1_c2_c2_c1"},{"SomethingSomething":"e1_c2_c2_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c2_c2_r"}}],"OwnedReferenceBranch":{"Date":"2120-01-01T00:00:00","Enum":"Three","Fraction":12.0,"NullableEnum":"Two","OwnedCollectionLeaf":[{"SomethingSomething":"e1_c2_r_c1"},{"SomethingSomething":"e1_c2_r_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_c2_r_r"}}},{"Name":"new Name","Number":142,"OwnedCollectionBranch":null,"OwnedReferenceBranch":{"Date":"2010-10-10T00:00:00","Enum":"Three","Fraction":42.42,"NullableEnum":null,"OwnedCollectionLeaf":null,"OwnedReferenceLeaf":null}}]' (Nullable = false) (Size = 1790) +@p1='1' + +UPDATE "JsonEntitiesBasic" SET "OwnedCollectionRoot" = @p0 +WHERE "Id" = @p1 +RETURNING 1; +""", + // + """ +SELECT "j"."Id", "j"."EntityBasicId", "j"."Name", "j"."OwnedCollectionRoot", "j"."OwnedReferenceRoot" +FROM "JsonEntitiesBasic" AS "j" +LIMIT 2 +"""); + } + public override async Task Add_entity_with_json() { await base.Add_entity_with_json(); @@ -107,6 +128,27 @@ public override async Task Add_entity_with_json() @p2=NULL (DbType = Int32) @p3='NewEntity' (Size = 9) +INSERT INTO "JsonEntitiesBasic" ("OwnedReferenceRoot", "Id", "EntityBasicId", "Name") +VALUES (@p0, @p1, @p2, @p3); +""", + // +""" +SELECT "j"."Id", "j"."EntityBasicId", "j"."Name", "j"."OwnedCollectionRoot", "j"."OwnedReferenceRoot" +FROM "JsonEntitiesBasic" AS "j" +"""); + } + + public override async Task Add_entity_with_json_null_navigations() + { + await base.Add_entity_with_json_null_navigations(); + + AssertSql( +""" +@p0='{"Name":"RootName","Number":42,"OwnedCollectionBranch":null,"OwnedReferenceBranch":{"Date":"2010-10-10T00:00:00","Enum":"Three","Fraction":42.42,"NullableEnum":null,"OwnedCollectionLeaf":[{"SomethingSomething":"ss1"},{"SomethingSomething":"ss2"}],"OwnedReferenceLeaf":null}}' (Nullable = false) (Size = 274) +@p1='2' +@p2=NULL (DbType = Int32) +@p3='NewEntity' (Size = 9) + INSERT INTO "JsonEntitiesBasic" ("OwnedReferenceRoot", "Id", "EntityBasicId", "Name") VALUES (@p0, @p1, @p2, @p3); """, @@ -184,7 +226,7 @@ public override async Task Delete_json_collection_branch() AssertSql( """ -@p0='[]' (Nullable = false) (Size = 2) +@p0=NULL (Nullable = false) @p1='1' UPDATE "JsonEntitiesBasic" SET "OwnedReferenceRoot" = json_set("OwnedReferenceRoot", '$.OwnedCollectionBranch', json(@p0)) @@ -205,7 +247,7 @@ public override async Task Delete_json_collection_root() AssertSql( """ -@p0='[]' (Nullable = false) (Size = 2) +@p0=NULL (Nullable = false) @p1='1' UPDATE "JsonEntitiesBasic" SET "OwnedCollectionRoot" = @p0 @@ -353,7 +395,7 @@ public override async Task Edit_element_in_json_branch_collection_and_add_elemen AssertSql( """ -@p0='[{"Date":"2101-01-01T00:00:00","Enum":"Two","Fraction":4321.3,"NullableEnum":"One","OwnedCollectionLeaf":[{"SomethingSomething":"e1_r_c1_c1"},{"SomethingSomething":"e1_r_c1_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_r_c1_r"}},{"Date":"2102-01-01T00:00:00","Enum":"Three","Fraction":10.2,"NullableEnum":"Two","OwnedCollectionLeaf":[{"SomethingSomething":"e1_r_c2_c1"},{"SomethingSomething":"e1_r_c2_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_r_c2_r"}},{"Date":"2222-11-11T00:00:00","Enum":"Three","Fraction":45.32,"NullableEnum":null,"OwnedCollectionLeaf":[],"OwnedReferenceLeaf":{"SomethingSomething":"cc"}}]' (Nullable = false) (Size = 628) +@p0='[{"Date":"2101-01-01T00:00:00","Enum":"Two","Fraction":4321.3,"NullableEnum":"One","OwnedCollectionLeaf":[{"SomethingSomething":"e1_r_c1_c1"},{"SomethingSomething":"e1_r_c1_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_r_c1_r"}},{"Date":"2102-01-01T00:00:00","Enum":"Three","Fraction":10.2,"NullableEnum":"Two","OwnedCollectionLeaf":[{"SomethingSomething":"e1_r_c2_c1"},{"SomethingSomething":"e1_r_c2_c2"}],"OwnedReferenceLeaf":{"SomethingSomething":"e1_r_c2_r"}},{"Date":"2222-11-11T00:00:00","Enum":"Three","Fraction":45.32,"NullableEnum":null,"OwnedCollectionLeaf":null,"OwnedReferenceLeaf":{"SomethingSomething":"cc"}}]' (Nullable = false) (Size = 630) @p1='1' UPDATE "JsonEntitiesBasic" SET "OwnedReferenceRoot" = json_set("OwnedReferenceRoot", '$.OwnedCollectionBranch', json(@p0)) @@ -1101,8 +1143,8 @@ public override async Task Edit_two_properties_on_same_entity_updates_the_entire AssertSql( """ -@p0='{"TestBoolean":false,"TestByte":25,"TestCharacter":"h","TestDateTime":"2100-11-11T12:34:56","TestDateTimeOffset":"2200-11-11T12:34:56-05:00","TestDecimal":-123450.01,"TestDefaultString":"MyDefaultStringInCollection1","TestDouble":-1.2345,"TestEnum":"One","TestEnumWithIntConverter":1,"TestGuid":"00000000-0000-0000-0000-000000000000","TestInt16":-12,"TestInt32":32,"TestInt64":64,"TestMaxLengthString":"Baz","TestNullableEnum":"One","TestNullableEnumWithConverterThatHandlesNulls":"Two","TestNullableEnumWithIntConverter":2,"TestNullableInt32":90,"TestSignedByte":-18,"TestSingle":-1.4,"TestTimeSpan":"06:05:04.0030000","TestUnsignedInt16":12,"TestUnsignedInt32":12345,"TestUnsignedInt64":1234567867}' (Nullable = false) (Size = 700) -@p1='{"TestBoolean":true,"TestByte":255,"TestCharacter":"a","TestDateTime":"2000-01-01T12:34:56","TestDateTimeOffset":"2000-01-01T12:34:56-08:00","TestDecimal":-1234567890.01,"TestDefaultString":"MyDefaultStringInReference1","TestDouble":-1.23456789,"TestEnum":"One","TestEnumWithIntConverter":1,"TestGuid":"12345678-1234-4321-7777-987654321000","TestInt16":-1234,"TestInt32":32,"TestInt64":64,"TestMaxLengthString":"Foo","TestNullableEnum":"One","TestNullableEnumWithConverterThatHandlesNulls":"Three","TestNullableEnumWithIntConverter":1,"TestNullableInt32":78,"TestSignedByte":-128,"TestSingle":-1.234,"TestTimeSpan":"10:09:08.0070000","TestUnsignedInt16":1234,"TestUnsignedInt32":1234565789,"TestUnsignedInt64":1234567890123456789}' (Nullable = false) (Size = 730) +@p0='{"TestBoolean":false,"TestByte":25,"TestCharacter":"h","TestDateTime":"2100-11-11T12:34:56","TestDateTimeOffset":"2200-11-11T12:34:56-05:00","TestDecimal":-123450.01,"TestDefaultString":"MyDefaultStringInCollection1","TestDouble":-1.2345,"TestEnum":"One","TestEnumWithIntConverter":1,"TestGuid":"00000000-0000-0000-0000-000000000000","TestInt16":-12,"TestInt32":32,"TestInt64":64,"TestMaxLengthString":"Baz","TestNullableEnum":"One","TestNullableEnumWithConverterThatHandlesNulls":"Two","TestNullableEnumWithIntConverter":2,"TestNullableInt32":90,"TestSignedByte":-18,"TestSingle":-1.4,"TestTimeSpan":"6:05:04.003","TestUnsignedInt16":12,"TestUnsignedInt32":12345,"TestUnsignedInt64":1234567867}' (Nullable = false) (Size = 695) +@p1='{"TestBoolean":true,"TestByte":255,"TestCharacter":"a","TestDateTime":"2000-01-01T12:34:56","TestDateTimeOffset":"2000-01-01T12:34:56-08:00","TestDecimal":-1234567890.01,"TestDefaultString":"MyDefaultStringInReference1","TestDouble":-1.23456789,"TestEnum":"One","TestEnumWithIntConverter":1,"TestGuid":"12345678-1234-4321-7777-987654321000","TestInt16":-1234,"TestInt32":32,"TestInt64":64,"TestMaxLengthString":"Foo","TestNullableEnum":"One","TestNullableEnumWithConverterThatHandlesNulls":"Three","TestNullableEnumWithIntConverter":1,"TestNullableInt32":78,"TestSignedByte":-128,"TestSingle":-1.234,"TestTimeSpan":"10:09:08.007","TestUnsignedInt16":1234,"TestUnsignedInt32":1234565789,"TestUnsignedInt64":1234567890123456789}' (Nullable = false) (Size = 726) @p2='1' UPDATE "JsonEntitiesAllTypes" SET "Collection" = json_set("Collection", '$[0]', json(@p0)), "Reference" = @p1