Skip to content

Commit

Permalink
Support querying over non-primitive JSON collections (#31369)
Browse files Browse the repository at this point in the history 
Closes #28616

Co-authored-by: Shay Rojansky <roji@roji.org>
  • Loading branch information
@maumar @roji
maumar and roji authored Aug 1, 2023
1 parent 4cda028 commit 7a572b0
Show file tree
Hide file tree
Showing 25 changed files with 2,576 additions and 536 deletions.
1 change: 1 addition & 0 deletions All.sln.DotSettings
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ The .NET Foundation licenses this file to you under the MIT license.&#xD;
<s:Boolean x:Key="/Default/UserDictionary/Words/=Includable/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=initializers/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=keyless/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Lite_0027s/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=materializer/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=materializers/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=materilization/@EntryIndexedValue">True</s:Boolean>
Expand Down
20 changes: 20 additions & 0 deletions src/EFCore.Relational/Properties/RelationalStrings.Designer.cs
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions src/EFCore.Relational/Properties/RelationalStrings.resx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,9 @@
<data name="JsonEntityMappedToDifferentViewThanOwner" xml:space="preserve">
<value>Entity '{jsonType}' is mapped to JSON and also to a view '{viewName}', but its owner '{ownerType}' is mapped to a different view '{ownerViewName}'. Every entity mapped to JSON must also map to the same view as its owner.</value>
</data>
<data name="JsonEntityMissingKeyInformation" xml:space="preserve">
<value>JSON entity '{jsonEntity}' is missing key information. This is not allowed for tracking queries since EF can't correctly build identity for this entity object.</value>
</data>
<data name="JsonEntityMultipleRootsMappedToTheSameJsonColumn" xml:space="preserve">
<value>Multiple owned root entities are mapped to the same JSON column '{column}' in table '{table}'. Each owned root entity must map to a different column.</value>
</data>
Expand Down Expand Up @@ -553,6 +556,9 @@
<data name="JsonReaderInvalidTokenType" xml:space="preserve">
<value>Invalid token type: '{tokenType}'.</value>
</data>
<data name="JsonQueryLinqOperatorsNotSupported" xml:space="preserve">
<value>Composing LINQ operators over collections inside JSON documents isn't supported or hasn't been implemented by your EF provider.</value>
</data>
<data name="JsonRequiredEntityWithNullJson" xml:space="preserve">
<value>Entity {entity} is required but the JSON element containing it is null.</value>
</data>
Expand Down Expand Up @@ -989,6 +995,9 @@
<data name="RelationalNotInUse" xml:space="preserve">
<value>Relational-specific methods can only be used when the context is using a relational database provider.</value>
</data>
<data name="SelectCanOnlyBeBuiltOnCollectionJsonQuery" xml:space="preserve">
<value>SelectExpression can only be built over a JsonQueryExpression that represents a collection within the JSON document.</value>
</data>
<data name="SelectExpressionNonTphWithCustomTable" xml:space="preserve">
<value>Cannot create a 'SelectExpression' with a custom 'TableExpressionBase' since the result type '{entityType}' is part of a hierarchy and does not contain a discriminator property.</value>
</data>
Expand Down
37 changes: 22 additions & 15 deletions src/EFCore.Relational/Query/JsonQueryExpression.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@ namespace Microsoft.EntityFrameworkCore.Query;
/// </summary>
public class JsonQueryExpression : Expression, IPrintableExpression
{
private readonly IReadOnlyDictionary<IProperty, ColumnExpression> _keyPropertyMap;

/// <summary>
/// Creates a new instance of the <see cref="JsonQueryExpression" /> class.
/// </summary>
Expand Down Expand Up @@ -57,7 +55,7 @@ private JsonQueryExpression(
EntityType = entityType;
JsonColumn = jsonColumn;
IsCollection = collection;
_keyPropertyMap = keyPropertyMap;
KeyPropertyMap = keyPropertyMap;
Type = type;
Path = path;
IsNullable = nullable;
Expand Down Expand Up @@ -88,6 +86,15 @@ private JsonQueryExpression(
/// </summary>
public virtual bool IsNullable { get; }

/// <summary>
/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to
/// the same compatibility standards as public APIs. It may be changed or removed without notice in
/// any release. You should only use it directly in your code with extreme caution and knowing that
/// doing so can result in application failures when updating to a new Entity Framework Core release.
/// </summary>
[EntityFrameworkInternal]
public virtual IReadOnlyDictionary<IProperty, ColumnExpression> KeyPropertyMap { get; }

/// <inheritdoc />
public override ExpressionType NodeType
=> ExpressionType.Extension;
Expand All @@ -107,7 +114,7 @@ public virtual SqlExpression BindProperty(IProperty property)
RelationalStrings.UnableToBindMemberToEntityProjection("property", property.Name, EntityType.DisplayName()));
}

if (_keyPropertyMap.TryGetValue(property, out var match))
if (KeyPropertyMap.TryGetValue(property, out var match))
{
return match;
}
Expand Down Expand Up @@ -145,11 +152,11 @@ public virtual JsonQueryExpression BindNavigation(INavigation navigation)
newPath.Add(new PathSegment(targetEntityType.GetJsonPropertyName()!));

var newKeyPropertyMap = new Dictionary<IProperty, ColumnExpression>();
var targetPrimaryKeyProperties = targetEntityType.FindPrimaryKey()!.Properties.Take(_keyPropertyMap.Count);
var sourcePrimaryKeyProperties = EntityType.FindPrimaryKey()!.Properties.Take(_keyPropertyMap.Count);
var targetPrimaryKeyProperties = targetEntityType.FindPrimaryKey()!.Properties.Take(KeyPropertyMap.Count);
var sourcePrimaryKeyProperties = EntityType.FindPrimaryKey()!.Properties.Take(KeyPropertyMap.Count);
foreach (var (target, source) in targetPrimaryKeyProperties.Zip(sourcePrimaryKeyProperties, (t, s) => (t, s)))
{
newKeyPropertyMap[target] = _keyPropertyMap[source];
newKeyPropertyMap[target] = KeyPropertyMap[source];
}

return new JsonQueryExpression(
Expand Down Expand Up @@ -178,7 +185,7 @@ public virtual JsonQueryExpression BindCollectionElement(SqlExpression collectio
return new JsonQueryExpression(
EntityType,
JsonColumn,
_keyPropertyMap,
KeyPropertyMap,
newPath,
EntityType.ClrType,
collection: false,
Expand All @@ -194,7 +201,7 @@ public virtual JsonQueryExpression BindCollectionElement(SqlExpression collectio
public virtual JsonQueryExpression MakeNullable()
{
var keyPropertyMap = new Dictionary<IProperty, ColumnExpression>();
foreach (var (property, columnExpression) in _keyPropertyMap)
foreach (var (property, columnExpression) in KeyPropertyMap)
{
keyPropertyMap[property] = columnExpression.MakeNullable();
}
Expand Down Expand Up @@ -223,7 +230,7 @@ protected override Expression VisitChildren(ExpressionVisitor visitor)
{
var jsonColumn = (ColumnExpression)visitor.Visit(JsonColumn);
var newKeyPropertyMap = new Dictionary<IProperty, ColumnExpression>();
foreach (var (property, column) in _keyPropertyMap)
foreach (var (property, column) in KeyPropertyMap)
{
newKeyPropertyMap[property] = (ColumnExpression)visitor.Visit(column);
}
Expand All @@ -242,8 +249,8 @@ public virtual JsonQueryExpression Update(
ColumnExpression jsonColumn,
IReadOnlyDictionary<IProperty, ColumnExpression> keyPropertyMap)
=> jsonColumn != JsonColumn
|| keyPropertyMap.Count != _keyPropertyMap.Count
|| keyPropertyMap.Zip(_keyPropertyMap, (n, o) => n.Value != o.Value).Any(x => x)
|| keyPropertyMap.Count != KeyPropertyMap.Count
|| keyPropertyMap.Zip(KeyPropertyMap, (n, o) => n.Value != o.Value).Any(x => x)
? new JsonQueryExpression(EntityType, jsonColumn, keyPropertyMap, Path, Type, IsCollection, IsNullable)
: this;

Expand All @@ -260,16 +267,16 @@ private bool Equals(JsonQueryExpression jsonQueryExpression)
&& IsCollection.Equals(jsonQueryExpression.IsCollection)
&& IsNullable == jsonQueryExpression.IsNullable
&& Path.SequenceEqual(jsonQueryExpression.Path)
&& KeyPropertyMapEquals(jsonQueryExpression._keyPropertyMap);
&& KeyPropertyMapEquals(jsonQueryExpression.KeyPropertyMap);

private bool KeyPropertyMapEquals(IReadOnlyDictionary<IProperty, ColumnExpression> other)
{
if (_keyPropertyMap.Count != other.Count)
if (KeyPropertyMap.Count != other.Count)
{
return false;
}

foreach (var (key, value) in _keyPropertyMap)
foreach (var (key, value) in KeyPropertyMap)
{
if (!other.TryGetValue(key, out var column) || !value.Equals(column))
{
Expand Down
72 changes: 67 additions & 5 deletions src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.

using System.Diagnostics.CodeAnalysis;
using System.Linq.Expressions;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
using Microsoft.EntityFrameworkCore.Query.Internal;
using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
Expand Down Expand Up @@ -223,6 +224,9 @@ when entityQueryRootExpression.GetType() == typeof(EntityQueryRootExpression)
char.ToLowerInvariant(sqlParameterExpression.Name.First(c => c != '_')).ToString())
?? base.VisitExtension(extensionExpression);

case JsonQueryExpression jsonQueryExpression:
return TransformJsonQueryToTable(jsonQueryExpression) ?? base.VisitExtension(extensionExpression);

default:
return base.VisitExtension(extensionExpression);
}
Expand Down Expand Up @@ -326,6 +330,19 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
string tableAlias)
=> null;

/// <summary>
/// Invoked when LINQ operators are composed over a collection within a JSON document.
/// Transforms the provided <see cref="JsonQueryExpression" /> - representing access to the collection - into a provider-specific
/// means to expand the JSON array into a relational table/rowset (e.g. SQL Server OPENJSON).
/// </summary>
/// <param name="jsonQueryExpression">The <see cref="JsonQueryExpression" /> referencing the JSON array.</param>
/// <returns>A <see cref="ShapedQueryExpression" /> if the translation was successful, otherwise <see langword="null" />.</returns>
protected virtual ShapedQueryExpression? TransformJsonQueryToTable(JsonQueryExpression jsonQueryExpression)
{
AddTranslationErrorDetails(RelationalStrings.JsonQueryLinqOperatorsNotSupported);
return null;
}

/// <summary>
/// Translates an inline collection into a queryable SQL VALUES expression.
/// </summary>
Expand Down Expand Up @@ -606,9 +623,9 @@ protected override ShapedQueryExpression TranslateConcat(ShapedQueryExpression s
protected override ShapedQueryExpression TranslateDistinct(ShapedQueryExpression source)
{
var selectExpression = (SelectExpression)source.QueryExpression;
if (selectExpression.Orderings.Count > 0
&& selectExpression.Limit == null
&& selectExpression.Offset == null)

if (selectExpression is { Orderings.Count: > 0, Limit: null, Offset: null }
&& !IsNaturallyOrdered(selectExpression))
{
_queryCompilationContext.Logger.DistinctAfterOrderByWithoutRowLimitingOperatorWarning();
}
Expand Down Expand Up @@ -1862,6 +1879,16 @@ protected virtual Expression ApplyInferredTypeMappings(
protected virtual bool IsOrdered(SelectExpression selectExpression)
=> selectExpression.Orderings.Count > 0;

/// <summary>
/// Determines whether the given <see cref="SelectExpression" /> is naturally ordered, meaning that any ordering has been added
/// automatically by EF to preserve e.g. the natural ordering of a JSON array, and not because the original LINQ query contained
/// an explicit ordering.
/// </summary>
/// <param name="selectExpression">The <see cref="SelectExpression" /> to check for ordering.</param>
/// <returns>Whether <paramref name="selectExpression"/> is ordered.</returns>
protected virtual bool IsNaturallyOrdered(SelectExpression selectExpression)
=> false;

private Expression RemapLambdaBody(ShapedQueryExpression shapedQueryExpression, LambdaExpression lambdaExpression)
{
var lambdaBody = ReplacingExpressionVisitor.Replace(
Expand Down Expand Up @@ -1923,11 +1950,10 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
source = Visit(source);

return TryExpand(source, MemberIdentity.Create(navigationName))
?? TryBindPrimitiveCollection(source, navigationName)
?? methodCallExpression.Update(null!, new[] { source, methodCallExpression.Arguments[1] });
}

// TODO: issue #28688
// when implementing collection of primitives, make sure EAOD is translated correctly for them
if (methodCallExpression.Method.IsGenericMethod
&& (methodCallExpression.Method.GetGenericMethodDefinition() == QueryableMethods.ElementAt
|| methodCallExpression.Method.GetGenericMethodDefinition() == QueryableMethods.ElementAtOrDefault))
Expand Down Expand Up @@ -2251,6 +2277,42 @@ static TableExpressionBase FindRootTableExpressionForColumn(ColumnExpression col
}
}

private Expression? TryBindPrimitiveCollection(Expression? source, string memberName)
{
while (source is IncludeExpression includeExpression)
{
source = includeExpression.EntityExpression;
}

source = source.UnwrapTypeConversion(out var convertedType);
if (source is not EntityShaperExpression entityShaperExpression)
{
return null;
}

var entityType = entityShaperExpression.EntityType;
if (convertedType != null)
{
entityType = entityType.GetRootType().GetDerivedTypesInclusive()
.FirstOrDefault(et => et.ClrType == convertedType);

if (entityType == null)
{
return null;
}
}

// TODO: Check that the property is a primitive collection property directly once we have that in metadata, rather than
// looking at the type mapping.
var property = entityType.FindProperty(memberName);
if (property?.GetRelationalTypeMapping().ElementTypeMapping is null)
{
return null;
}

return source.CreateEFPropertyExpression(property);
}

private sealed class AnnotationApplyingExpressionVisitor : ExpressionVisitor
{
private readonly IReadOnlyList<IAnnotation> _annotations;
Expand Down
Loading

0 comments on commit 7a572b0

Please sign in to comment.