Support querying over non-primitive JSON collections

Closes #28616

All.sln.DotSettings

@@ -299,6 +299,7 @@ The .NET Foundation licenses this file to you under the MIT license.
  <s:Boolean x:Key="/Default/UserDictionary/Words/=Includable/@EntryIndexedValue">True</s:Boolean>
  <s:Boolean x:Key="/Default/UserDictionary/Words/=initializers/@EntryIndexedValue">True</s:Boolean>
  <s:Boolean x:Key="/Default/UserDictionary/Words/=keyless/@EntryIndexedValue">True</s:Boolean>
+ <s:Boolean x:Key="/Default/UserDictionary/Words/=Lite_0027s/@EntryIndexedValue">True</s:Boolean>
  <s:Boolean x:Key="/Default/UserDictionary/Words/=materializer/@EntryIndexedValue">True</s:Boolean>
  <s:Boolean x:Key="/Default/UserDictionary/Words/=materializers/@EntryIndexedValue">True</s:Boolean>
  <s:Boolean x:Key="/Default/UserDictionary/Words/=materilization/@EntryIndexedValue">True</s:Boolean>

src/EFCore.Relational/Properties/RelationalStrings.resx

@@ -553,6 +553,9 @@
  <data name="JsonReaderInvalidTokenType" xml:space="preserve">
  <value>Invalid token type: '{tokenType}'.</value>
  </data>
+ <data name="JsonQueryLinqOperatorsNotSupported" xml:space="preserve">
+ <value>Composing LINQ operators over collections inside JSON documents isn't supported or hasn't been implemented by your EF provider.</value>
+ </data>
  <data name="JsonRequiredEntityWithNullJson" xml:space="preserve">
  <value>Entity {entity} is required but the JSON element containing it is null.</value>
  </data>
@@ -989,6 +992,9 @@
  <data name="RelationalNotInUse" xml:space="preserve">
  <value>Relational-specific methods can only be used when the context is using a relational database provider.</value>
  </data>
+ <data name="SelectCanOnlyBeBuiltOnCollectionJsonQuery" xml:space="preserve">
+ <value>SelectExpression can only be built over a JsonQueryExpression that represents a collection within the JSON document.</value>
+ </data>
  <data name="SelectExpressionNonTphWithCustomTable" xml:space="preserve">
  <value>Cannot create a 'SelectExpression' with a custom 'TableExpressionBase' since the result type '{entityType}' is part of a hierarchy and does not contain a discriminator property.</value>
  </data>

src/EFCore.Relational/Query/JsonQueryExpression.cs

@@ -16,8 +16,6 @@ namespace Microsoft.EntityFrameworkCore.Query;
 /// </summary>
 public class JsonQueryExpression : Expression, IPrintableExpression
 {
- private readonly IReadOnlyDictionary<IProperty, ColumnExpression> _keyPropertyMap;
-
  /// <summary>
  /// Creates a new instance of the <see cref="JsonQueryExpression" /> class.
  /// </summary>
@@ -57,7 +55,7 @@ private JsonQueryExpression(
  EntityType = entityType;
  JsonColumn = jsonColumn;
  IsCollection = collection;
- _keyPropertyMap = keyPropertyMap;
+ KeyPropertyMap = keyPropertyMap;
  Type = type;
  Path = path;
  IsNullable = nullable;
@@ -88,6 +86,15 @@ private JsonQueryExpression(
  /// </summary>
  public virtual bool IsNullable { get; }
 
+ /// <summary>
+ /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+ /// the same compatibility standards as public APIs. It may be changed or removed without notice in
+ /// any release. You should only use it directly in your code with extreme caution and knowing that
+ /// doing so can result in application failures when updating to a new Entity Framework Core release.
+ /// </summary>
+ [EntityFrameworkInternal]
+ public virtual IReadOnlyDictionary<IProperty, ColumnExpression> KeyPropertyMap { get; }
+
  /// <inheritdoc />
  public override ExpressionType NodeType
  => ExpressionType.Extension;
@@ -107,7 +114,7 @@ public virtual SqlExpression BindProperty(IProperty property)
  RelationalStrings.UnableToBindMemberToEntityProjection("property", property.Name, EntityType.DisplayName()));
  }
 
- if (_keyPropertyMap.TryGetValue(property, out var match))
+ if (KeyPropertyMap.TryGetValue(property, out var match))
  {
  return match;
  }
@@ -145,11 +152,11 @@ public virtual JsonQueryExpression BindNavigation(INavigation navigation)
  newPath.Add(new PathSegment(targetEntityType.GetJsonPropertyName()!));
 
  var newKeyPropertyMap = new Dictionary<IProperty, ColumnExpression>();
- var targetPrimaryKeyProperties = targetEntityType.FindPrimaryKey()!.Properties.Take(_keyPropertyMap.Count);
- var sourcePrimaryKeyProperties = EntityType.FindPrimaryKey()!.Properties.Take(_keyPropertyMap.Count);
+ var targetPrimaryKeyProperties = targetEntityType.FindPrimaryKey()!.Properties.Take(KeyPropertyMap.Count);
+ var sourcePrimaryKeyProperties = EntityType.FindPrimaryKey()!.Properties.Take(KeyPropertyMap.Count);
  foreach (var (target, source) in targetPrimaryKeyProperties.Zip(sourcePrimaryKeyProperties, (t, s) => (t, s)))
  {
- newKeyPropertyMap[target] = _keyPropertyMap[source];
+ newKeyPropertyMap[target] = KeyPropertyMap[source];
  }
 
  return new JsonQueryExpression(
@@ -178,7 +185,7 @@ public virtual JsonQueryExpression BindCollectionElement(SqlExpression collectio
  return new JsonQueryExpression(
  EntityType,
  JsonColumn,
- _keyPropertyMap,
+ KeyPropertyMap,
  newPath,
  EntityType.ClrType,
  collection: false,
@@ -194,7 +201,7 @@ public virtual JsonQueryExpression BindCollectionElement(SqlExpression collectio
  public virtual JsonQueryExpression MakeNullable()
  {
  var keyPropertyMap = new Dictionary<IProperty, ColumnExpression>();
- foreach (var (property, columnExpression) in _keyPropertyMap)
+ foreach (var (property, columnExpression) in KeyPropertyMap)
  {
  keyPropertyMap[property] = columnExpression.MakeNullable();
  }
@@ -223,7 +230,7 @@ protected override Expression VisitChildren(ExpressionVisitor visitor)
  {
  var jsonColumn = (ColumnExpression)visitor.Visit(JsonColumn);
  var newKeyPropertyMap = new Dictionary<IProperty, ColumnExpression>();
- foreach (var (property, column) in _keyPropertyMap)
+ foreach (var (property, column) in KeyPropertyMap)
  {
  newKeyPropertyMap[property] = (ColumnExpression)visitor.Visit(column);
  }
@@ -242,8 +249,8 @@ public virtual JsonQueryExpression Update(
  ColumnExpression jsonColumn,
  IReadOnlyDictionary<IProperty, ColumnExpression> keyPropertyMap)
  => jsonColumn != JsonColumn
- || keyPropertyMap.Count != _keyPropertyMap.Count
- || keyPropertyMap.Zip(_keyPropertyMap, (n, o) => n.Value != o.Value).Any(x => x)
+ || keyPropertyMap.Count != KeyPropertyMap.Count
+ || keyPropertyMap.Zip(KeyPropertyMap, (n, o) => n.Value != o.Value).Any(x => x)
  ? new JsonQueryExpression(EntityType, jsonColumn, keyPropertyMap, Path, Type, IsCollection, IsNullable)
  : this;
 
@@ -260,16 +267,16 @@ private bool Equals(JsonQueryExpression jsonQueryExpression)
  && IsCollection.Equals(jsonQueryExpression.IsCollection)
  && IsNullable == jsonQueryExpression.IsNullable
  && Path.SequenceEqual(jsonQueryExpression.Path)
- && KeyPropertyMapEquals(jsonQueryExpression._keyPropertyMap);
+ && KeyPropertyMapEquals(jsonQueryExpression.KeyPropertyMap);
 
  private bool KeyPropertyMapEquals(IReadOnlyDictionary<IProperty, ColumnExpression> other)
  {
- if (_keyPropertyMap.Count != other.Count)
+ if (KeyPropertyMap.Count != other.Count)
  {
  return false;
  }
 
- foreach (var (key, value) in _keyPropertyMap)
+ foreach (var (key, value) in KeyPropertyMap)
  {
  if (!other.TryGetValue(key, out var column) || !value.Equals(column))
  {

src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs

@@ -223,6 +223,9 @@ when entityQueryRootExpression.GetType() == typeof(EntityQueryRootExpression)
  char.ToLowerInvariant(sqlParameterExpression.Name.First(c => c != '_')).ToString())
  ?? base.VisitExtension(extensionExpression);
 
+ case JsonQueryExpression jsonQueryExpression:
+ return TransformJsonQueryToTable(jsonQueryExpression) ?? base.VisitExtension(extensionExpression);
+
  default:
  return base.VisitExtension(extensionExpression);
  }
@@ -326,6 +329,19 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
  string tableAlias)
  => null;
 
+ /// <summary>
+ /// Invoked when LINQ operators are composed over a collection within a JSON document.
+ /// Transforms the provided <see cref="JsonQueryExpression" /> - representing access to the collection - into a provider-specific
+ /// means to expand the JSON array into a relational table/rowset (e.g. SQL Server OPENJSON).
+ /// </summary>
+ /// <param name="jsonQueryExpression">The <see cref="JsonQueryExpression" /> referencing the JSON array.</param>
+ /// <returns>A <see cref="ShapedQueryExpression" /> if the translation was successful, otherwise <see langword="null" />.</returns>
+ protected virtual ShapedQueryExpression? TransformJsonQueryToTable(JsonQueryExpression jsonQueryExpression)
+ {
+ AddTranslationErrorDetails(RelationalStrings.JsonQueryLinqOperatorsNotSupported);
+ return null;
+ }
+
  /// <summary>
  /// Translates an inline collection into a queryable SQL VALUES expression.
  /// </summary>
@@ -606,9 +622,9 @@ protected override ShapedQueryExpression TranslateConcat(ShapedQueryExpression s
  protected override ShapedQueryExpression TranslateDistinct(ShapedQueryExpression source)
  {
  var selectExpression = (SelectExpression)source.QueryExpression;
- if (selectExpression.Orderings.Count > 0
-  && selectExpression.Limit == null
- && selectExpression.Offset == null)
+
+ if (selectExpression is { Orderings.Count: > 0, Limit: null, Offset: null }
+ && !IsNaturallyOrdered(selectExpression))
  {
  _queryCompilationContext.Logger.DistinctAfterOrderByWithoutRowLimitingOperatorWarning();
  }
@@ -1862,6 +1878,16 @@ protected virtual Expression ApplyInferredTypeMappings(
  protected virtual bool IsOrdered(SelectExpression selectExpression)
  => selectExpression.Orderings.Count > 0;
 
+ /// <summary>
+ /// Determines whether the given <see cref="SelectExpression" /> is naturally ordered, meaning that any ordering has been added
+ /// automatically by EF to preserve e.g. the natural ordering of a JSON array, and not because the original LINQ query contained
+ /// an explicit ordering.
+ /// </summary>
+ /// <param name="selectExpression">The <see cref="SelectExpression" /> to check for ordering.</param>
+ /// <returns>Whether <paramref name="selectExpression"/> is ordered.</returns>
+ protected virtual bool IsNaturallyOrdered(SelectExpression selectExpression)
+ => false;
+
  private Expression RemapLambdaBody(ShapedQueryExpression shapedQueryExpression, LambdaExpression lambdaExpression)
  {
  var lambdaBody = ReplacingExpressionVisitor.Replace(