Query: Defer inline-ing owned navigation expansion till all joins are…

… generated (#26641)

Issue: Expanding owned navigations in relational falls into 3 categories
- Collection navigation - which always generates a subquery. The predicate is generated in LINQ to allow mutation of outer SelectExpression
- Reference navigation sharing table - which picks column from same table without needing to add additional join. This only mutate the projection list for SelectExpression at subquery level
- Reference navigation mapped to separate table - which generates additional joins. Generating join can cause push down on outer SelectExpression if it has facets (e.g. limit/offset). This pushdown causes owned expansion from category 2 to be outdated and invalid SQL since they get pushed down to subquery. While their relevant entity projection is updated inside SelectExpression we already inlined older object in the tree at this point.

In order to avoid issue with outdated owned expansion, we defer actual inline-ing while processing owned navigations so that all navigations are expanded (causing any mutation on SelectExpression) before we inline values.

This PR introduces DeferredOwnedExpansionExpression which remembers the source projection binding to SelectExpression (which will remain accurate through pushdown), and navigation chain to traverse entity projections to reach entity shaper for final owned navigation. This way, we get up-to-date information from SelectExpression after all joins are generated.
We also find updated entity projection through binding after we generate a join if pushdown was required.

Resolves #26592

The issue was also present in 5.0 release causing non-performant SQL rather than invalid SQL. During 5.0 we expanded owned navigations again while during client eval phase (which happens in customer scenario due to include). This caused to earlier owned reference to have correct columns. Though the entity projection for owner was changed due to pushdown so we didn't add latter reference navigation binding in correct entity projection causing us to expand it again during 2nd pass.
The exact same issue doesn't occur for InMemory provider (due to slightly different implementation) though we should also make InMemory provider work this way, submitting in a different PR in case we need to cherry-pick this for patch.

src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs

@@ -1197,14 +1197,6 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
  ?? methodCallExpression.Update(null!, new[] { source, methodCallExpression.Arguments[1] });
  }
 
- if (methodCallExpression.TryGetEFPropertyArguments(out source, out navigationName))
- {
- source = Visit(source);
-
- return TryExpand(source, MemberIdentity.Create(navigationName))
- ?? methodCallExpression.Update(source, new[] { methodCallExpression.Arguments[0] });
- }
-
  return base.VisitMethodCall(methodCallExpression);
  }
 

src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Linq.Expressions;
 using System.Reflection;
@@ -999,6 +1000,7 @@ private static readonly MethodInfo _objectEqualsMethodInfo
  private readonly ISqlExpressionFactory _sqlExpressionFactory;
 
  private SelectExpression _selectExpression;
+ private DeferredOwnedExpansionRemovingVisitor _deferredOwnedExpansionRemover;
 
  public SharedTypeEntityExpandingExpressionVisitor(
  RelationalSqlTranslatingExpressionVisitor sqlTranslator,
@@ -1007,13 +1009,15 @@ public SharedTypeEntityExpandingExpressionVisitor(
  _sqlTranslator = sqlTranslator;
  _sqlExpressionFactory = sqlExpressionFactory;
  _selectExpression = null!;
+ _deferredOwnedExpansionRemover = null!;
  }
 
  public Expression Expand(SelectExpression selectExpression, Expression lambdaBody)
  {
  _selectExpression = selectExpression;
+ _deferredOwnedExpansionRemover = new(_selectExpression);
 
- return Visit(lambdaBody);
+ return _deferredOwnedExpansionRemover.Visit(Visit(lambdaBody));
  }
 
  protected override Expression VisitMember(MemberExpression memberExpression)
@@ -1034,14 +1038,6 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
  ?? methodCallExpression.Update(null!, new[] { source, methodCallExpression.Arguments[1] });
  }
 
- if (methodCallExpression.TryGetEFPropertyArguments(out source, out navigationName))
- {
- source = Visit(source);
-
- return TryExpand(source, MemberIdentity.Create(navigationName))
- ?? methodCallExpression.Update(source, new[] { methodCallExpression.Arguments[1] });
- }
-
  return base.VisitMethodCall(methodCallExpression);
  }
 
@@ -1053,7 +1049,13 @@ protected override Expression VisitExtension(Expression extensionExpression)
 
  private Expression? TryExpand(Expression? source, MemberIdentity member)
  {
+
  source = source.UnwrapTypeConversion(out var convertedType);
+ var doee = source as DeferredOwnedExpansionExpression;
+ if (doee is not null)
+ {
+ source = _deferredOwnedExpansionRemover.UnwrapDeferredEntityProjectionExpression(doee);
+ }
  if (source is not EntityShaperExpression entityShaperExpression)
  {
  return null;
@@ -1139,11 +1141,7 @@ outerKey is NewArrayExpression newArrayExpression
  Expression.Quote(correlationPredicate));
  }
 
- var entityProjectionExpression = (EntityProjectionExpression)
- (entityShaperExpression.ValueBufferExpression is ProjectionBindingExpression projectionBindingExpression
- ? _selectExpression.GetProjection(projectionBindingExpression)
- : entityShaperExpression.ValueBufferExpression);
-
+ var entityProjectionExpression = GetEntityProjectionExpression(entityShaperExpression);
  var innerShaper = entityProjectionExpression.BindNavigation(navigation);
  if (innerShaper == null)
  {
@@ -1206,7 +1204,23 @@ outerKey is NewArrayExpression newArrayExpression
  makeNullable);
 
  var joinPredicate = _sqlTranslator.Translate(Expression.Equal(outerKey, innerKey))!;
+ // Following conditions should match conditions for pushdown on outer during SelectExpression.AddJoin method
+ var pushdownRequired = _selectExpression.Limit != null
+ || _selectExpression.Offset != null
+ || _selectExpression.IsDistinct
+ || _selectExpression.GroupBy.Count > 0;
  _selectExpression.AddLeftJoin(innerSelectExpression, joinPredicate);
+
+ // If pushdown was required on SelectExpression then we need to fetch the updated entity projection
+ if (pushdownRequired)
+ {
+ if (doee is not null)
+ {
+ entityShaperExpression = _deferredOwnedExpansionRemover.UnwrapDeferredEntityProjectionExpression(doee);
+ }
+ entityProjectionExpression = GetEntityProjectionExpression(entityShaperExpression);
+ }
+
  var leftJoinTable = _selectExpression.Tables.Last();
 
  innerShaper = new RelationalEntityShaperExpression(
@@ -1219,13 +1233,104 @@ outerKey is NewArrayExpression newArrayExpression
  entityProjectionExpression.AddNavigationBinding(navigation, innerShaper);
  }
 
- return innerShaper;
+ return doee is not null
+ ? doee.AddNavigation(targetEntityType, navigation)
+ : new DeferredOwnedExpansionExpression(targetEntityType,
+ (ProjectionBindingExpression)entityShaperExpression.ValueBufferExpression,
+ navigation);
  }
 
  private static Expression AddConvertToObject(Expression expression)
  => expression.Type.IsValueType
  ? Expression.Convert(expression, typeof(object))
  : expression;
+
+ private EntityProjectionExpression GetEntityProjectionExpression(EntityShaperExpression entityShaperExpression)
+ => entityShaperExpression.ValueBufferExpression switch
+ {
+ ProjectionBindingExpression projectionBindingExpression
+ => (EntityProjectionExpression)_selectExpression.GetProjection(projectionBindingExpression),
+ EntityProjectionExpression entityProjectionExpression => entityProjectionExpression,
+ _ => throw new InvalidOperationException(),
+ };
+
+ private sealed class DeferredOwnedExpansionExpression : Expression
+ {
+ private readonly IEntityType _entityType;
+
+ public DeferredOwnedExpansionExpression(
+ IEntityType entityType,
+ ProjectionBindingExpression projectionBindingExpression,
+ INavigation navigation)
+ {
+ _entityType = entityType;
+ ProjectionBindingExpression = projectionBindingExpression;
+ NavigationChain = new List<INavigation> { navigation };
+ }
+
+ private DeferredOwnedExpansionExpression(
+ IEntityType entityType,
+ ProjectionBindingExpression projectionBindingExpression,
+ List<INavigation> navigationChain)
+ {
+ _entityType = entityType;
+ ProjectionBindingExpression = projectionBindingExpression;
+ NavigationChain = navigationChain;
+ }
+
+ public ProjectionBindingExpression ProjectionBindingExpression { get; }
+ public List<INavigation> NavigationChain { get; }
+
+ public DeferredOwnedExpansionExpression AddNavigation(IEntityType entityType, INavigation navigation)
+ {
+ var navigationChain = new List<INavigation>(NavigationChain.Count + 1);
+ navigationChain.AddRange(NavigationChain);
+ navigationChain.Add(navigation);
+
+ return new DeferredOwnedExpansionExpression(
+ entityType,
+ ProjectionBindingExpression,
+ navigationChain);
+ }
+
+ public override Type Type => _entityType.ClrType;
+
+ public override ExpressionType NodeType => ExpressionType.Extension;
+ }
+
+ private sealed class DeferredOwnedExpansionRemovingVisitor : ExpressionVisitor
+ {
+ private readonly SelectExpression _selectExpression;
+
+ public DeferredOwnedExpansionRemovingVisitor(SelectExpression selectExpression)
+ {
+ _selectExpression = selectExpression;
+ }
+
+ [return: NotNullIfNotNull("expression")]
+ public override Expression? Visit(Expression? expression)
+ => expression switch
+ {
+ DeferredOwnedExpansionExpression doee => UnwrapDeferredEntityProjectionExpression(doee),
+ // For the source entity shaper or owned collection expansion
+ EntityShaperExpression _ or ShapedQueryExpression _ => expression,
+ _ => base.Visit(expression),
+ };
+
+ public EntityShaperExpression UnwrapDeferredEntityProjectionExpression(DeferredOwnedExpansionExpression doee)
+ {
+ var entityProjection = (EntityProjectionExpression)_selectExpression.GetProjection(doee.ProjectionBindingExpression);
+ var entityShaper = entityProjection.BindNavigation(doee.NavigationChain[0])!;
+
+ for (var i = 1; i < doee.NavigationChain.Count; i++)
+ {
+ entityProjection = (EntityProjectionExpression)entityShaper.ValueBufferExpression;
+ entityShaper = entityProjection.BindNavigation(doee.NavigationChain[i])!;
+ }
+
+ return entityShaper;
+ }
+ }
  }
 
  private ShapedQueryExpression TranslateTwoParameterSelector(ShapedQueryExpression source, LambdaExpression resultSelector)

test/EFCore.InMemory.FunctionalTests/Query/OwnedEntityQueryInMemoryTest.cs

@@ -72,5 +72,33 @@ protected class Foo
  public virtual Bar? Bar { get; set; }
  }
 #nullable disable
+
+ [ConditionalTheory]
+ [MemberData(nameof(IsAsyncData))]
+ public virtual async Task Owned_references_on_same_level_expanded_at_different_times_around_take(bool async)
+ {
+ var contextFactory = await InitializeAsync<MyContext26592>(seed: c => c.Seed());
+ using var context = contextFactory.CreateContext();
+
+ await base.Owned_references_on_same_level_expanded_at_different_times_around_take_helper(context, async);
+ }
+
+ [ConditionalTheory]
+ [MemberData(nameof(IsAsyncData))]
+ public virtual async Task Owned_references_on_same_level_nested_expanded_at_different_times_around_take(bool async)
+ {
+ var contextFactory = await InitializeAsync<MyContext26592>(seed: c => c.Seed());
+ using var context = contextFactory.CreateContext();
+
+ await base.Owned_references_on_same_level_nested_expanded_at_different_times_around_take_helper(context, async);
+ }
+
+ protected class MyContext26592 : MyContext26592Base
+ {
+ public MyContext26592(DbContextOptions options)
+ : base(options)
+ {
+ }
+ }
  }
 }

test/EFCore.Relational.Specification.Tests/Query/OwnedEntityQueryRelationalTestBase.cs

@@ -164,5 +164,54 @@ protected class PublishTokenType25680
  public string TokenGroupId { get; set; }
  public string IssuerName { get; set; }
  }
+
+ [ConditionalTheory]
+ [MemberData(nameof(IsAsyncData))]
+ public virtual async Task Owned_reference_mapped_to_different_table_updated_correctly_after_subquery_pushdown(bool async)
+ {
+ var contextFactory = await InitializeAsync<MyContext26592>(seed: c => c.Seed());
+ using var context = contextFactory.CreateContext();
+
+ await base.Owned_references_on_same_level_expanded_at_different_times_around_take_helper(context, async);
+ }
+
+ [ConditionalTheory]
+ [MemberData(nameof(IsAsyncData))]
+ public virtual async Task Owned_reference_mapped_to_different_table_nested_updated_correctly_after_subquery_pushdown(bool async)
+ {
+ var contextFactory = await InitializeAsync<MyContext26592>(seed: c => c.Seed());
+ using var context = contextFactory.CreateContext();
+
+ await base.Owned_references_on_same_level_nested_expanded_at_different_times_around_take_helper(context, async);
+ }
+
+ protected class MyContext26592 : MyContext26592Base
+ {
+ public MyContext26592(DbContextOptions options)
+ : base(options)
+ {
+ }
+
+ protected override void OnModelCreating(ModelBuilder modelBuilder)
+ {
+ modelBuilder.Entity<Company>(
+ b =>
+ {
+ b.OwnsOne(e => e.CustomerData).ToTable("CustomerData");
+ b.OwnsOne(e => e.SupplierData).ToTable("SupplierData");
+ });
+
+ modelBuilder.Entity<Owner>(
+ b =>
+ {
+ b.OwnsOne(e => e.OwnedEntity, o =>
+ {
+ o.ToTable("IntermediateOwnedEntity");
+ o.OwnsOne(e => e.CustomerData).ToTable("IM_CustomerData");
+ o.OwnsOne(e => e.SupplierData).ToTable("IM_SupplierData");
+ });
+ });
+ }
+ }
  }
 }