From 74bfeeb6cc917633fc7ae67da305c1e9dfc72d13 Mon Sep 17 00:00:00 2001 From: Smit Patel Date: Thu, 11 Nov 2021 12:44:58 -0800 Subject: [PATCH] Query: Defer inline-ing owned navigation expansion till all joins are generated Issue: Expanding owned navigations in relational falls into 3 categories - Collection navigation - which always generates a subquery. The predicate is generated in LINQ to allow mutation of outer SelectExpression - Reference navigation sharing table - which picks column from same table without needing to add additional join. This only mutate the projection list for SelectExpression at subquery level - Reference navigation mapped to separate table - which generates additional joins. Generating join can cause push down on outer SelectExpression if it has facets (e.g. limit/offset). This pushdown causes owned expansion from category 2 to be outdated and invalid SQL since they get pushed down to subquery. While their relevant entity projection is updated inside SelectExpression we already inlined older object in the tree at this point. In order to avoid issue with outdated owned expansion, we defer actual inline-ing while processing owned navigations so that all navigations are expanded (causing any mutation on SelectExpression) before we inline values. This PR introduces DeferredOwnedExpansionExpression which remembers the source projection binding to SelectExpression (which will remain accurate through pushdown), and navigation chain to traverse entity projections to reach entity shaper for final owned navigation. This way, we get up-to-date information from SelectExpression after all joins are generated. We also find updated entity projection through binding after we generate a join if pushdown was required. Resolves #26592 The issue was also present in 5.0 release causing non-performant SQL rather than invalid SQL. During 5.0 we expanded owned navigations again while during client eval phase (which happens in customer scenario due to include). This caused to earlier owned reference to have correct columns. Though the entity projection for owner was changed due to pushdown so we didn't add latter reference navigation binding in correct entity projection causing us to expand it again during 2nd pass. The exact same issue doesn't occur for InMemory provider (due to slightly different implementation) though we should also make InMemory provider work this way, submitting in a different PR in case we need to cherry-pick this for patch. --- ...yableMethodTranslatingExpressionVisitor.cs | 8 -- ...yableMethodTranslatingExpressionVisitor.cs | 135 ++++++++++++++++-- .../Query/OwnedEntityQueryInMemoryTest.cs | 28 ++++ .../OwnedEntityQueryRelationalTestBase.cs | 49 +++++++ .../Query/OwnedEntityQueryTestBase.cs | 111 ++++++++++++++ .../Query/OwnedEntityQuerySqlServerTest.cs | 39 +++++ 6 files changed, 347 insertions(+), 23 deletions(-) diff --git a/src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs b/src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs index 811d39a728a..21eb3fce4f2 100644 --- a/src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs +++ b/src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs @@ -1197,14 +1197,6 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp ?? methodCallExpression.Update(null!, new[] { source, methodCallExpression.Arguments[1] }); } - if (methodCallExpression.TryGetEFPropertyArguments(out source, out navigationName)) - { - source = Visit(source); - - return TryExpand(source, MemberIdentity.Create(navigationName)) - ?? methodCallExpression.Update(source, new[] { methodCallExpression.Arguments[0] }); - } - return base.VisitMethodCall(methodCallExpression); } diff --git a/src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs b/src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs index 336560ca752..f5c3741cbf8 100644 --- a/src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs +++ b/src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Linq.Expressions; using System.Reflection; @@ -999,6 +1000,7 @@ private static readonly MethodInfo _objectEqualsMethodInfo private readonly ISqlExpressionFactory _sqlExpressionFactory; private SelectExpression _selectExpression; + private DeferredOwnedExpansionRemovingVisitor _deferredOwnedExpansionRemover; public SharedTypeEntityExpandingExpressionVisitor( RelationalSqlTranslatingExpressionVisitor sqlTranslator, @@ -1007,13 +1009,15 @@ public SharedTypeEntityExpandingExpressionVisitor( _sqlTranslator = sqlTranslator; _sqlExpressionFactory = sqlExpressionFactory; _selectExpression = null!; + _deferredOwnedExpansionRemover = null!; } public Expression Expand(SelectExpression selectExpression, Expression lambdaBody) { _selectExpression = selectExpression; + _deferredOwnedExpansionRemover = new(_selectExpression); - return Visit(lambdaBody); + return _deferredOwnedExpansionRemover.Visit(Visit(lambdaBody)); } protected override Expression VisitMember(MemberExpression memberExpression) @@ -1034,14 +1038,6 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp ?? methodCallExpression.Update(null!, new[] { source, methodCallExpression.Arguments[1] }); } - if (methodCallExpression.TryGetEFPropertyArguments(out source, out navigationName)) - { - source = Visit(source); - - return TryExpand(source, MemberIdentity.Create(navigationName)) - ?? methodCallExpression.Update(source, new[] { methodCallExpression.Arguments[1] }); - } - return base.VisitMethodCall(methodCallExpression); } @@ -1053,7 +1049,13 @@ protected override Expression VisitExtension(Expression extensionExpression) private Expression? TryExpand(Expression? source, MemberIdentity member) { + source = source.UnwrapTypeConversion(out var convertedType); + var doee = source as DeferredOwnedExpansionExpression; + if (doee is not null) + { + source = _deferredOwnedExpansionRemover.UnwrapDeferredEntityProjectionExpression(doee); + } if (source is not EntityShaperExpression entityShaperExpression) { return null; @@ -1139,11 +1141,7 @@ outerKey is NewArrayExpression newArrayExpression Expression.Quote(correlationPredicate)); } - var entityProjectionExpression = (EntityProjectionExpression) - (entityShaperExpression.ValueBufferExpression is ProjectionBindingExpression projectionBindingExpression - ? _selectExpression.GetProjection(projectionBindingExpression) - : entityShaperExpression.ValueBufferExpression); - + var entityProjectionExpression = GetEntityProjectionExpression(entityShaperExpression); var innerShaper = entityProjectionExpression.BindNavigation(navigation); if (innerShaper == null) { @@ -1206,7 +1204,23 @@ outerKey is NewArrayExpression newArrayExpression makeNullable); var joinPredicate = _sqlTranslator.Translate(Expression.Equal(outerKey, innerKey))!; + // Following conditions should match conditions for pushdown on outer during SelectExpression.AddJoin method + var pushdownRequired = _selectExpression.Limit != null + || _selectExpression.Offset != null + || _selectExpression.IsDistinct + || _selectExpression.GroupBy.Count > 0; _selectExpression.AddLeftJoin(innerSelectExpression, joinPredicate); + + // If pushdown was required on SelectExpression then we need to fetch the updated entity projection + if (pushdownRequired) + { + if (doee is not null) + { + entityShaperExpression = _deferredOwnedExpansionRemover.UnwrapDeferredEntityProjectionExpression(doee); + } + entityProjectionExpression = GetEntityProjectionExpression(entityShaperExpression); + } + var leftJoinTable = _selectExpression.Tables.Last(); innerShaper = new RelationalEntityShaperExpression( @@ -1219,13 +1233,104 @@ outerKey is NewArrayExpression newArrayExpression entityProjectionExpression.AddNavigationBinding(navigation, innerShaper); } - return innerShaper; + return doee is not null + ? doee.AddNavigation(targetEntityType, navigation) + : new DeferredOwnedExpansionExpression(targetEntityType, + (ProjectionBindingExpression)entityShaperExpression.ValueBufferExpression, + navigation); } private static Expression AddConvertToObject(Expression expression) => expression.Type.IsValueType ? Expression.Convert(expression, typeof(object)) : expression; + + private EntityProjectionExpression GetEntityProjectionExpression(EntityShaperExpression entityShaperExpression) + => entityShaperExpression.ValueBufferExpression switch + { + ProjectionBindingExpression projectionBindingExpression + => (EntityProjectionExpression)_selectExpression.GetProjection(projectionBindingExpression), + EntityProjectionExpression entityProjectionExpression => entityProjectionExpression, + _ => throw new InvalidOperationException(), + }; + + private sealed class DeferredOwnedExpansionExpression : Expression + { + private readonly IEntityType _entityType; + + public DeferredOwnedExpansionExpression( + IEntityType entityType, + ProjectionBindingExpression projectionBindingExpression, + INavigation navigation) + { + _entityType = entityType; + ProjectionBindingExpression = projectionBindingExpression; + NavigationChain = new List { navigation }; + } + + private DeferredOwnedExpansionExpression( + IEntityType entityType, + ProjectionBindingExpression projectionBindingExpression, + List navigationChain) + { + _entityType = entityType; + ProjectionBindingExpression = projectionBindingExpression; + NavigationChain = navigationChain; + } + + public ProjectionBindingExpression ProjectionBindingExpression { get; } + public List NavigationChain { get; } + + public DeferredOwnedExpansionExpression AddNavigation(IEntityType entityType, INavigation navigation) + { + var navigationChain = new List(NavigationChain.Count + 1); + navigationChain.AddRange(NavigationChain); + navigationChain.Add(navigation); + + return new DeferredOwnedExpansionExpression( + entityType, + ProjectionBindingExpression, + navigationChain); + } + + public override Type Type => _entityType.ClrType; + + public override ExpressionType NodeType => ExpressionType.Extension; + } + + private sealed class DeferredOwnedExpansionRemovingVisitor : ExpressionVisitor + { + private readonly SelectExpression _selectExpression; + + public DeferredOwnedExpansionRemovingVisitor(SelectExpression selectExpression) + { + _selectExpression = selectExpression; + } + + [return: NotNullIfNotNull("expression")] + public override Expression? Visit(Expression? expression) + => expression switch + { + DeferredOwnedExpansionExpression doee => UnwrapDeferredEntityProjectionExpression(doee), + // For the source entity shaper or owned collection expansion + EntityShaperExpression _ or ShapedQueryExpression _ => expression, + _ => base.Visit(expression), + }; + + public EntityShaperExpression UnwrapDeferredEntityProjectionExpression(DeferredOwnedExpansionExpression doee) + { + var entityProjection = (EntityProjectionExpression)_selectExpression.GetProjection(doee.ProjectionBindingExpression); + var entityShaper = entityProjection.BindNavigation(doee.NavigationChain[0])!; + + for (var i = 1; i < doee.NavigationChain.Count; i++) + { + entityProjection = (EntityProjectionExpression)entityShaper.ValueBufferExpression; + entityShaper = entityProjection.BindNavigation(doee.NavigationChain[i])!; + } + + return entityShaper; + } + } } private ShapedQueryExpression TranslateTwoParameterSelector(ShapedQueryExpression source, LambdaExpression resultSelector) diff --git a/test/EFCore.InMemory.FunctionalTests/Query/OwnedEntityQueryInMemoryTest.cs b/test/EFCore.InMemory.FunctionalTests/Query/OwnedEntityQueryInMemoryTest.cs index 8e220067c2f..b83637b7243 100644 --- a/test/EFCore.InMemory.FunctionalTests/Query/OwnedEntityQueryInMemoryTest.cs +++ b/test/EFCore.InMemory.FunctionalTests/Query/OwnedEntityQueryInMemoryTest.cs @@ -72,5 +72,33 @@ protected class Foo public virtual Bar? Bar { get; set; } } #nullable disable + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual async Task Owned_references_on_same_level_expanded_at_different_times_around_take(bool async) + { + var contextFactory = await InitializeAsync(seed: c => c.Seed()); + using var context = contextFactory.CreateContext(); + + await base.Owned_references_on_same_level_expanded_at_different_times_around_take_helper(context, async); + } + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual async Task Owned_references_on_same_level_nested_expanded_at_different_times_around_take(bool async) + { + var contextFactory = await InitializeAsync(seed: c => c.Seed()); + using var context = contextFactory.CreateContext(); + + await base.Owned_references_on_same_level_nested_expanded_at_different_times_around_take_helper(context, async); + } + + protected class MyContext26592 : MyContext26592Base + { + public MyContext26592(DbContextOptions options) + : base(options) + { + } + } } } diff --git a/test/EFCore.Relational.Specification.Tests/Query/OwnedEntityQueryRelationalTestBase.cs b/test/EFCore.Relational.Specification.Tests/Query/OwnedEntityQueryRelationalTestBase.cs index a4f604e6b12..26fab1ed17c 100644 --- a/test/EFCore.Relational.Specification.Tests/Query/OwnedEntityQueryRelationalTestBase.cs +++ b/test/EFCore.Relational.Specification.Tests/Query/OwnedEntityQueryRelationalTestBase.cs @@ -164,5 +164,54 @@ protected class PublishTokenType25680 public string TokenGroupId { get; set; } public string IssuerName { get; set; } } + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual async Task Owned_reference_mapped_to_different_table_updated_correctly_after_subquery_pushdown(bool async) + { + var contextFactory = await InitializeAsync(seed: c => c.Seed()); + using var context = contextFactory.CreateContext(); + + await base.Owned_references_on_same_level_expanded_at_different_times_around_take_helper(context, async); + } + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual async Task Owned_reference_mapped_to_different_table_nested_updated_correctly_after_subquery_pushdown(bool async) + { + var contextFactory = await InitializeAsync(seed: c => c.Seed()); + using var context = contextFactory.CreateContext(); + + await base.Owned_references_on_same_level_nested_expanded_at_different_times_around_take_helper(context, async); + } + + protected class MyContext26592 : MyContext26592Base + { + public MyContext26592(DbContextOptions options) + : base(options) + { + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity( + b => + { + b.OwnsOne(e => e.CustomerData).ToTable("CustomerData"); + b.OwnsOne(e => e.SupplierData).ToTable("SupplierData"); + }); + + modelBuilder.Entity( + b => + { + b.OwnsOne(e => e.OwnedEntity, o => + { + o.ToTable("IntermediateOwnedEntity"); + o.OwnsOne(e => e.CustomerData).ToTable("IM_CustomerData"); + o.OwnsOne(e => e.SupplierData).ToTable("IM_SupplierData"); + }); + }); + } + } } } diff --git a/test/EFCore.Specification.Tests/Query/OwnedEntityQueryTestBase.cs b/test/EFCore.Specification.Tests/Query/OwnedEntityQueryTestBase.cs index f549de2e367..3fe276e7ce2 100644 --- a/test/EFCore.Specification.Tests/Query/OwnedEntityQueryTestBase.cs +++ b/test/EFCore.Specification.Tests/Query/OwnedEntityQueryTestBase.cs @@ -308,5 +308,116 @@ protected class WarehouseModel public ICollection DestinationCountryCodes { get; set; } } + protected virtual async Task Owned_references_on_same_level_expanded_at_different_times_around_take_helper(MyContext26592Base context, bool async) + { + var query = context.Companies.Where(e => e.CustomerData != null).OrderBy(e => e.Id).Take(10); + var result = async + ? await query.ToListAsync() + : query.ToList(); + + var company = Assert.Single(result); + Assert.Equal("Acme Inc.", company.Name); + Assert.Equal("Regular", company.CustomerData.AdditionalCustomerData); + Assert.Equal("Free shipping", company.SupplierData.AdditionalSupplierData); + } + + protected virtual async Task Owned_references_on_same_level_nested_expanded_at_different_times_around_take_helper( + MyContext26592Base context, bool async) + { + var query = context.Owners.Where(e => e.OwnedEntity.CustomerData != null).OrderBy(e => e.Id).Take(10); + var result = async + ? await query.ToListAsync() + : query.ToList(); + + var owner = Assert.Single(result); + Assert.Equal("Owner1", owner.Name); + Assert.Equal("Intermediate1", owner.OwnedEntity.Name); + Assert.Equal("IM Regular", owner.OwnedEntity.CustomerData.AdditionalCustomerData); + Assert.Equal("IM Free shipping", owner.OwnedEntity.SupplierData.AdditionalSupplierData); + } + + protected abstract class MyContext26592Base : DbContext + { + protected MyContext26592Base(DbContextOptions options) + : base(options) + { + } + + public DbSet Companies { get; set; } + public DbSet Owners { get; set; } + + public void Seed() + { + Add(new Company + { + Name = "Acme Inc.", + CustomerData = new CustomerData + { + AdditionalCustomerData = "Regular" + }, + SupplierData = new SupplierData + { + AdditionalSupplierData = "Free shipping" + } + }); + + Add(new Owner + { + Name = "Owner1", + OwnedEntity = new IntermediateOwnedEntity + { + Name = "Intermediate1", + CustomerData = new CustomerData + { + AdditionalCustomerData = "IM Regular" + }, + SupplierData = new SupplierData + { + AdditionalSupplierData = "IM Free shipping" + } + } + }); + + SaveChanges(); + } + } + + protected class Company + { + public int Id { get; set; } + public string Name { get; set; } + public CustomerData CustomerData { get; set; } + public SupplierData SupplierData { get; set; } + } + + [Owned] + protected class CustomerData + { + public int Id { get; set; } + public string AdditionalCustomerData { get; set; } + } + + [Owned] + protected class SupplierData + { + public int Id { get; set; } + public string AdditionalSupplierData { get; set; } + } + + protected class Owner + { + public int Id { get; set; } + public string Name { get; set; } + public IntermediateOwnedEntity OwnedEntity { get; set; } + } + + [Owned] + protected class IntermediateOwnedEntity + { + public int Id { get; set; } + public string Name { get; set; } + public CustomerData CustomerData { get; set; } + public SupplierData SupplierData { get; set; } + } } } diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/OwnedEntityQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/OwnedEntityQuerySqlServerTest.cs index e30e0f64966..fb3ed254543 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/OwnedEntityQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/OwnedEntityQuerySqlServerTest.cs @@ -99,5 +99,44 @@ FROM [Location25680] AS [l] WHERE [l].[Id] = @__id_0 ORDER BY [l].[Id]"); } + + public override async Task Owned_reference_mapped_to_different_table_updated_correctly_after_subquery_pushdown(bool async) + { + await base.Owned_reference_mapped_to_different_table_updated_correctly_after_subquery_pushdown(async); + + AssertSql( + @"@__p_0='10' + +SELECT [t].[Id], [t].[Name], [t].[CompanyId], [t].[AdditionalCustomerData], [t].[Id0], [s].[CompanyId], [s].[AdditionalSupplierData], [s].[Id] +FROM ( + SELECT TOP(@__p_0) [c].[Id], [c].[Name], [c0].[CompanyId], [c0].[AdditionalCustomerData], [c0].[Id] AS [Id0] + FROM [Companies] AS [c] + LEFT JOIN [CustomerData] AS [c0] ON [c].[Id] = [c0].[CompanyId] + WHERE [c0].[CompanyId] IS NOT NULL + ORDER BY [c].[Id] +) AS [t] +LEFT JOIN [SupplierData] AS [s] ON [t].[Id] = [s].[CompanyId] +ORDER BY [t].[Id]"); + } + + public override async Task Owned_reference_mapped_to_different_table_nested_updated_correctly_after_subquery_pushdown(bool async) + { + await base.Owned_reference_mapped_to_different_table_nested_updated_correctly_after_subquery_pushdown(async); + + AssertSql( + @"@__p_0='10' + +SELECT [t].[Id], [t].[Name], [t].[OwnerId], [t].[Id0], [t].[Name0], [t].[IntermediateOwnedEntityOwnerId], [t].[AdditionalCustomerData], [t].[Id1], [i1].[IntermediateOwnedEntityOwnerId], [i1].[AdditionalSupplierData], [i1].[Id] +FROM ( + SELECT TOP(@__p_0) [o].[Id], [o].[Name], [i].[OwnerId], [i].[Id] AS [Id0], [i].[Name] AS [Name0], [i0].[IntermediateOwnedEntityOwnerId], [i0].[AdditionalCustomerData], [i0].[Id] AS [Id1] + FROM [Owners] AS [o] + LEFT JOIN [IntermediateOwnedEntity] AS [i] ON [o].[Id] = [i].[OwnerId] + LEFT JOIN [IM_CustomerData] AS [i0] ON [i].[OwnerId] = [i0].[IntermediateOwnedEntityOwnerId] + WHERE [i0].[IntermediateOwnedEntityOwnerId] IS NOT NULL + ORDER BY [o].[Id] +) AS [t] +LEFT JOIN [IM_SupplierData] AS [i1] ON [t].[OwnerId] = [i1].[IntermediateOwnedEntityOwnerId] +ORDER BY [t].[Id]"); + } } }