Query: Perform join operations when one of the source has client eval (…

…#21201)

Resolves #19247
Resolves #17763
Lays ground work for #20291
Required for #20892

src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs

@@ -548,11 +548,13 @@ protected override ShapedQueryExpression TranslateJoin(
  innerKeySelector,
  transparentIdentifierType);
 
+#pragma warning disable CS0618 // Type or member is obsolete See issue#21200
  return TranslateResultSelectorForJoin(
  outer,
  resultSelector,
  inner.ShaperExpression,
  transparentIdentifierType);
+#pragma warning restore CS0618 // Type or member is obsolete
  }
 
  private (LambdaExpression OuterKeySelector, LambdaExpression InnerKeySelector) ProcessJoinKeySelector(
@@ -703,11 +705,13 @@ protected override ShapedQueryExpression TranslateLeftJoin(
  innerKeySelector,
  transparentIdentifierType);
 
+#pragma warning disable CS0618 // Type or member is obsolete See issue#21200
  return TranslateResultSelectorForJoin(
  outer,
  resultSelector,
  MarkShaperNullable(inner.ShaperExpression),
  transparentIdentifierType);
+#pragma warning restore CS0618 // Type or member is obsolete
  }
 
  /// <summary>
@@ -961,11 +965,13 @@ protected override ShapedQueryExpression TranslateSelectMany(
  ((InMemoryQueryExpression)source.QueryExpression).AddSelectMany(
  (InMemoryQueryExpression)inner.QueryExpression, transparentIdentifierType, defaultIfEmpty);
 
+#pragma warning disable CS0618 // Type or member is obsolete See issue#21200
  return TranslateResultSelectorForJoin(
  source,
  resultSelector,
  innerShaperExpression,
  transparentIdentifierType);
+#pragma warning restore CS0618 // Type or member is obsolete
  }
 
  return null;

src/EFCore.Relational/Query/Internal/RelationalProjectionBindingExpressionVisitor.cs

@@ -9,6 +9,7 @@
 using JetBrains.Annotations;
 using Microsoft.EntityFrameworkCore.Diagnostics;
 using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
 using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
 using Microsoft.EntityFrameworkCore.Storage;
 using Microsoft.EntityFrameworkCore.Utilities;
@@ -27,6 +28,7 @@ public class RelationalProjectionBindingExpressionVisitor : ExpressionVisitor
  private readonly RelationalSqlTranslatingExpressionVisitor _sqlTranslator;
 
  private SelectExpression _selectExpression;
+ private SqlExpression[] _existingProjections;
  private bool _clientEval;
 
  private readonly IDictionary<ProjectionMember, Expression> _projectionMapping
@@ -69,6 +71,8 @@ public virtual Expression Translate([NotNull] SelectExpression selectExpression,
  _clientEval = true;
 
  expandedExpression = _queryableMethodTranslatingExpressionVisitor.ExpandWeakEntities(_selectExpression, expression);
+ _existingProjections = _selectExpression.Projection.Select(e => e.Expression).ToArray();
+ _selectExpression.ClearProjection();
  result = Visit(expandedExpression);
 
  _projectionMapping.Clear();
@@ -115,6 +119,16 @@ public override Expression Visit(Expression expression)
  case ConstantExpression _:
  return expression;
 
+ case ProjectionBindingExpression projectionBindingExpression:
+ if (projectionBindingExpression.Index is int index)
+ {
+ var newIndex = _selectExpression.AddToProjection(_existingProjections[index]);
+
+ return new ProjectionBindingExpression(_selectExpression, newIndex, expression.Type);
+ }
+
+ throw new InvalidOperationException(CoreStrings.TranslationFailed(projectionBindingExpression.Print()));
+
  case ParameterExpression parameterExpression:
  if (parameterExpression.Name?.StartsWith(QueryCompilationContext.QueryParameterPrefix, StringComparison.Ordinal) == true)
  {
@@ -227,10 +241,29 @@ protected override Expression VisitExtension(Expression extensionExpression)
 
  if (extensionExpression is EntityShaperExpression entityShaperExpression)
  {
+ // TODO: Make this easier to understand some day.
  EntityProjectionExpression entityProjectionExpression;
  if (entityShaperExpression.ValueBufferExpression is ProjectionBindingExpression projectionBindingExpression)
  {
  VerifySelectExpression(projectionBindingExpression);
+ // If projectionBinding is not mapped then SelectExpression has client projection
+ // Hence force client eval
+ if (projectionBindingExpression.ProjectionMember == null)
+ {
+ if (_clientEval)
+ {
+ var indexMap = new Dictionary<IProperty, int>();
+ foreach (var item in projectionBindingExpression.IndexMap)
+ {
+ indexMap[item.Key] = _selectExpression.AddToProjection(_existingProjections[item.Value]);
+ }
+
+ return entityShaperExpression.Update(new ProjectionBindingExpression(_selectExpression, indexMap));
+ }
+
+ return null;
+ }
+
  entityProjectionExpression = (EntityProjectionExpression)_selectExpression.GetMappedProjection(
  projectionBindingExpression.ProjectionMember);
  }

src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Linq.Expressions;
+using System.Reflection;
 using JetBrains.Annotations;
 using Microsoft.EntityFrameworkCore.Diagnostics;
 using Microsoft.EntityFrameworkCore.Infrastructure;
@@ -535,18 +536,11 @@ protected override ShapedQueryExpression TranslateJoin(
  var joinPredicate = CreateJoinPredicate(outer, outerKeySelector, inner, innerKeySelector);
  if (joinPredicate != null)
  {
- var transparentIdentifierType = TransparentIdentifierFactory.Create(
- resultSelector.Parameters[0].Type,
- resultSelector.Parameters[1].Type);
-
- ((SelectExpression)outer.QueryExpression).AddInnerJoin(
- (SelectExpression)inner.QueryExpression, joinPredicate, transparentIdentifierType);
-
- return TranslateResultSelectorForJoin(
- outer,
- resultSelector,
- inner.ShaperExpression,
- transparentIdentifierType);
+ var outerSelectExpression = (SelectExpression)outer.QueryExpression;
+ var outerShaperExpression = outerSelectExpression.AddInnerJoin(inner, joinPredicate, outer.ShaperExpression);
+ outer = outer.UpdateShaperExpression(outerShaperExpression);
+
+ return TranslateTwoParameterSelector(outer, resultSelector);
  }
 
  return null;
@@ -567,18 +561,12 @@ protected override ShapedQueryExpression TranslateLeftJoin(
  var joinPredicate = CreateJoinPredicate(outer, outerKeySelector, inner, innerKeySelector);
  if (joinPredicate != null)
  {
- var transparentIdentifierType = TransparentIdentifierFactory.Create(
- resultSelector.Parameters[0].Type,
- resultSelector.Parameters[1].Type);
-
- ((SelectExpression)outer.QueryExpression).AddLeftJoin(
- (SelectExpression)inner.QueryExpression, joinPredicate, transparentIdentifierType);
-
- return TranslateResultSelectorForJoin(
- outer,
- resultSelector,
- MarkShaperNullable(inner.ShaperExpression),
- transparentIdentifierType);
+ var outerSelectExpression = (SelectExpression)outer.QueryExpression;
+ inner = inner.UpdateShaperExpression(MarkShaperNullable(inner.ShaperExpression));
+ var outerShaperExpression = outerSelectExpression.AddLeftJoin(inner, joinPredicate, outer.ShaperExpression);
+ outer = outer.UpdateShaperExpression(outerShaperExpression);
+
+ return TranslateTwoParameterSelector(outer, resultSelector);
  }
 
  return null;
@@ -880,28 +868,14 @@ protected override ShapedQueryExpression TranslateSelectMany(
  var collectionSelectorBody = RemapLambdaBody(source, newCollectionSelector);
  if (Visit(collectionSelectorBody) is ShapedQueryExpression inner)
  {
- var transparentIdentifierType = TransparentIdentifierFactory.Create(
- resultSelector.Parameters[0].Type,
- resultSelector.Parameters[1].Type);
-
- var innerShaperExpression = inner.ShaperExpression;
- if (defaultIfEmpty)
- {
- ((SelectExpression)source.QueryExpression).AddOuterApply(
- (SelectExpression)inner.QueryExpression, transparentIdentifierType);
- innerShaperExpression = MarkShaperNullable(innerShaperExpression);
- }
- else
- {
- ((SelectExpression)source.QueryExpression).AddCrossApply(
- (SelectExpression)inner.QueryExpression, transparentIdentifierType);
- }
-
- return TranslateResultSelectorForJoin(
- source,
- resultSelector,
- innerShaperExpression,
- transparentIdentifierType);
+ var innerSelectExpression = (SelectExpression)source.QueryExpression;
+ var shaper = defaultIfEmpty
+ ? innerSelectExpression.AddOuterApply(
+ inner.UpdateShaperExpression(MarkShaperNullable(inner.ShaperExpression)),
+ source.ShaperExpression)
+ : innerSelectExpression.AddCrossApply(inner, source.ShaperExpression);
+
+ return TranslateTwoParameterSelector(source.UpdateShaperExpression(shaper), resultSelector);
  }
  }
  else
@@ -917,18 +891,10 @@ protected override ShapedQueryExpression TranslateSelectMany(
  }
  }
 
- var transparentIdentifierType = TransparentIdentifierFactory.Create(
- resultSelector.Parameters[0].Type,
- resultSelector.Parameters[1].Type);
+ var innerSelectExpression = (SelectExpression)source.QueryExpression;
+ var shaper = innerSelectExpression.AddCrossJoin(inner, source.ShaperExpression);
 
- ((SelectExpression)source.QueryExpression).AddCrossJoin(
- (SelectExpression)inner.QueryExpression, transparentIdentifierType);
-
- return TranslateResultSelectorForJoin(
- source,
- resultSelector,
- inner.ShaperExpression,
- transparentIdentifierType);
+ return TranslateTwoParameterSelector(source.UpdateShaperExpression(shaper), resultSelector);
  }
  }
 
@@ -1358,7 +1324,7 @@ private Expression TryExpand(Expression source, MemberIdentity member)
  makeNullable);
 
  var joinPredicate = _sqlTranslator.Translate(Expression.Equal(outerKey, innerKey));
- _selectExpression.AddLeftJoin(innerSelectExpression, joinPredicate, null);
+ _selectExpression.AddLeftJoin(innerSelectExpression, joinPredicate);
  var leftJoinTable = ((LeftJoinExpression)_selectExpression.Tables.Last()).Table;
  innerShaper = new RelationalEntityShaperExpression(
  targetEntityType,
@@ -1415,6 +1381,30 @@ private static IDictionary<IProperty, ColumnExpression> GetPropertyExpressionsFr
  }
  }
 
+ private ShapedQueryExpression TranslateTwoParameterSelector(ShapedQueryExpression source, LambdaExpression resultSelector)
+ {
+ var transparentIdentifierType = source.ShaperExpression.Type;
+ var transparentIdentifierParameter = Expression.Parameter(transparentIdentifierType);
+
+ Expression original1 = resultSelector.Parameters[0];
+ var replacement1 = AccessField(transparentIdentifierType, transparentIdentifierParameter, "Outer");
+ Expression original2 = resultSelector.Parameters[1];
+ var replacement2 = AccessField(transparentIdentifierType, transparentIdentifierParameter, "Inner");
+ var newResultSelector = Expression.Lambda(
+ new ReplacingExpressionVisitor(
+ new[] { original1, original2 }, new[] { replacement1, replacement2 })
+ .Visit(resultSelector.Body),
+ transparentIdentifierParameter);
+
+ return TranslateSelect(source, newResultSelector);
+ }
+
+ private static Expression AccessField(
+ Type transparentIdentifierType,
+ Expression targetExpression,
+ string fieldName)
+ => Expression.Field(targetExpression, transparentIdentifierType.GetTypeInfo().GetDeclaredField(fieldName));
+
  private ShapedQueryExpression AggregateResultShaper(
  ShapedQueryExpression source, Expression projection, bool throwWhenEmpty, Type resultType)
  {

src/EFCore.Relational/Query/SqlExpressionFactory.cs

@@ -846,7 +846,7 @@ private void AddInnerJoin(
  {
  var joinPredicate = GenerateJoinPredicate(selectExpression, foreignKey, table, skipInnerJoins, out var innerSelect);
 
- selectExpression.AddInnerJoin(innerSelect, joinPredicate, null);
+ selectExpression.AddInnerJoin(innerSelect, joinPredicate);
  }
 
  private SqlExpression GenerateJoinPredicate(