Skip to content

Commit

Permalink
Set operations: finalization
Browse files Browse the repository at this point in the history 
* Include/nav rewriting is now supported.
* We now push down to subquery when OrderBy, Take or Skip are applied to set operations, to avoid using the hack where ColumnExpression with no table alias was used. #16244 was opened to track for post-3.0.
* Added missing support for union over subselect projection mappings.
* Added Other type to SetOperationType so that providers can define extra set operations (e.g. PostgreSQL `INTERSECT ALL`, `EXCEPT ALL`).

Completes #6812
Fixes #13196
Fixes #16065
Fixes #16165
  • Loading branch information
@roji
roji committed Jun 25, 2019
1 parent 7e0e01e commit 68ae5a8
Show file tree
Hide file tree
Showing 4 changed files with 352 additions and 186 deletions.
251 changes: 142 additions & 109 deletions src/EFCore.Relational/Query/Pipeline/SqlExpressions/SelectExpression.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using JetBrains.Annotations;
using Microsoft.EntityFrameworkCore.Diagnostics;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Internal;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
using Microsoft.EntityFrameworkCore.Query.Internal;
Expand Down Expand Up @@ -262,9 +263,8 @@ public void ApplyPredicate(SqlExpression expression)

public void ApplyOrdering(OrderingExpression orderingExpression)
{
if (IsDistinct
|| Limit != null
|| Offset != null)
// TODO: We should not be pushing down set operations, see #16244
if (IsDistinct || Limit != null || Offset != null || IsSetOperation)
{
orderingExpression = orderingExpression.Update(
new SqlRemappingVisitor(PushdownIntoSubquery())
Expand All @@ -285,7 +285,8 @@ public void AppendOrdering(OrderingExpression orderingExpression)

public void ApplyLimit(SqlExpression sqlExpression)
{
if (Limit != null)
// TODO: We should not be pushing down set operations, see #16244
if (Limit != null || IsSetOperation)
{
PushdownIntoSubquery();
}
Expand All @@ -295,8 +296,8 @@ public void ApplyLimit(SqlExpression sqlExpression)

public void ApplyOffset(SqlExpression sqlExpression)
{
if (Limit != null
|| Offset != null)
// TODO: We should not be pushing down set operations, see #16244
if (Limit != null || Offset != null || IsSetOperation)
{
PushdownIntoSubquery();
}
Expand Down Expand Up @@ -370,11 +371,14 @@ public Expression ApplySetOperation(
select1._projectionMapping = new Dictionary<ProjectionMember, Expression>(_projectionMapping);
_projectionMapping.Clear();

select1._identifyingProjection.AddRange(_identifyingProjection);
_identifyingProjection.Clear();

var select2 = otherSelectExpression;

if (_projection.Any())
{
throw new NotImplementedException("Set operation on SelectExpression with populated _projection");
throw new NotSupportedException("Can't process set operations after client evaluation, consider moving the operation before the last Select() call (see issue #16243)");
}
else
{
Expand All @@ -394,98 +398,17 @@ public Expression ApplySetOperation(
if (joinedMapping.Value1 is EntityProjectionExpression entityProjection1
&& joinedMapping.Value2 is EntityProjectionExpression entityProjection2)
{
var propertyExpressions = new Dictionary<IProperty, ColumnExpression>();

if (entityProjection1.EntityType == entityProjection2.EntityType)
{
foreach (var property in GetAllPropertiesInHierarchy(entityProjection1.EntityType))
{
propertyExpressions[property] = AddSetOperationColumnProjections(
property,
select1, entityProjection1.GetProperty(property),
select2, entityProjection2.GetProperty(property));
}

_projectionMapping[joinedMapping.Key] = new EntityProjectionExpression(entityProjection1.EntityType, propertyExpressions);
continue;
}

// We're doing a set operation over two different entity types (within the same hierarchy).
// Since both sides of the set operations must produce the same result shape, find the
// closest common ancestor and load all the columns for that, adding null projections where
// necessary. Note this means we add null projections for properties which neither sibling
// actually needs, since the shaper doesn't know that only those sibling types will be coming
// back.
var commonParentEntityType = entityProjection1.EntityType.GetClosestCommonParent(entityProjection2.EntityType);

if (commonParentEntityType == null)
{
throw new NotSupportedException(RelationalStrings.SetOperationNotWithinEntityTypeHierarchy);
}

var properties1 = GetAllPropertiesInHierarchy(entityProjection1.EntityType).ToArray();
var properties2 = GetAllPropertiesInHierarchy(entityProjection2.EntityType).ToArray();

foreach (var property in properties1.Intersect(properties2))
{
propertyExpressions[property] = AddSetOperationColumnProjections(
property,
select1, entityProjection1.GetProperty(property),
select2, entityProjection2.GetProperty(property));
}

foreach (var property in properties1.Except(properties2))
{
propertyExpressions[property] = AddSetOperationColumnProjections(
property,
select1,entityProjection1.GetProperty(property),
select2, null);
}

foreach (var property in properties2.Except(properties1))
{
propertyExpressions[property] = AddSetOperationColumnProjections(
property,
select1, null,
select2, entityProjection2.GetProperty(property));
}

foreach (var property in GetAllPropertiesInHierarchy(commonParentEntityType)
.Except(properties1).Except(properties2))
{
propertyExpressions[property] = AddSetOperationColumnProjections(
property,
select1, null,
select2, null);
}

_projectionMapping[joinedMapping.Key] = new EntityProjectionExpression(commonParentEntityType, propertyExpressions);

if (commonParentEntityType != entityProjection1.EntityType)
{
if (!(shaperExpression.RemoveConvert() is EntityShaperExpression entityShaperExpression))
{
throw new Exception("Non-entity shaper expression while handling set operation over siblings.");
}

shaperExpression = new EntityShaperExpression(
commonParentEntityType, entityShaperExpression.ValueBufferExpression, entityShaperExpression.Nullable);
}

continue;
HandleEntityMapping(joinedMapping.Key, select1, entityProjection1, select2, entityProjection2);
continue;
}

if (joinedMapping.Value1 is ColumnExpression innerColumn1
&& joinedMapping.Value2 is ColumnExpression innerColumn2)
if (joinedMapping.Value1 is ColumnExpression && joinedMapping.Value2 is ColumnExpression
|| joinedMapping.Value1 is SubSelectExpression && joinedMapping.Value2 is SubSelectExpression)
{
// The actual columns may actually be different, but we don't care as long as the type and alias
// coming out of the two operands are the same
var alias = joinedMapping.Key.LastMember?.Name;
var index = select1.AddToProjection(innerColumn1, alias);
var projectionExpression1 = select1._projection[index];
select2.AddToProjection(innerColumn2, alias);
var outerColumn = new ColumnExpression(projectionExpression1, select1, IsNullableProjection(projectionExpression1));
_projectionMapping[joinedMapping.Key] = outerColumn;
HandleColumnMapping(
joinedMapping.Key,
select1, (SqlExpression)joinedMapping.Value1,
select2, (SqlExpression)joinedMapping.Value2);
continue;
}

Expand All @@ -504,7 +427,97 @@ public Expression ApplySetOperation(
SetOperationType = setOperationType;
return shaperExpression;

static ColumnExpression AddSetOperationColumnProjections(
void HandleEntityMapping(
ProjectionMember projectionMember,
SelectExpression select1, EntityProjectionExpression projection1,
SelectExpression select2, EntityProjectionExpression projection2)
{
var propertyExpressions = new Dictionary<IProperty, ColumnExpression>();

if (projection1.EntityType == projection2.EntityType)
{
foreach (var property in GetAllPropertiesInHierarchy(projection1.EntityType))
{
propertyExpressions[property] = AddSetOperationColumnProjections(
property,
select1, projection1.GetProperty(property),
select2, projection2.GetProperty(property));
}

_projectionMapping[projectionMember] = new EntityProjectionExpression(projection1.EntityType, propertyExpressions);
return;
}

// We're doing a set operation over two different entity types (within the same hierarchy).
// Since both sides of the set operations must produce the same result shape, find the
// closest common ancestor and load all the columns for that, adding null projections where
// necessary. Note this means we add null projections for properties which neither sibling
// actually needs, since the shaper doesn't know that only those sibling types will be coming
// back.
var commonParentEntityType = projection1.EntityType.GetClosestCommonParent(projection2.EntityType);

if (commonParentEntityType == null)
{
throw new NotSupportedException(RelationalStrings.SetOperationNotWithinEntityTypeHierarchy);
}

var properties1 = GetAllPropertiesInHierarchy(projection1.EntityType).ToArray();
var properties2 = GetAllPropertiesInHierarchy(projection2.EntityType).ToArray();

foreach (var property in properties1.Intersect(properties2))
{
propertyExpressions[property] = AddSetOperationColumnProjections(
property,
select1, projection1.GetProperty(property),
select2, projection2.GetProperty(property));
}

foreach (var property in properties1.Except(properties2))
{
propertyExpressions[property] = AddSetOperationColumnProjections(
property,
select1,projection1.GetProperty(property),
select2, null);
}

foreach (var property in properties2.Except(properties1))
{
propertyExpressions[property] = AddSetOperationColumnProjections(
property,
select1, null,
select2, projection2.GetProperty(property));
}

foreach (var property in GetAllPropertiesInHierarchy(commonParentEntityType)
.Except(properties1).Except(properties2))
{
propertyExpressions[property] = AddSetOperationColumnProjections(
property,
select1, null,
select2, null);
}

_projectionMapping[projectionMember] = new EntityProjectionExpression(commonParentEntityType, propertyExpressions);

if (commonParentEntityType != projection1.EntityType)
{
// The first source has been up-cast by the set operation, so we also need to change the shaper expression.
var entityShaperExpression =
shaperExpression as EntityShaperExpression ?? (
shaperExpression is UnaryExpression unary
&& unary.NodeType == ExpressionType.Convert
&& unary.Type == commonParentEntityType.ClrType
? unary.Operand as EntityShaperExpression : null);

if (entityShaperExpression != null)
{
shaperExpression = new EntityShaperExpression(
commonParentEntityType, entityShaperExpression.ValueBufferExpression, entityShaperExpression.Nullable);
}
}
}

ColumnExpression AddSetOperationColumnProjections(
IProperty property,
SelectExpression select1, ColumnExpression column1,
SelectExpression select2, ColumnExpression column2)
Expand All @@ -531,8 +544,29 @@ static ColumnExpression AddSetOperationColumnProjections(

var projectionExpression = select1._projection[select1._projection.Count - 1];
var outerColumn = new ColumnExpression(projectionExpression, select1, IsNullableProjection(projectionExpression));

if (select1._identifyingProjection.Contains(column1))
{
_identifyingProjection.Add(outerColumn);
}

return outerColumn;
}

void HandleColumnMapping(
ProjectionMember projectionMember,
SelectExpression select1, SqlExpression innerColumn1,
SelectExpression select2, SqlExpression innerColumn2)
{
// The actual columns may actually be different, but we don't care as long as the type and alias
// coming out of the two operands are the same
var alias = projectionMember.LastMember?.Name;
var index = select1.AddToProjection(innerColumn1, alias);
var projectionExpression1 = select1._projection[index];
select2.AddToProjection(innerColumn2, alias);
var outerColumn = new ColumnExpression(projectionExpression1, select1, IsNullableProjection(projectionExpression1));
_projectionMapping[projectionMember] = outerColumn;
}
}

public IDictionary<SqlExpression, ColumnExpression> PushdownIntoSubquery()
Expand Down Expand Up @@ -680,11 +714,10 @@ public RelationalCollectionShaperExpression ApplyCollectionJoin(int collectionId
var (outer, inner) = TryExtractJoinKey(innerSelectExpression);
if (outer != null)
{
if (IsDistinct
|| Limit != null
|| Offset != null)
if (IsDistinct || Limit != null || Offset != null || IsSetOperation)
{
outer = new SqlRemappingVisitor(PushdownIntoSubquery()).Remap(outer);
var pushdown = PushdownIntoSubquery();
outer = new SqlRemappingVisitor(pushdown).Remap(outer);
}

if (innerSelectExpression.Offset != null
Expand Down Expand Up @@ -883,9 +916,7 @@ public void AddInnerJoin(SelectExpression innerSelectExpression, SqlExpression j

public void AddLeftJoin(SelectExpression innerSelectExpression, SqlExpression joinPredicate, Type transparentIdentifierType)
{
if (Limit != null
|| Offset != null
|| IsDistinct)
if (Limit != null || Offset != null || IsDistinct || IsSetOperation)
{
joinPredicate = new SqlRemappingVisitor(PushdownIntoSubquery())
.Remap(joinPredicate);
Expand Down Expand Up @@ -933,10 +964,7 @@ public void AddLeftJoin(SelectExpression innerSelectExpression, SqlExpression jo

public void AddCrossJoin(SelectExpression innerSelectExpression, Type transparentIdentifierType)
{
if (Limit != null
|| Offset != null
|| IsDistinct
|| Predicate != null)
if (Limit != null || Offset != null || IsDistinct || Predicate != null || IsSetOperation)
{
PushdownIntoSubquery();
}
Expand Down Expand Up @@ -1304,7 +1332,12 @@ public enum SetOperationType
/// <summary>
/// Represents an SQL EXCEPT set operation.
/// </summary>
Except = 4
Except = 4,

/// <summary>
/// Represents a custom, provider-specific set operation.
/// </summary>
Other = 9999
}
}

Loading

0 comments on commit 68ae5a8

Please sign in to comment.