Skip to content

Commit

Permalink
Query: Implement collection materialization in async
Browse files Browse the repository at this point in the history 
Part of #15611
  • Loading branch information
@smitpatel
smitpatel committed Jun 24, 2019
1 parent d7e85ed commit 300ca50
Show file tree
Hide file tree
Showing 15 changed files with 999 additions and 1,151 deletions.
34 changes: 25 additions & 9 deletions src/EFCore.Relational/Query/Pipeline/AsyncQueryingEnumerable.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ private class AsyncQueryingEnumerable<T> : IAsyncEnumerable<T>
{
private readonly RelationalQueryContext _relationalQueryContext;
private readonly SelectExpression _selectExpression;
private readonly Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> _shaper;
private readonly Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> _shaper;
private readonly IQuerySqlGeneratorFactory _querySqlGeneratorFactory;
private readonly Type _contextType;
private readonly IDiagnosticsLogger<DbLoggerCategory.Query> _logger;
Expand All @@ -33,7 +33,7 @@ public AsyncQueryingEnumerable(
ISqlExpressionFactory sqlExpressionFactory,
IParameterNameGeneratorFactory parameterNameGeneratorFactory,
SelectExpression selectExpression,
Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> shaper,
Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> shaper,
Type contextType,
IDiagnosticsLogger<DbLoggerCategory.Query> logger)
{
Expand All @@ -57,7 +57,7 @@ private sealed class AsyncEnumerator : IAsyncEnumerator<T>
private ResultCoordinator _resultCoordinator;
private readonly RelationalQueryContext _relationalQueryContext;
private readonly SelectExpression _selectExpression;
private readonly Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> _shaper;
private readonly Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> _shaper;
private readonly IQuerySqlGeneratorFactory _querySqlGeneratorFactory;
private readonly Type _contextType;
private readonly IDiagnosticsLogger<DbLoggerCategory.Query> _logger;
Expand Down Expand Up @@ -136,13 +136,29 @@ public async ValueTask<bool> MoveNextAsync()
_resultCoordinator = new ResultCoordinator();
}

var hasNext = _resultCoordinator.HasNext ?? await _dataReader.ReadAsync(_cancellationToken);
_resultCoordinator.HasNext = null;
var hasNext = _resultCoordinator.HasNext ?? await _dataReader.ReadAsync();
Current = default;

Current
= hasNext
? _shaper(_relationalQueryContext, _dataReader.DbDataReader, _indexMap, _resultCoordinator)
: default;
if (hasNext)
{
while (true)
{
_resultCoordinator.ResultReady = true;
_resultCoordinator.HasNext = null;
Current = _shaper(_relationalQueryContext, _dataReader.DbDataReader, Current, _indexMap, _resultCoordinator);
if (_resultCoordinator.ResultReady)
{
break;
}

if (!await _dataReader.ReadAsync())
{
_resultCoordinator.HasNext = false;

break;
}
}
}

return hasNext;
}
Expand Down
258 changes: 146 additions & 112 deletions src/EFCore.Relational/Query/Pipeline/IncludeCompilingExpressionVisitor.cs
View file

Large diffs are not rendered by default.

32 changes: 24 additions & 8 deletions src/EFCore.Relational/Query/Pipeline/QueryingEnumerable.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ private class QueryingEnumerable<T> : IEnumerable<T>
{
private readonly RelationalQueryContext _relationalQueryContext;
private readonly SelectExpression _selectExpression;
private readonly Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> _shaper;
private readonly Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> _shaper;
private readonly IQuerySqlGeneratorFactory _querySqlGeneratorFactory;
private readonly Type _contextType;
private readonly IDiagnosticsLogger<DbLoggerCategory.Query> _logger;
Expand All @@ -31,7 +31,7 @@ public QueryingEnumerable(RelationalQueryContext relationalQueryContext,
ISqlExpressionFactory sqlExpressionFactory,
IParameterNameGeneratorFactory parameterNameGeneratorFactory,
SelectExpression selectExpression,
Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> shaper,
Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> shaper,
Type contextType,
IDiagnosticsLogger<DbLoggerCategory.Query> logger)
{
Expand All @@ -55,7 +55,7 @@ private sealed class Enumerator : IEnumerator<T>
private ResultCoordinator _resultCoordinator;
private readonly RelationalQueryContext _relationalQueryContext;
private readonly SelectExpression _selectExpression;
private readonly Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> _shaper;
private readonly Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> _shaper;
private readonly IQuerySqlGeneratorFactory _querySqlGeneratorFactory;
private readonly Type _contextType;
private readonly IDiagnosticsLogger<DbLoggerCategory.Query> _logger;
Expand Down Expand Up @@ -132,12 +132,28 @@ public bool MoveNext()
}

var hasNext = _resultCoordinator.HasNext ?? _dataReader.Read();
_resultCoordinator.HasNext = null;
Current = default;

Current
= hasNext
? _shaper(_relationalQueryContext, _dataReader.DbDataReader, _indexMap, _resultCoordinator)
: default;
if (hasNext)
{
while (true)
{
_resultCoordinator.ResultReady = true;
_resultCoordinator.HasNext = null;
Current = _shaper(_relationalQueryContext, _dataReader.DbDataReader, Current, _indexMap, _resultCoordinator);
if (_resultCoordinator.ResultReady)
{
break;
}

if (!_dataReader.Read())
{
_resultCoordinator.HasNext = false;

break;
}
}
}

return hasNext;
}
Expand Down
75 changes: 58 additions & 17 deletions src/EFCore.Relational/Query/Pipeline/RelationalCollectionShaperExpression.cs
View file
Original file line number Diff line number Diff line change
@@ -1,47 +1,88 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Linq.Expressions;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Query.Expressions.Internal;
using Microsoft.EntityFrameworkCore.Query.Internal;
using Microsoft.EntityFrameworkCore.Query.Pipeline;

namespace Microsoft.EntityFrameworkCore.Relational.Query.Pipeline
{
public class RelationalCollectionShaperExpression : CollectionShaperExpression
public class RelationalCollectionShaperExpression : Expression, IPrintable
{
public RelationalCollectionShaperExpression(
int collectionIndex,
Expression outerKeySelector,
Expression innerKeySelector,
int collectionId,
Expression parentIdentifier,
Expression outerIdentifier,
Expression selfIdentifier,
Expression innerShaper,
INavigation navigation)
: base(null, innerShaper, navigation)
{
CollectionIndex = collectionIndex;
OuterKeySelector = outerKeySelector;
InnerKeySelector = innerKeySelector;
CollectionId = collectionId;
ParentIdentifier = parentIdentifier;
OuterIdentifier = outerIdentifier;
SelfIdentifier = selfIdentifier;
InnerShaper = innerShaper;
Navigation = navigation;
}

public int CollectionIndex { get; }
public Expression OuterKeySelector { get; }
public Expression InnerKeySelector { get; }
public int CollectionId { get; }
public Expression ParentIdentifier { get; }
public Expression OuterIdentifier { get; }
public Expression SelfIdentifier { get; }
public Expression InnerShaper { get; }
public INavigation Navigation { get; }

public override Type Type => typeof(IEnumerable<>).MakeGenericType(InnerShaper.Type);
public override ExpressionType NodeType => ExpressionType.Extension;

protected override Expression VisitChildren(ExpressionVisitor visitor)
{
// Projection is always null so we do not need to visit it.
var outerKeySelector = visitor.Visit(OuterKeySelector);
var innerKeySelector = visitor.Visit(InnerKeySelector);
var parentIdentifier = visitor.Visit(ParentIdentifier);
var outerIdentifier = visitor.Visit(OuterIdentifier);
var selfIdentifier = visitor.Visit(SelfIdentifier);
var innerShaper = visitor.Visit(InnerShaper);

return Update(outerKeySelector, innerKeySelector, innerShaper);
return Update(parentIdentifier, outerIdentifier, selfIdentifier, innerShaper);
}

public RelationalCollectionShaperExpression Update(
Expression outerKeySelector, Expression innerKeySelector, Expression innerShaper)
Expression parentIdentifier, Expression outerIdentifier, Expression selfIdentifier, Expression innerShaper)
{
return outerKeySelector != OuterKeySelector || innerKeySelector != InnerKeySelector || innerShaper != InnerShaper
? new RelationalCollectionShaperExpression(CollectionIndex, outerKeySelector, innerKeySelector, innerShaper, Navigation)
return parentIdentifier != ParentIdentifier
|| outerIdentifier != OuterIdentifier
|| selfIdentifier != SelfIdentifier
|| innerShaper != InnerShaper
? new RelationalCollectionShaperExpression(
CollectionId, parentIdentifier, outerIdentifier, selfIdentifier, innerShaper, Navigation)
: this;
}

public void Print(ExpressionPrinter expressionPrinter)
{
expressionPrinter.StringBuilder.AppendLine("RelationalCollectionShaper:");
using (expressionPrinter.StringBuilder.Indent())
{
expressionPrinter.StringBuilder.AppendLine($"CollectionId: {CollectionId}");
expressionPrinter.StringBuilder.Append("ParentIdentifier:");
expressionPrinter.Visit(ParentIdentifier);
expressionPrinter.StringBuilder.AppendLine();
expressionPrinter.StringBuilder.Append("OuterIdentifier:");
expressionPrinter.Visit(OuterIdentifier);
expressionPrinter.StringBuilder.AppendLine();
expressionPrinter.StringBuilder.Append("SelfIdentifier:");
expressionPrinter.Visit(SelfIdentifier);
expressionPrinter.StringBuilder.AppendLine();
expressionPrinter.StringBuilder.Append("InnerShaper:");
expressionPrinter.Visit(InnerShaper);
expressionPrinter.StringBuilder.AppendLine();
expressionPrinter.StringBuilder.AppendLine($"Navigation: {Navigation.Name}");

}
}
}
}
10 changes: 0 additions & 10 deletions src/EFCore.Relational/Query/Pipeline/RelationalProjectionBindingRemovingExpressionVisitor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,16 +106,6 @@ protected override Expression VisitExtension(Expression extensionExpression)
projectionBindingExpression.Type);
}

if (extensionExpression is RelationalCollectionShaperExpression relationalCollectionShaperExpression)
{
return new RelationalCollectionShaperExpression(
relationalCollectionShaperExpression.CollectionIndex,
Visit(relationalCollectionShaperExpression.OuterKeySelector),
Visit(relationalCollectionShaperExpression.InnerKeySelector),
Visit(relationalCollectionShaperExpression.InnerShaper),
relationalCollectionShaperExpression.Navigation);
}

return base.VisitExtension(extensionExpression);
}

Expand Down
58 changes: 43 additions & 15 deletions src/EFCore.Relational/Query/Pipeline/RelationalShapedQueryCompilingExpressionVisitor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.Linq;
using System.Linq.Expressions;
Expand Down Expand Up @@ -42,30 +43,33 @@ public RelationalShapedQueryCompilingExpressionVisitor(

protected override Expression VisitShapedQueryExpression(ShapedQueryExpression shapedQueryExpression)
{
var shaperBody = InjectEntityMaterializer(shapedQueryExpression.ShaperExpression);

var selectExpression = (SelectExpression)shapedQueryExpression.QueryExpression;

var dataReaderParameter = Expression.Parameter(typeof(DbDataReader), "dataReader");
var indexMapParameter = Expression.Parameter(typeof(int[]), "indexMap");
var resultCoordinatorParameter = Expression.Parameter(typeof(ResultCoordinator), "resultCoordinator");
var indexMapParameter = Expression.Parameter(typeof(int[]), "indexMap");

var shaper = new ShaperExpressionProcessingExpressionVisitor(
selectExpression,
dataReaderParameter,
resultCoordinatorParameter,
indexMapParameter)
.Inject(shapedQueryExpression.ShaperExpression);

shaper = InjectEntityMaterializer(shaper);

shaperBody = new RelationalProjectionBindingRemovingExpressionVisitor(selectExpression, dataReaderParameter)
.Visit(shaperBody);
shaperBody = new IncludeCompilingExpressionVisitor(dataReaderParameter, resultCoordinatorParameter, TrackQueryResults)
.Visit(shaperBody);
shaper = new RelationalProjectionBindingRemovingExpressionVisitor(selectExpression, dataReaderParameter)
.Visit(shaper);
shaper = new CollectionMaterializingExpressionVisitor(
dataReaderParameter, resultCoordinatorParameter, TrackQueryResults)
.Visit(shaper);

if (selectExpression.IsNonComposedFromSql())
{
shaperBody = new IndexMapInjectingExpressionVisitor(indexMapParameter).Visit(shaperBody);
shaper = new IndexMapInjectingExpressionVisitor(indexMapParameter).Visit(shaper);
}

var shaperLambda = Expression.Lambda(
shaperBody,
QueryCompilationContext.QueryContextParameter,
dataReaderParameter,
indexMapParameter,
resultCoordinatorParameter);
var shaperLambda = (LambdaExpression)shaper;

return Expression.New(
(Async
Expand Down Expand Up @@ -110,8 +114,32 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp

private class ResultCoordinator
{
public bool ResultReady { get; set; }
public bool? HasNext { get; set; }
public object[] KeyValues { get; set; }
public IList<CollectionMaterializationContext> Collections { get; } = new List<CollectionMaterializationContext>();
public object[] OuterKeyValues { get; set; }
public object[] InnerKeyValues { get; set; }
}

private class CollectionMaterializationContext
{
public CollectionMaterializationContext(object parent, object collection, object[] outerIdentifier)
{
Parent = parent;
Collection = collection;
OuterIdentifier = outerIdentifier;
}

public object Parent { get; }
public object Collection { get; }
public object Current { get; private set; }
public object[] OuterIdentifier { get; }
public object[] SelfIdentifier { get; set; }

public void UpdateCurrent(object current)
{
Current = current;
}
}
}
}
12 changes: 5 additions & 7 deletions src/EFCore.Relational/Query/Pipeline/RelationalShapedQueryOptimizingExpressionVisitors.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,28 +36,26 @@ public override Expression Visit(Expression query)
query = new SqlExpressionOptimizingVisitor(SqlExpressionFactory, UseRelationalNulls).Visit(query);
query = new NullComparisonTransformingExpressionVisitor().Visit(query);

if (query is ShapedQueryExpression shapedQueryExpression)
{
shapedQueryExpression.ShaperExpression
= new ShaperExpressionProcessingExpressionVisitor((SelectExpression)shapedQueryExpression.QueryExpression)
.Inject(shapedQueryExpression.ShaperExpression);
}

return query;
}
}

public class CollectionJoinApplyingExpressionVisitor : ExpressionVisitor
{
private int _collectionId;

protected override Expression VisitExtension(Expression extensionExpression)
{
if (extensionExpression is CollectionShaperExpression collectionShaperExpression)
{
var collectionId = _collectionId++;

var innerShaper = Visit(collectionShaperExpression.InnerShaper);

var selectExpression = (SelectExpression)collectionShaperExpression.Projection.QueryExpression;
return selectExpression.ApplyCollectionJoin(
collectionShaperExpression.Projection.Index.Value,
collectionId,
innerShaper,
collectionShaperExpression.Navigation);
}
Expand Down
Loading

0 comments on commit 300ca50

Please sign in to comment.