From 9067e4371312aafb651c86045353f49029579fa2 Mon Sep 17 00:00:00 2001 From: Smit Patel Date: Fri, 2 Sep 2022 16:20:37 -0700 Subject: [PATCH] Query : Add support for final GroupBy operator Resolves #19929 --- .../Properties/InMemoryStrings.Designer.cs | 6 + .../Properties/InMemoryStrings.resx | 3 + ...yableMethodTranslatingExpressionVisitor.cs | 10 + .../GroupBySingleQueryingEnumerable.cs | 528 +++++++++ .../GroupBySplitQueryingEnumerable.cs | 517 +++++++++ .../Internal/SingleQueryCollectionContext.cs | 17 +- .../Internal/SingleQueryingEnumerable.cs | 12 + .../Internal/SplitQueryCollectionContext.cs | 10 +- .../Internal/SplitQueryDataReaderContext.cs | 6 +- .../Internal/SplitQueryResultCoordinator.cs | 8 + .../RelationalCollectionShaperExpression.cs | 12 +- .../RelationalGroupByResultExpression.cs | 112 ++ ...ocessingExpressionVisitor.ClientMethods.cs | 994 ++++++++++++++++ ...sitor.ShaperProcessingExpressionVisitor.cs | 1014 +---------------- ...alShapedQueryCompilingExpressionVisitor.cs | 145 ++- .../SqlExpressions/SelectExpression.Helper.cs | 10 + .../Query/SqlExpressions/SelectExpression.cs | 168 ++- ...ingExpressionVisitor.ExpressionVisitors.cs | 2 +- ...nExpandingExpressionVisitor.Expressions.cs | 2 +- .../NavigationExpandingExpressionVisitor.cs | 74 +- ...NavigationsCollectionsQueryInMemoryTest.cs | 37 + ...sCollectionsSharedTypeQueryInMemoryTest.cs | 37 + .../NorthwindGroupByQueryInMemoryTest.cs | 52 + ...plexNavigationsCollectionsQueryTestBase.cs | 81 ++ .../Query/NorthwindGroupByQueryTestBase.cs | 180 ++- ...avigationsCollectionsQuerySqlServerTest.cs | 88 ++ ...CollectionsSharedTypeQuerySqlServerTest.cs | 134 +++ ...tionsCollectionsSplitQuerySqlServerTest.cs | 115 ++ .../NorthwindGroupByQuerySqlServerTest.cs | 129 ++- ...avigationsCollectionsQuerySqlServerTest.cs | 88 ++ 30 files changed, 3514 insertions(+), 1077 deletions(-) create mode 100644 src/EFCore.Relational/Query/Internal/GroupBySingleQueryingEnumerable.cs create mode 100644 src/EFCore.Relational/Query/Internal/GroupBySplitQueryingEnumerable.cs create mode 100644 src/EFCore.Relational/Query/RelationalGroupByResultExpression.cs create mode 100644 src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.ClientMethods.cs diff --git a/src/EFCore.InMemory/Properties/InMemoryStrings.Designer.cs b/src/EFCore.InMemory/Properties/InMemoryStrings.Designer.cs index 21cc04b226b..c73b0214b01 100644 --- a/src/EFCore.InMemory/Properties/InMemoryStrings.Designer.cs +++ b/src/EFCore.InMemory/Properties/InMemoryStrings.Designer.cs @@ -79,6 +79,12 @@ public static string UnableToBindMemberToEntityProjection(object? memberType, ob GetString("UnableToBindMemberToEntityProjection", nameof(memberType), nameof(member), nameof(entityType)), memberType, member, entityType); + /// + /// A 'GroupBy' operation which is not composed into aggregate or projection of elements is not supported. + /// + public static string UncomposedGroupByNotSupported + => GetString("UncomposedGroupByNotSupported"); + /// /// Attempted to update or delete an entity that does not exist in the store. /// diff --git a/src/EFCore.InMemory/Properties/InMemoryStrings.resx b/src/EFCore.InMemory/Properties/InMemoryStrings.resx index f914f2fc4a0..418dd03aa23 100644 --- a/src/EFCore.InMemory/Properties/InMemoryStrings.resx +++ b/src/EFCore.InMemory/Properties/InMemoryStrings.resx @@ -149,6 +149,9 @@ Unable to bind '{memberType}' '{member}' to entity projection of '{entityType}'. + + A 'GroupBy' operation which is not composed into aggregate or projection of elements is not supported. + Attempted to update or delete an entity that does not exist in the store. diff --git a/src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs b/src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs index 5f9ff6cba25..ab9061fb928 100644 --- a/src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs +++ b/src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.InMemory.Internal; + namespace Microsoft.EntityFrameworkCore.InMemory.Query.Internal; /// @@ -91,6 +93,14 @@ protected override Expression VisitExtension(Expression extensionExpression) /// protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression) { + if (methodCallExpression.Method.IsGenericMethod + && (methodCallExpression.Method.GetGenericMethodDefinition() == QueryableMethods.GroupByWithKeySelector + || methodCallExpression.Method.GetGenericMethodDefinition() == QueryableMethods.GroupByWithKeyElementSelector)) + { + throw new InvalidOperationException( + CoreStrings.TranslationFailedWithDetails(methodCallExpression.Print(), InMemoryStrings.UncomposedGroupByNotSupported)); + } + if (methodCallExpression.Method.IsGenericMethod && methodCallExpression.Arguments.Count == 1 && methodCallExpression.Arguments[0].Type.TryGetSequenceType() != null diff --git a/src/EFCore.Relational/Query/Internal/GroupBySingleQueryingEnumerable.cs b/src/EFCore.Relational/Query/Internal/GroupBySingleQueryingEnumerable.cs new file mode 100644 index 00000000000..b28bc751315 --- /dev/null +++ b/src/EFCore.Relational/Query/Internal/GroupBySingleQueryingEnumerable.cs @@ -0,0 +1,528 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using Microsoft.EntityFrameworkCore.Internal; + +namespace Microsoft.EntityFrameworkCore.Query.Internal; + +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public class GroupBySingleQueryingEnumerable + : IEnumerable>, IAsyncEnumerable>, IRelationalQueryingEnumerable +{ + private readonly RelationalQueryContext _relationalQueryContext; + private readonly RelationalCommandCache _relationalCommandCache; + private readonly IReadOnlyList? _readerColumns; + private readonly Func _keySelector; + private readonly Func _keyIdentifier; + private readonly IReadOnlyList _keyIdentifierValueComparers; + private readonly Func _elementSelector; + private readonly Type _contextType; + private readonly IDiagnosticsLogger _queryLogger; + private readonly bool _standAloneStateManager; + private readonly bool _detailedErrorsEnabled; + private readonly bool _threadSafetyChecksEnabled; + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public GroupBySingleQueryingEnumerable( + RelationalQueryContext relationalQueryContext, + RelationalCommandCache relationalCommandCache, + IReadOnlyList? readerColumns, + Func keySelector, + Func keyIdentifier, + IReadOnlyList keyIdentifierValueComparers, + Func elementSelector, + Type contextType, + bool standAloneStateManager, + bool detailedErrorsEnabled, + bool threadSafetyChecksEnabled) + { + _relationalQueryContext = relationalQueryContext; + _relationalCommandCache = relationalCommandCache; + _readerColumns = readerColumns; + _keySelector = keySelector; + _keyIdentifier = keyIdentifier; + _keyIdentifierValueComparers = keyIdentifierValueComparers; + _elementSelector = elementSelector; + _contextType = contextType; + _queryLogger = relationalQueryContext.QueryLogger; + _standAloneStateManager = standAloneStateManager; + _detailedErrorsEnabled = detailedErrorsEnabled; + _threadSafetyChecksEnabled = threadSafetyChecksEnabled; + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + _relationalQueryContext.CancellationToken = cancellationToken; + + return new AsyncEnumerator(this); + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual IEnumerator> GetEnumerator() + => new Enumerator(this); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + IEnumerator IEnumerable.GetEnumerator() + => GetEnumerator(); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual DbCommand CreateDbCommand() + => _relationalCommandCache + .GetRelationalCommandTemplate(_relationalQueryContext.ParameterValues) + .CreateDbCommand( + new RelationalCommandParameterObject( + _relationalQueryContext.Connection, + _relationalQueryContext.ParameterValues, + null, + null, + null, CommandSource.LinqQuery), + Guid.Empty, + (DbCommandMethod)(-1)); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual string ToQueryString() + { + using var dbCommand = CreateDbCommand(); + return _relationalQueryContext.RelationalQueryStringFactory.Create(dbCommand); + } + + private sealed class InternalGrouping : IGrouping + { + private readonly List _elements; + + public InternalGrouping(TKey key) + { + Key = key; + _elements = new(); + } + + internal void Add(TElement element) => _elements.Add(element); + + public TKey Key { get; } + + public IEnumerator GetEnumerator() => _elements.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + private static bool CompareIdentifiers(IReadOnlyList valueComparers, object[] left, object[] right) + { + // Ignoring size check on all for perf as they should be same unless bug in code. + for (var i = 0; i < left.Length; i++) + { + if (!valueComparers[i].Equals(left[i], right[i])) + { + return false; + } + } + + return true; + } + + private sealed class Enumerator : IEnumerator> + { + private readonly RelationalQueryContext _relationalQueryContext; + private readonly RelationalCommandCache _relationalCommandCache; + private readonly IReadOnlyList? _readerColumns; + private readonly Func _keySelector; + private readonly Func _keyIdentifier; + private readonly IReadOnlyList _keyIdentifierValueComparers; + private readonly Func _elementSelector; + private readonly Type _contextType; + private readonly IDiagnosticsLogger _queryLogger; + private readonly bool _standAloneStateManager; + private readonly bool _detailedErrorsEnabled; + private readonly IConcurrencyDetector? _concurrencyDetector; + private readonly IExceptionDetector _exceptionDetector; + + private IRelationalCommand? _relationalCommand; + private RelationalDataReader? _dataReader; + private DbDataReader? _dbDataReader; + private SingleQueryResultCoordinator? _resultCoordinator; + + public Enumerator(GroupBySingleQueryingEnumerable queryingEnumerable) + { + _relationalQueryContext = queryingEnumerable._relationalQueryContext; + _relationalCommandCache = queryingEnumerable._relationalCommandCache; + _readerColumns = queryingEnumerable._readerColumns; + _keySelector = queryingEnumerable._keySelector; + _keyIdentifier = queryingEnumerable._keyIdentifier; + _keyIdentifierValueComparers = queryingEnumerable._keyIdentifierValueComparers; + _elementSelector = queryingEnumerable._elementSelector; + _contextType = queryingEnumerable._contextType; + _queryLogger = queryingEnumerable._queryLogger; + _standAloneStateManager = queryingEnumerable._standAloneStateManager; + _detailedErrorsEnabled = queryingEnumerable._detailedErrorsEnabled; + _exceptionDetector = _relationalQueryContext.ExceptionDetector; + Current = default!; + + _concurrencyDetector = queryingEnumerable._threadSafetyChecksEnabled + ? _relationalQueryContext.ConcurrencyDetector + : null; + } + + public IGrouping Current { get; private set; } + + object IEnumerator.Current + => Current!; + + public bool MoveNext() + { + try + { + _concurrencyDetector?.EnterCriticalSection(); + + try + { + if (_dataReader == null) + { + _relationalQueryContext.ExecutionStrategy.Execute( + this, static (_, enumerator) => InitializeReader(enumerator), null); + } + + var hasNext = _resultCoordinator!.HasNext ?? _dataReader!.Read(); + + if (hasNext) + { + var key = _keySelector(_relationalQueryContext, _dbDataReader!); + var keyIdentifier = _keyIdentifier(_relationalQueryContext, _dbDataReader!); + var group = new InternalGrouping(key); + do + { + _resultCoordinator.ResultReady = true; + _resultCoordinator.HasNext = null; + var element = _elementSelector( + _relationalQueryContext, _dbDataReader!, _resultCoordinator.ResultContext, _resultCoordinator); + if (_resultCoordinator.ResultReady) + { + _resultCoordinator.ResultContext.Values = null; + group.Add(element); + } + + if (_resultCoordinator!.HasNext ?? _dbDataReader!.Read()) + { + if (!_resultCoordinator.ResultReady) + { + // If result isn't ready, we are still materializing element. + continue; + } + + // Check if grouping key changed + if (!CompareIdentifiers( + _keyIdentifierValueComparers, keyIdentifier, _keyIdentifier(_relationalQueryContext, _dbDataReader!))) + { + _resultCoordinator.HasNext = true; + Current = group; + break; + } + } + else + { + // End of enumeration so materialize final element if any and add it. + if (!_resultCoordinator.ResultReady) + { + _resultCoordinator.HasNext = false; + _resultCoordinator.ResultReady = true; + element = _elementSelector( + _relationalQueryContext, _dbDataReader!, _resultCoordinator.ResultContext, _resultCoordinator); + + group.Add(element); + } + Current = group; + break; + } + } + while (true); + } + else + { + Current = default!; + } + + return hasNext; + } + finally + { + _concurrencyDetector?.ExitCriticalSection(); + } + } + catch (Exception exception) + { + if (_exceptionDetector.IsCancellation(exception)) + { + _queryLogger.QueryCanceled(_contextType); + } + else + { + _queryLogger.QueryIterationFailed(_contextType, exception); + } + + throw; + } + } + + private static bool InitializeReader(Enumerator enumerator) + { + EntityFrameworkEventSource.Log.QueryExecuting(); + + var relationalCommand = enumerator._relationalCommand = + enumerator._relationalCommandCache.RentAndPopulateRelationalCommand(enumerator._relationalQueryContext); + + var dataReader = enumerator._dataReader = relationalCommand.ExecuteReader( + new RelationalCommandParameterObject( + enumerator._relationalQueryContext.Connection, + enumerator._relationalQueryContext.ParameterValues, + enumerator._readerColumns, + enumerator._relationalQueryContext.Context, + enumerator._relationalQueryContext.CommandLogger, + enumerator._detailedErrorsEnabled, + CommandSource.LinqQuery)); + enumerator._dbDataReader = dataReader.DbDataReader; + + enumerator._resultCoordinator = new SingleQueryResultCoordinator(); + + enumerator._relationalQueryContext.InitializeStateManager(enumerator._standAloneStateManager); + + return false; + } + + public void Dispose() + { + if (_dataReader is not null) + { + _relationalQueryContext.Connection.ReturnCommand(_relationalCommand!); + _dataReader.Dispose(); + _dataReader = null; + _dbDataReader = null; + } + } + + public void Reset() + => throw new NotSupportedException(CoreStrings.EnumerableResetNotSupported); + } + + private sealed class AsyncEnumerator : IAsyncEnumerator> + { + private readonly RelationalQueryContext _relationalQueryContext; + private readonly RelationalCommandCache _relationalCommandCache; + private readonly IReadOnlyList? _readerColumns; + private readonly Func _keySelector; + private readonly Func _keyIdentifier; + private readonly IReadOnlyList _keyIdentifierValueComparers; + private readonly Func _elementSelector; + private readonly Type _contextType; + private readonly IDiagnosticsLogger _queryLogger; + private readonly bool _standAloneStateManager; + private readonly bool _detailedErrorsEnabled; + private readonly IConcurrencyDetector? _concurrencyDetector; + private readonly IExceptionDetector _exceptionDetector; + private readonly CancellationToken _cancellationToken; + + private IRelationalCommand? _relationalCommand; + private RelationalDataReader? _dataReader; + private DbDataReader? _dbDataReader; + private SingleQueryResultCoordinator? _resultCoordinator; + + public AsyncEnumerator(GroupBySingleQueryingEnumerable queryingEnumerable) + { + _relationalQueryContext = queryingEnumerable._relationalQueryContext; + _relationalCommandCache = queryingEnumerable._relationalCommandCache; + _readerColumns = queryingEnumerable._readerColumns; + _keySelector = queryingEnumerable._keySelector; + _keyIdentifier = queryingEnumerable._keyIdentifier; + _keyIdentifierValueComparers = queryingEnumerable._keyIdentifierValueComparers; + _elementSelector = queryingEnumerable._elementSelector; + _contextType = queryingEnumerable._contextType; + _queryLogger = queryingEnumerable._queryLogger; + _standAloneStateManager = queryingEnumerable._standAloneStateManager; + _detailedErrorsEnabled = queryingEnumerable._detailedErrorsEnabled; + _exceptionDetector = _relationalQueryContext.ExceptionDetector; + _cancellationToken = _relationalQueryContext.CancellationToken; + Current = default!; + + _concurrencyDetector = queryingEnumerable._threadSafetyChecksEnabled + ? _relationalQueryContext.ConcurrencyDetector + : null; + } + + public IGrouping Current { get; private set; } + + public async ValueTask MoveNextAsync() + { + try + { + _concurrencyDetector?.EnterCriticalSection(); + + try + { + if (_dataReader == null) + { + await _relationalQueryContext.ExecutionStrategy.ExecuteAsync( + this, + static (_, enumerator, cancellationToken) => InitializeReaderAsync(enumerator, cancellationToken), + null, + _cancellationToken) + .ConfigureAwait(false); + } + + var hasNext = _resultCoordinator!.HasNext ?? await _dataReader!.ReadAsync(_cancellationToken).ConfigureAwait(false); + + if (hasNext) + { + var key = _keySelector(_relationalQueryContext, _dbDataReader!); + var keyIdentifier = _keyIdentifier(_relationalQueryContext, _dbDataReader!); + var group = new InternalGrouping(key); + do + { + _resultCoordinator.ResultReady = true; + _resultCoordinator.HasNext = null; + var element = _elementSelector( + _relationalQueryContext, _dbDataReader!, _resultCoordinator.ResultContext, _resultCoordinator); + if (_resultCoordinator.ResultReady) + { + _resultCoordinator.ResultContext.Values = null; + group.Add(element); + } + + if (_resultCoordinator!.HasNext ?? await _dataReader!.ReadAsync(_cancellationToken).ConfigureAwait(false)) + { + if (!_resultCoordinator.ResultReady) + { + // If result isn't ready, we are still materializing element. + continue; + } + + // Check if grouping key changed + if (!CompareIdentifiers( + _keyIdentifierValueComparers, keyIdentifier, _keyIdentifier(_relationalQueryContext, _dbDataReader!))) + { + _resultCoordinator.HasNext = true; + Current = group; + break; + } + } + else + { + // End of enumeration so materialize final element if any and add it. + if (!_resultCoordinator.ResultReady) + { + _resultCoordinator.HasNext = false; + _resultCoordinator.ResultReady = true; + element = _elementSelector( + _relationalQueryContext, _dbDataReader!, _resultCoordinator.ResultContext, _resultCoordinator); + + group.Add(element); + } + Current = group; + break; + } + } + while (true); + } + else + { + Current = default!; + } + + return hasNext; + } + finally + { + _concurrencyDetector?.ExitCriticalSection(); + } + } + catch (Exception exception) + { + if (_exceptionDetector.IsCancellation(exception, _cancellationToken)) + { + _queryLogger.QueryCanceled(_contextType); + } + else + { + _queryLogger.QueryIterationFailed(_contextType, exception); + } + + throw; + } + } + + private static async Task InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) + { + EntityFrameworkEventSource.Log.QueryExecuting(); + + var relationalCommand = enumerator._relationalCommand = + enumerator._relationalCommandCache.RentAndPopulateRelationalCommand(enumerator._relationalQueryContext); + + var dataReader = enumerator._dataReader = await relationalCommand.ExecuteReaderAsync( + new RelationalCommandParameterObject( + enumerator._relationalQueryContext.Connection, + enumerator._relationalQueryContext.ParameterValues, + enumerator._readerColumns, + enumerator._relationalQueryContext.Context, + enumerator._relationalQueryContext.CommandLogger, + enumerator._detailedErrorsEnabled, CommandSource.LinqQuery), + cancellationToken) + .ConfigureAwait(false); + enumerator._dbDataReader = dataReader.DbDataReader; + + enumerator._resultCoordinator = new SingleQueryResultCoordinator(); + + enumerator._relationalQueryContext.InitializeStateManager(enumerator._standAloneStateManager); + + return false; + } + + public ValueTask DisposeAsync() + { + if (_dataReader is not null) + { + _relationalQueryContext.Connection.ReturnCommand(_relationalCommand!); + + var dataReader = _dataReader; + _dataReader = null; + _dbDataReader = null; + + return dataReader.DisposeAsync(); + } + + return default; + } + } +} diff --git a/src/EFCore.Relational/Query/Internal/GroupBySplitQueryingEnumerable.cs b/src/EFCore.Relational/Query/Internal/GroupBySplitQueryingEnumerable.cs new file mode 100644 index 00000000000..32598ee8021 --- /dev/null +++ b/src/EFCore.Relational/Query/Internal/GroupBySplitQueryingEnumerable.cs @@ -0,0 +1,517 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using Microsoft.EntityFrameworkCore.Internal; + +namespace Microsoft.EntityFrameworkCore.Query.Internal; + +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public class GroupBySplitQueryingEnumerable + : IEnumerable>, IAsyncEnumerable>, IRelationalQueryingEnumerable +{ + private readonly RelationalQueryContext _relationalQueryContext; + private readonly RelationalCommandCache _relationalCommandCache; + private readonly IReadOnlyList? _readerColumns; + private readonly Func _keySelector; + private readonly Func _keyIdentifier; + private readonly IReadOnlyList _keyIdentifierValueComparers; + private readonly Func _elementSelector; + private readonly Action? _relatedDataLoaders; + private readonly Func? _relatedDataLoadersAsync; + private readonly Type _contextType; + private readonly IDiagnosticsLogger _queryLogger; + private readonly bool _standAloneStateManager; + private readonly bool _detailedErrorsEnabled; + private readonly bool _threadSafetyChecksEnabled; + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public GroupBySplitQueryingEnumerable( + RelationalQueryContext relationalQueryContext, + RelationalCommandCache relationalCommandCache, + IReadOnlyList? readerColumns, + Func keySelector, + Func keyIdentifier, + IReadOnlyList keyIdentifierValueComparers, + Func elementSelector, + Action? relatedDataLoaders, + Func? relatedDataLoadersAsync, + Type contextType, + bool standAloneStateManager, + bool detailedErrorsEnabled, + bool threadSafetyChecksEnabled) + { + _relationalQueryContext = relationalQueryContext; + _relationalCommandCache = relationalCommandCache; + _readerColumns = readerColumns; + _keySelector = keySelector; + _keyIdentifier = keyIdentifier; + _keyIdentifierValueComparers = keyIdentifierValueComparers; + _elementSelector = elementSelector; + _relatedDataLoaders = relatedDataLoaders; + _relatedDataLoadersAsync = relatedDataLoadersAsync; + _contextType = contextType; + _queryLogger = relationalQueryContext.QueryLogger; + _standAloneStateManager = standAloneStateManager; + _detailedErrorsEnabled = detailedErrorsEnabled; + _threadSafetyChecksEnabled = threadSafetyChecksEnabled; + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + _relationalQueryContext.CancellationToken = cancellationToken; + + return new AsyncEnumerator(this); + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual IEnumerator> GetEnumerator() + => new Enumerator(this); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + IEnumerator IEnumerable.GetEnumerator() + => GetEnumerator(); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual DbCommand CreateDbCommand() + => _relationalCommandCache + .GetRelationalCommandTemplate(_relationalQueryContext.ParameterValues) + .CreateDbCommand( + new RelationalCommandParameterObject( + _relationalQueryContext.Connection, + _relationalQueryContext.ParameterValues, + null, + null, + null, CommandSource.LinqQuery), + Guid.Empty, + (DbCommandMethod)(-1)); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual string ToQueryString() + { + using var dbCommand = CreateDbCommand(); + return _relationalQueryContext.RelationalQueryStringFactory.Create(dbCommand); + } + + private sealed class InternalGrouping : IGrouping + { + private readonly List _elements; + + public InternalGrouping(TKey key) + { + Key = key; + _elements = new(); + } + + internal void Add(TElement element) => _elements.Add(element); + + public TKey Key { get; } + + public IEnumerator GetEnumerator() => _elements.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + private static bool CompareIdentifiers(IReadOnlyList valueComparers, object[] left, object[] right) + { + // Ignoring size check on all for perf as they should be same unless bug in code. + for (var i = 0; i < left.Length; i++) + { + if (!valueComparers[i].Equals(left[i], right[i])) + { + return false; + } + } + + return true; + } + + private sealed class Enumerator : IEnumerator> + { + private readonly RelationalQueryContext _relationalQueryContext; + private readonly RelationalCommandCache _relationalCommandCache; + private readonly IReadOnlyList? _readerColumns; + private readonly Func _keySelector; + private readonly Func _keyIdentifier; + private readonly IReadOnlyList _keyIdentifierValueComparers; + private readonly Func _elementSelector; + private readonly Action? _relatedDataLoaders; + private readonly Type _contextType; + private readonly IDiagnosticsLogger _queryLogger; + private readonly bool _standAloneStateManager; + private readonly bool _detailedErrorsEnabled; + private readonly IConcurrencyDetector? _concurrencyDetector; + private readonly IExceptionDetector _exceptionDetector; + + private IRelationalCommand? _relationalCommand; + private RelationalDataReader? _dataReader; + private DbDataReader? _dbDataReader; + private SplitQueryResultCoordinator? _resultCoordinator; + + public Enumerator(GroupBySplitQueryingEnumerable queryingEnumerable) + { + _relationalQueryContext = queryingEnumerable._relationalQueryContext; + _relationalCommandCache = queryingEnumerable._relationalCommandCache; + _readerColumns = queryingEnumerable._readerColumns; + _keySelector = queryingEnumerable._keySelector; + _keyIdentifier = queryingEnumerable._keyIdentifier; + _keyIdentifierValueComparers = queryingEnumerable._keyIdentifierValueComparers; + _elementSelector = queryingEnumerable._elementSelector; + _relatedDataLoaders = queryingEnumerable._relatedDataLoaders; + _contextType = queryingEnumerable._contextType; + _queryLogger = queryingEnumerable._queryLogger; + _standAloneStateManager = queryingEnumerable._standAloneStateManager; + _detailedErrorsEnabled = queryingEnumerable._detailedErrorsEnabled; + _exceptionDetector = _relationalQueryContext.ExceptionDetector; + Current = default!; + + _concurrencyDetector = queryingEnumerable._threadSafetyChecksEnabled + ? _relationalQueryContext.ConcurrencyDetector + : null; + } + + public IGrouping Current { get; private set; } + + object IEnumerator.Current + => Current!; + + public bool MoveNext() + { + try + { + _concurrencyDetector?.EnterCriticalSection(); + + try + { + if (_dataReader == null) + { + _relationalQueryContext.ExecutionStrategy.Execute( + this, static (_, enumerator) => InitializeReader(enumerator), null); + } + + var hasNext = _resultCoordinator!.HasNext ?? _dataReader!.Read(); + + if (hasNext) + { + var key = _keySelector(_relationalQueryContext, _dbDataReader!); + var keyIdentifier = _keyIdentifier(_relationalQueryContext, _dbDataReader!); + var group = new InternalGrouping(key); + do + { + _resultCoordinator.HasNext = null; + _resultCoordinator!.ResultContext.Values = null; + var element = _elementSelector( + _relationalQueryContext, _dbDataReader!, _resultCoordinator.ResultContext, _resultCoordinator); + if (_relatedDataLoaders != null) + { + _relatedDataLoaders.Invoke( + _relationalQueryContext, _relationalQueryContext.ExecutionStrategy, _resultCoordinator); + element = _elementSelector( + _relationalQueryContext, _dbDataReader!, _resultCoordinator.ResultContext, _resultCoordinator); + } + + group.Add(element); + + if (_resultCoordinator!.HasNext ?? _dbDataReader!.Read()) + { + // Check if grouping key changed + if (!CompareIdentifiers( + _keyIdentifierValueComparers, keyIdentifier, _keyIdentifier(_relationalQueryContext, _dbDataReader!))) + { + _resultCoordinator.HasNext = true; + Current = group; + break; + } + } + else + { + // End of enumeration + Current = group; + break; + } + } + while (true); + } + else + { + Current = default!; + } + + return hasNext; + } + finally + { + _concurrencyDetector?.ExitCriticalSection(); + } + } + catch (Exception exception) + { + if (_exceptionDetector.IsCancellation(exception)) + { + _queryLogger.QueryCanceled(_contextType); + } + else + { + _queryLogger.QueryIterationFailed(_contextType, exception); + } + + throw; + } + } + + private static bool InitializeReader(Enumerator enumerator) + { + EntityFrameworkEventSource.Log.QueryExecuting(); + + var relationalCommand = enumerator._relationalCommand = + enumerator._relationalCommandCache.RentAndPopulateRelationalCommand(enumerator._relationalQueryContext); + + var dataReader = enumerator._dataReader = relationalCommand.ExecuteReader( + new RelationalCommandParameterObject( + enumerator._relationalQueryContext.Connection, + enumerator._relationalQueryContext.ParameterValues, + enumerator._readerColumns, + enumerator._relationalQueryContext.Context, + enumerator._relationalQueryContext.CommandLogger, + enumerator._detailedErrorsEnabled, + CommandSource.LinqQuery)); + enumerator._dbDataReader = dataReader.DbDataReader; + + enumerator._resultCoordinator = new SplitQueryResultCoordinator(); + + enumerator._relationalQueryContext.InitializeStateManager(enumerator._standAloneStateManager); + + return false; + } + + public void Dispose() + { + if (_dataReader is not null) + { + _relationalQueryContext.Connection.ReturnCommand(_relationalCommand!); + _dataReader.Dispose(); + _dataReader = null; + _dbDataReader = null; + } + } + + public void Reset() + => throw new NotSupportedException(CoreStrings.EnumerableResetNotSupported); + } + + private sealed class AsyncEnumerator : IAsyncEnumerator> + { + private readonly RelationalQueryContext _relationalQueryContext; + private readonly RelationalCommandCache _relationalCommandCache; + private readonly IReadOnlyList? _readerColumns; + private readonly Func _keySelector; + private readonly Func _keyIdentifier; + private readonly IReadOnlyList _keyIdentifierValueComparers; + private readonly Func _elementSelector; + private readonly Func? _relatedDataLoaders; + private readonly Type _contextType; + private readonly IDiagnosticsLogger _queryLogger; + private readonly bool _standAloneStateManager; + private readonly bool _detailedErrorsEnabled; + private readonly IConcurrencyDetector? _concurrencyDetector; + private readonly IExceptionDetector _exceptionDetector; + private readonly CancellationToken _cancellationToken; + + private IRelationalCommand? _relationalCommand; + private RelationalDataReader? _dataReader; + private DbDataReader? _dbDataReader; + private SplitQueryResultCoordinator? _resultCoordinator; + + public AsyncEnumerator(GroupBySplitQueryingEnumerable queryingEnumerable) + { + _relationalQueryContext = queryingEnumerable._relationalQueryContext; + _relationalCommandCache = queryingEnumerable._relationalCommandCache; + _readerColumns = queryingEnumerable._readerColumns; + _keySelector = queryingEnumerable._keySelector; + _keyIdentifier = queryingEnumerable._keyIdentifier; + _keyIdentifierValueComparers = queryingEnumerable._keyIdentifierValueComparers; + _elementSelector = queryingEnumerable._elementSelector; + _relatedDataLoaders = queryingEnumerable._relatedDataLoadersAsync; + _contextType = queryingEnumerable._contextType; + _queryLogger = queryingEnumerable._queryLogger; + _standAloneStateManager = queryingEnumerable._standAloneStateManager; + _detailedErrorsEnabled = queryingEnumerable._detailedErrorsEnabled; + _exceptionDetector = _relationalQueryContext.ExceptionDetector; + _cancellationToken = _relationalQueryContext.CancellationToken; + Current = default!; + + _concurrencyDetector = queryingEnumerable._threadSafetyChecksEnabled + ? _relationalQueryContext.ConcurrencyDetector + : null; + } + + public IGrouping Current { get; private set; } + + public async ValueTask MoveNextAsync() + { + try + { + _concurrencyDetector?.EnterCriticalSection(); + + try + { + if (_dataReader == null) + { + await _relationalQueryContext.ExecutionStrategy.ExecuteAsync( + this, + static (_, enumerator, cancellationToken) => InitializeReaderAsync(enumerator, cancellationToken), + null, + _cancellationToken) + .ConfigureAwait(false); + } + + var hasNext = _resultCoordinator!.HasNext ?? await _dataReader!.ReadAsync(_cancellationToken).ConfigureAwait(false); + + if (hasNext) + { + var key = _keySelector(_relationalQueryContext, _dbDataReader!); + var keyIdentifier = _keyIdentifier(_relationalQueryContext, _dbDataReader!); + var group = new InternalGrouping(key); + do + { + _resultCoordinator.HasNext = null; + _resultCoordinator!.ResultContext.Values = null; + var element = _elementSelector( + _relationalQueryContext, _dbDataReader!, _resultCoordinator.ResultContext, _resultCoordinator); + if (_relatedDataLoaders != null) + { + await _relatedDataLoaders( + _relationalQueryContext, _relationalQueryContext.ExecutionStrategy, _resultCoordinator) + .ConfigureAwait(false); + element = _elementSelector( + _relationalQueryContext, _dbDataReader!, _resultCoordinator.ResultContext, _resultCoordinator); + } + + group.Add(element); + + if (_resultCoordinator!.HasNext ?? await _dataReader!.ReadAsync(_cancellationToken).ConfigureAwait(false)) + { + // Check if grouping key changed + if (!CompareIdentifiers( + _keyIdentifierValueComparers, keyIdentifier, _keyIdentifier(_relationalQueryContext, _dbDataReader!))) + { + _resultCoordinator.HasNext = true; + Current = group; + break; + } + } + else + { + // End of enumeration + Current = group; + break; + } + } + while (true); + } + else + { + Current = default!; + } + + return hasNext; + } + finally + { + _concurrencyDetector?.ExitCriticalSection(); + } + } + catch (Exception exception) + { + if (_exceptionDetector.IsCancellation(exception, _cancellationToken)) + { + _queryLogger.QueryCanceled(_contextType); + } + else + { + _queryLogger.QueryIterationFailed(_contextType, exception); + } + + throw; + } + } + + private static async Task InitializeReaderAsync(AsyncEnumerator enumerator, CancellationToken cancellationToken) + { + EntityFrameworkEventSource.Log.QueryExecuting(); + + var relationalCommand = enumerator._relationalCommand = + enumerator._relationalCommandCache.RentAndPopulateRelationalCommand(enumerator._relationalQueryContext); + + var dataReader = enumerator._dataReader = await relationalCommand.ExecuteReaderAsync( + new RelationalCommandParameterObject( + enumerator._relationalQueryContext.Connection, + enumerator._relationalQueryContext.ParameterValues, + enumerator._readerColumns, + enumerator._relationalQueryContext.Context, + enumerator._relationalQueryContext.CommandLogger, + enumerator._detailedErrorsEnabled, CommandSource.LinqQuery), + cancellationToken) + .ConfigureAwait(false); + enumerator._dbDataReader = dataReader.DbDataReader; + + enumerator._resultCoordinator = new SplitQueryResultCoordinator(); + + enumerator._relationalQueryContext.InitializeStateManager(enumerator._standAloneStateManager); + + return false; + } + + public ValueTask DisposeAsync() + { + if (_dataReader is not null) + { + _relationalQueryContext.Connection.ReturnCommand(_relationalCommand!); + + var dataReader = _dataReader; + _dataReader = null; + _dbDataReader = null; + + return dataReader.DisposeAsync(); + } + + return default; + } + } +} diff --git a/src/EFCore.Relational/Query/Internal/SingleQueryCollectionContext.cs b/src/EFCore.Relational/Query/Internal/SingleQueryCollectionContext.cs index 5f42abb7863..7602713635d 100644 --- a/src/EFCore.Relational/Query/Internal/SingleQueryCollectionContext.cs +++ b/src/EFCore.Relational/Query/Internal/SingleQueryCollectionContext.cs @@ -3,13 +3,14 @@ namespace Microsoft.EntityFrameworkCore.Query.Internal; + /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// -public class SingleQueryCollectionContext +public sealed class SingleQueryCollectionContext { /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -37,7 +38,7 @@ public SingleQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual ResultContext ResultContext { get; } + public ResultContext ResultContext { get; } /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -45,7 +46,7 @@ public SingleQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual object? Parent { get; } + public object? Parent { get; } /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -53,7 +54,7 @@ public SingleQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual object? Collection { get; } + public object? Collection { get; } /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -61,7 +62,7 @@ public SingleQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual object[] ParentIdentifier { get; } + public object[] ParentIdentifier { get; } /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -69,7 +70,7 @@ public SingleQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual object[] OuterIdentifier { get; } + public object[] OuterIdentifier { get; } /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -77,7 +78,7 @@ public SingleQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual object[]? SelfIdentifier { get; private set; } + public object[]? SelfIdentifier { get; private set; } /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -85,6 +86,6 @@ public SingleQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual void UpdateSelfIdentifier(object[]? selfIdentifier) + public void UpdateSelfIdentifier(object[]? selfIdentifier) => SelfIdentifier = selfIdentifier; } diff --git a/src/EFCore.Relational/Query/Internal/SingleQueryingEnumerable.cs b/src/EFCore.Relational/Query/Internal/SingleQueryingEnumerable.cs index e9e4dea628d..53a3c372294 100644 --- a/src/EFCore.Relational/Query/Internal/SingleQueryingEnumerable.cs +++ b/src/EFCore.Relational/Query/Internal/SingleQueryingEnumerable.cs @@ -185,6 +185,12 @@ public bool MoveNext() break; } + // If we are already pointing to next row, we don't need to call Read + if (_resultCoordinator.HasNext == true) + { + continue; + } + if (!_dataReader!.Read()) { _resultCoordinator.HasNext = false; @@ -340,6 +346,12 @@ await _relationalQueryContext.ExecutionStrategy.ExecuteAsync( break; } + // If we are already pointing to next row, we don't need to call Read + if (_resultCoordinator.HasNext == true) + { + continue; + } + if (!await _dataReader!.ReadAsync(_cancellationToken).ConfigureAwait(false)) { _resultCoordinator.HasNext = false; diff --git a/src/EFCore.Relational/Query/Internal/SplitQueryCollectionContext.cs b/src/EFCore.Relational/Query/Internal/SplitQueryCollectionContext.cs index 923f0d92b29..c8e866612e8 100644 --- a/src/EFCore.Relational/Query/Internal/SplitQueryCollectionContext.cs +++ b/src/EFCore.Relational/Query/Internal/SplitQueryCollectionContext.cs @@ -9,7 +9,7 @@ namespace Microsoft.EntityFrameworkCore.Query.Internal; /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// -public class SplitQueryCollectionContext +public sealed class SplitQueryCollectionContext { /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -34,7 +34,7 @@ public SplitQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual ResultContext ResultContext { get; } + public ResultContext ResultContext { get; } /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -42,7 +42,7 @@ public SplitQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual object? Parent { get; } + public object? Parent { get; } /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -50,7 +50,7 @@ public SplitQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual object? Collection { get; } + public object? Collection { get; } /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -58,5 +58,5 @@ public SplitQueryCollectionContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual object[] ParentIdentifier { get; } + public object[] ParentIdentifier { get; } } diff --git a/src/EFCore.Relational/Query/Internal/SplitQueryDataReaderContext.cs b/src/EFCore.Relational/Query/Internal/SplitQueryDataReaderContext.cs index b814d40e4c4..a94387edbe2 100644 --- a/src/EFCore.Relational/Query/Internal/SplitQueryDataReaderContext.cs +++ b/src/EFCore.Relational/Query/Internal/SplitQueryDataReaderContext.cs @@ -9,7 +9,7 @@ namespace Microsoft.EntityFrameworkCore.Query.Internal; /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// -public class SplitQueryDataReaderContext +public sealed class SplitQueryDataReaderContext { /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -29,7 +29,7 @@ public SplitQueryDataReaderContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual bool? HasNext { get; set; } + public bool? HasNext { get; set; } /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -37,5 +37,5 @@ public SplitQueryDataReaderContext( /// any release. You should only use it directly in your code with extreme caution and knowing that /// doing so can result in application failures when updating to a new Entity Framework Core release. /// - public virtual RelationalDataReader DataReader { get; } + public RelationalDataReader DataReader { get; } } diff --git a/src/EFCore.Relational/Query/Internal/SplitQueryResultCoordinator.cs b/src/EFCore.Relational/Query/Internal/SplitQueryResultCoordinator.cs index 23bdcb41d61..78c60bd3183 100644 --- a/src/EFCore.Relational/Query/Internal/SplitQueryResultCoordinator.cs +++ b/src/EFCore.Relational/Query/Internal/SplitQueryResultCoordinator.cs @@ -30,6 +30,14 @@ public SplitQueryResultCoordinator() /// public ResultContext ResultContext { get; } + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public bool? HasNext { get; set; } + /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in diff --git a/src/EFCore.Relational/Query/RelationalCollectionShaperExpression.cs b/src/EFCore.Relational/Query/RelationalCollectionShaperExpression.cs index 60e9ef43b16..c25a34ad455 100644 --- a/src/EFCore.Relational/Query/RelationalCollectionShaperExpression.cs +++ b/src/EFCore.Relational/Query/RelationalCollectionShaperExpression.cs @@ -31,9 +31,9 @@ public RelationalCollectionShaperExpression( Expression parentIdentifier, Expression outerIdentifier, Expression selfIdentifier, - IReadOnlyList? parentIdentifierValueComparers, - IReadOnlyList? outerIdentifierValueComparers, - IReadOnlyList? selfIdentifierValueComparers, + IReadOnlyList parentIdentifierValueComparers, + IReadOnlyList outerIdentifierValueComparers, + IReadOnlyList selfIdentifierValueComparers, Expression innerShaper, INavigationBase? navigation, Type elementType) @@ -67,17 +67,17 @@ public RelationalCollectionShaperExpression( /// /// The list of value comparers to compare parent identifier. /// - public virtual IReadOnlyList? ParentIdentifierValueComparers { get; } + public virtual IReadOnlyList ParentIdentifierValueComparers { get; } /// /// The list of value comparers to compare outer identifier. /// - public virtual IReadOnlyList? OuterIdentifierValueComparers { get; } + public virtual IReadOnlyList OuterIdentifierValueComparers { get; } /// /// The list of value comparers to compare self identifier. /// - public virtual IReadOnlyList? SelfIdentifierValueComparers { get; } + public virtual IReadOnlyList SelfIdentifierValueComparers { get; } /// /// The expression to create inner elements. diff --git a/src/EFCore.Relational/Query/RelationalGroupByResultExpression.cs b/src/EFCore.Relational/Query/RelationalGroupByResultExpression.cs new file mode 100644 index 00000000000..b64ede9a714 --- /dev/null +++ b/src/EFCore.Relational/Query/RelationalGroupByResultExpression.cs @@ -0,0 +1,112 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.EntityFrameworkCore.Metadata.Internal; + +namespace Microsoft.EntityFrameworkCore.Query; + +/// +/// +/// An expression that represents creation of a grouping for relational provider in +/// . +/// +/// +/// This type is typically used by database providers (and other extensions). It is generally +/// not used in application code. +/// +/// +public class RelationalGroupByResultExpression : Expression, IPrintableExpression +{ + /// + /// Creates a new instance of the class. + /// + /// An identifier for the parent element. + /// A list of value comparers to compare parent identifier. + /// An expression used to create individual elements of the collection. + /// An expression used to create individual elements of the collection. + public RelationalGroupByResultExpression( + Expression keyIdentifier, + IReadOnlyList keyIdentifierValueComparers, + Expression keyShaper, + Expression elementShaper) + { + KeyIdentifier = keyIdentifier; + KeyIdentifierValueComparers = keyIdentifierValueComparers; + KeyShaper = keyShaper; + ElementShaper = elementShaper; + Type = typeof(IGrouping<,>).MakeGenericType(keyShaper.Type, elementShaper.Type); + } + + /// + /// The identifier for the grouping key. + /// + public virtual Expression KeyIdentifier { get; } + + /// + /// The list of value comparers to compare key identifier. + /// + public virtual IReadOnlyList KeyIdentifierValueComparers { get; } + + /// + /// The expression to create the grouping key. + /// + public virtual Expression KeyShaper { get; } + + /// + /// The expression to create elements in the group. + /// + public virtual Expression ElementShaper { get; } + + /// + public override Type Type { get; } + + /// + public sealed override ExpressionType NodeType + => ExpressionType.Extension; + + /// + protected override Expression VisitChildren(ExpressionVisitor visitor) + { + var keyIdentifer = visitor.Visit(KeyIdentifier); + var keyShaper = visitor.Visit(KeyShaper); + var elementShaper = visitor.Visit(ElementShaper); + + return Update(keyIdentifer, keyShaper, elementShaper); + } + + /// + /// Creates a new expression that is like this one, but using the supplied children. If all of the children are the same, it will + /// return this expression. + /// + /// The property of the result. + /// The property of the result. + /// The property of the result. + /// This expression if no children changed, or an expression with the updated children. + public virtual RelationalGroupByResultExpression Update( + Expression keyIdentifier, + Expression keyShaper, + Expression elementShaper) + => keyIdentifier != KeyIdentifier + || keyShaper != KeyShaper + || elementShaper != ElementShaper + ? new RelationalGroupByResultExpression(keyIdentifier, KeyIdentifierValueComparers, keyShaper, elementShaper) + : this; + + /// + void IPrintableExpression.Print(ExpressionPrinter expressionPrinter) + { + expressionPrinter.AppendLine("RelationalGroupByResultExpression:"); + using (expressionPrinter.Indent()) + { + expressionPrinter.Append("KeyIdentifier:"); + expressionPrinter.Visit(KeyIdentifier); + expressionPrinter.AppendLine(","); + expressionPrinter.Append("KeyShaper:"); + expressionPrinter.Visit(KeyShaper); + expressionPrinter.AppendLine(","); + expressionPrinter.Append("ElementShaper:"); + expressionPrinter.Visit(ElementShaper); + expressionPrinter.AppendLine(); + } + } +} diff --git a/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.ClientMethods.cs b/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.ClientMethods.cs new file mode 100644 index 00000000000..5e0481f0887 --- /dev/null +++ b/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.ClientMethods.cs @@ -0,0 +1,994 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Xml.Linq; +using Microsoft.EntityFrameworkCore.Internal; +using Microsoft.EntityFrameworkCore.Query.Internal; + +namespace Microsoft.EntityFrameworkCore.Query; + +public partial class RelationalShapedQueryCompilingExpressionVisitor +{ + private sealed partial class ShaperProcessingExpressionVisitor : ExpressionVisitor + { + private static readonly MethodInfo ThrowReadValueExceptionMethod = + typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(ThrowReadValueException))!; + + private static readonly MethodInfo ThrowExtractJsonPropertyExceptionMethod = + typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(ThrowExtractJsonPropertyException))!; + + // Performing collection materialization + private static readonly MethodInfo IncludeReferenceMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(IncludeReference))!; + + private static readonly MethodInfo InitializeIncludeCollectionMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(InitializeIncludeCollection))!; + + private static readonly MethodInfo PopulateIncludeCollectionMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateIncludeCollection))!; + + private static readonly MethodInfo InitializeSplitIncludeCollectionMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(InitializeSplitIncludeCollection))!; + + private static readonly MethodInfo PopulateSplitIncludeCollectionMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateSplitIncludeCollection))!; + + private static readonly MethodInfo PopulateSplitIncludeCollectionAsyncMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateSplitIncludeCollectionAsync))!; + + private static readonly MethodInfo InitializeCollectionMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(InitializeCollection))!; + + private static readonly MethodInfo PopulateCollectionMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateCollection))!; + + private static readonly MethodInfo InitializeSplitCollectionMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(InitializeSplitCollection))!; + + private static readonly MethodInfo PopulateSplitCollectionMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateSplitCollection))!; + + private static readonly MethodInfo PopulateSplitCollectionAsyncMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateSplitCollectionAsync))!; + + private static readonly MethodInfo TaskAwaiterMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(TaskAwaiter))!; + + private static readonly MethodInfo IncludeJsonEntityReferenceMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(IncludeJsonEntityReference))!; + + private static readonly MethodInfo IncludeJsonEntityCollectionMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(IncludeJsonEntityCollection))!; + + private static readonly MethodInfo MaterializeJsonEntityMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(MaterializeJsonEntity))!; + + private static readonly MethodInfo MaterializeJsonEntityCollectionMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(MaterializeJsonEntityCollection))!; + + private static readonly MethodInfo ExtractJsonPropertyMethodInfo + = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(ExtractJsonProperty))!; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static TValue ThrowReadValueException( + Exception exception, + object? value, + Type expectedType, + IPropertyBase? property = null) + { + var actualType = value?.GetType(); + + string message; + + if (property != null) + { + var entityType = property.DeclaringType.DisplayName(); + var propertyName = property.Name; + if (expectedType == typeof(object)) + { + expectedType = property.ClrType; + } + + message = exception is NullReferenceException + || Equals(value, DBNull.Value) + ? RelationalStrings.ErrorMaterializingPropertyNullReference(entityType, propertyName, expectedType) + : exception is InvalidCastException + ? CoreStrings.ErrorMaterializingPropertyInvalidCast(entityType, propertyName, expectedType, actualType) + : RelationalStrings.ErrorMaterializingProperty(entityType, propertyName); + } + else + { + message = exception is NullReferenceException + || Equals(value, DBNull.Value) + ? RelationalStrings.ErrorMaterializingValueNullReference(expectedType) + : exception is InvalidCastException + ? RelationalStrings.ErrorMaterializingValueInvalidCast(expectedType, actualType) + : RelationalStrings.ErrorMaterializingValue; + } + + throw new InvalidOperationException(message, exception); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static TValue ThrowExtractJsonPropertyException( + Exception exception, + IProperty property) + { + var entityType = property.DeclaringType.DisplayName(); + var propertyName = property.Name; + + throw new InvalidOperationException( + RelationalStrings.JsonErrorExtractingJsonProperty(entityType, propertyName), + exception); + } + + private static object? ExtractJsonProperty(JsonElement element, string propertyName, Type returnType) + { + var jsonElementProperty = element.GetProperty(propertyName); + + return jsonElementProperty.Deserialize(returnType); + } + + private static void IncludeReference( + QueryContext queryContext, + TEntity entity, + TIncludedEntity? relatedEntity, + INavigationBase navigation, + INavigationBase? inverseNavigation, + Action fixup, + bool trackingQuery) + where TEntity : class + where TIncludingEntity : class, TEntity + where TIncludedEntity : class + { + if (entity is TIncludingEntity includingEntity) + { + if (trackingQuery + && navigation.DeclaringEntityType.FindPrimaryKey() != null) + { + // For non-null relatedEntity StateManager will set the flag + if (relatedEntity == null) + { + queryContext.SetNavigationIsLoaded(includingEntity, navigation); + } + } + else + { + navigation.SetIsLoadedWhenNoTracking(includingEntity); + if (relatedEntity != null) + { + fixup(includingEntity, relatedEntity); + if (inverseNavigation != null + && !inverseNavigation.IsCollection) + { + inverseNavigation.SetIsLoadedWhenNoTracking(relatedEntity); + } + } + } + } + } + + private static void InitializeIncludeCollection( + int collectionId, + QueryContext queryContext, + DbDataReader dbDataReader, + SingleQueryResultCoordinator resultCoordinator, + TParent entity, + Func parentIdentifier, + Func outerIdentifier, + INavigationBase navigation, + IClrCollectionAccessor? clrCollectionAccessor, + bool trackingQuery, + bool setLoaded) + where TParent : class + where TNavigationEntity : class, TParent + { + object? collection = null; + if (entity is TNavigationEntity) + { + if (setLoaded) + { + if (trackingQuery) + { + queryContext.SetNavigationIsLoaded(entity, navigation); + } + else + { + navigation.SetIsLoadedWhenNoTracking(entity); + } + } + + collection = clrCollectionAccessor?.GetOrCreate(entity, forMaterialization: true); + } + + var parentKey = parentIdentifier(queryContext, dbDataReader); + var outerKey = outerIdentifier(queryContext, dbDataReader); + + var collectionMaterializationContext = new SingleQueryCollectionContext(entity, collection, parentKey, outerKey); + + resultCoordinator.SetSingleQueryCollectionContext(collectionId, collectionMaterializationContext); + } + + private static void PopulateIncludeCollection( + int collectionId, + QueryContext queryContext, + DbDataReader dbDataReader, + SingleQueryResultCoordinator resultCoordinator, + Func parentIdentifier, + Func outerIdentifier, + Func selfIdentifier, + IReadOnlyList parentIdentifierValueComparers, + IReadOnlyList outerIdentifierValueComparers, + IReadOnlyList selfIdentifierValueComparers, + Func innerShaper, + INavigationBase? inverseNavigation, + Action fixup, + bool trackingQuery) + where TIncludingEntity : class + where TIncludedEntity : class + { + var collectionMaterializationContext = resultCoordinator.Collections[collectionId]!; + if (collectionMaterializationContext.Parent is TIncludingEntity entity) + { + if (resultCoordinator.HasNext == false) + { + // Outer Enumerator has ended + GenerateCurrentElementIfPending(); + return; + } + + if (!CompareIdentifiers( + outerIdentifierValueComparers, + outerIdentifier(queryContext, dbDataReader), collectionMaterializationContext.OuterIdentifier)) + { + // Outer changed so collection has ended. Materialize last element. + GenerateCurrentElementIfPending(); + // If parent also changed then this row is now pointing to element of next collection + if (!CompareIdentifiers( + parentIdentifierValueComparers, + parentIdentifier(queryContext, dbDataReader), collectionMaterializationContext.ParentIdentifier)) + { + resultCoordinator.HasNext = true; + } + + return; + } + + var innerKey = selfIdentifier(queryContext, dbDataReader); + if (innerKey.All(e => e == null)) + { + // No correlated element + return; + } + + if (collectionMaterializationContext.SelfIdentifier != null) + { + if (CompareIdentifiers(selfIdentifierValueComparers, innerKey, collectionMaterializationContext.SelfIdentifier)) + { + // repeated row for current element + // If it is pending materialization then it may have nested elements + if (collectionMaterializationContext.ResultContext.Values != null) + { + ProcessCurrentElementRow(); + } + + resultCoordinator.ResultReady = false; + return; + } + + // Row for new element which is not first element + // So materialize the element + GenerateCurrentElementIfPending(); + resultCoordinator.HasNext = null; + collectionMaterializationContext.UpdateSelfIdentifier(innerKey); + } + else + { + // First row for current element + collectionMaterializationContext.UpdateSelfIdentifier(innerKey); + } + + ProcessCurrentElementRow(); + resultCoordinator.ResultReady = false; + } + + void ProcessCurrentElementRow() + { + var previousResultReady = resultCoordinator.ResultReady; + resultCoordinator.ResultReady = true; + var relatedEntity = innerShaper( + queryContext, dbDataReader, collectionMaterializationContext.ResultContext, resultCoordinator); + if (resultCoordinator.ResultReady) + { + // related entity is materialized + collectionMaterializationContext.ResultContext.Values = null; + if (!trackingQuery) + { + fixup(entity, relatedEntity); + if (inverseNavigation != null) + { + inverseNavigation.SetIsLoadedWhenNoTracking(relatedEntity); + } + } + } + + resultCoordinator.ResultReady &= previousResultReady; + } + + void GenerateCurrentElementIfPending() + { + if (collectionMaterializationContext.ResultContext.Values != null) + { + resultCoordinator.HasNext = false; + ProcessCurrentElementRow(); + } + + collectionMaterializationContext.UpdateSelfIdentifier(null); + } + } + + private static void InitializeSplitIncludeCollection( + int collectionId, + QueryContext queryContext, + DbDataReader parentDataReader, + SplitQueryResultCoordinator resultCoordinator, + TParent entity, + Func parentIdentifier, + INavigationBase navigation, + IClrCollectionAccessor clrCollectionAccessor, + bool trackingQuery, + bool setLoaded) + where TParent : class + where TNavigationEntity : class, TParent + { + object? collection = null; + if (entity is TNavigationEntity) + { + if (setLoaded) + { + if (trackingQuery) + { + queryContext.SetNavigationIsLoaded(entity, navigation); + } + else + { + navigation.SetIsLoadedWhenNoTracking(entity); + } + } + + collection = clrCollectionAccessor.GetOrCreate(entity, forMaterialization: true); + } + + var parentKey = parentIdentifier(queryContext, parentDataReader); + + var splitQueryCollectionContext = new SplitQueryCollectionContext(entity, collection, parentKey); + + resultCoordinator.SetSplitQueryCollectionContext(collectionId, splitQueryCollectionContext); + } + + private static void PopulateSplitIncludeCollection( + int collectionId, + RelationalQueryContext queryContext, + IExecutionStrategy executionStrategy, + RelationalCommandCache relationalCommandCache, + IReadOnlyList? readerColumns, + bool detailedErrorsEnabled, + SplitQueryResultCoordinator resultCoordinator, + Func childIdentifier, + IReadOnlyList identifierValueComparers, + Func innerShaper, + Action? relatedDataLoaders, + INavigationBase? inverseNavigation, + Action fixup, + bool trackingQuery) + where TIncludingEntity : class + where TIncludedEntity : class + { + if (resultCoordinator.DataReaders.Count <= collectionId + || resultCoordinator.DataReaders[collectionId] == null) + { + // Execute and fetch data reader + var dataReader = executionStrategy.Execute( + (queryContext, relationalCommandCache, readerColumns, detailedErrorsEnabled), + ((RelationalQueryContext, RelationalCommandCache, IReadOnlyList?, bool) tup) + => InitializeReader(tup.Item1, tup.Item2, tup.Item3, tup.Item4), + verifySucceeded: null); + + static RelationalDataReader InitializeReader( + RelationalQueryContext queryContext, + RelationalCommandCache relationalCommandCache, + IReadOnlyList? readerColumns, + bool detailedErrorsEnabled) + { + var relationalCommand = relationalCommandCache.RentAndPopulateRelationalCommand(queryContext); + + return relationalCommand.ExecuteReader( + new RelationalCommandParameterObject( + queryContext.Connection, + queryContext.ParameterValues, + readerColumns, + queryContext.Context, + queryContext.CommandLogger, + detailedErrorsEnabled, CommandSource.LinqQuery)); + } + + resultCoordinator.SetDataReader(collectionId, dataReader); + } + + var splitQueryCollectionContext = resultCoordinator.Collections[collectionId]!; + var dataReaderContext = resultCoordinator.DataReaders[collectionId]!; + var dbDataReader = dataReaderContext.DataReader.DbDataReader; + if (splitQueryCollectionContext.Parent is TIncludingEntity entity) + { + while (dataReaderContext.HasNext ?? dbDataReader.Read()) + { + if (!CompareIdentifiers( + identifierValueComparers, + splitQueryCollectionContext.ParentIdentifier, childIdentifier(queryContext, dbDataReader))) + { + dataReaderContext.HasNext = true; + + return; + } + + dataReaderContext.HasNext = null; + splitQueryCollectionContext.ResultContext.Values = null; + + innerShaper(queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); + relatedDataLoaders?.Invoke(queryContext, executionStrategy, resultCoordinator); + var relatedEntity = innerShaper( + queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); + + if (!trackingQuery) + { + fixup(entity, relatedEntity); + inverseNavigation?.SetIsLoadedWhenNoTracking(relatedEntity); + } + } + + dataReaderContext.HasNext = false; + } + } + + private static async Task PopulateSplitIncludeCollectionAsync( + int collectionId, + RelationalQueryContext queryContext, + IExecutionStrategy executionStrategy, + RelationalCommandCache relationalCommandCache, + IReadOnlyList? readerColumns, + bool detailedErrorsEnabled, + SplitQueryResultCoordinator resultCoordinator, + Func childIdentifier, + IReadOnlyList identifierValueComparers, + Func innerShaper, + Func? relatedDataLoaders, + INavigationBase? inverseNavigation, + Action fixup, + bool trackingQuery) + where TIncludingEntity : class + where TIncludedEntity : class + { + if (resultCoordinator.DataReaders.Count <= collectionId + || resultCoordinator.DataReaders[collectionId] == null) + { + // Execute and fetch data reader + var dataReader = await executionStrategy.ExecuteAsync( + (queryContext, relationalCommandCache, readerColumns, detailedErrorsEnabled), + ((RelationalQueryContext, RelationalCommandCache, IReadOnlyList?, bool) tup, CancellationToken cancellationToken) + => InitializeReaderAsync(tup.Item1, tup.Item2, tup.Item3, tup.Item4, cancellationToken), + verifySucceeded: null, + queryContext.CancellationToken) + .ConfigureAwait(false); + + static async Task InitializeReaderAsync( + RelationalQueryContext queryContext, + RelationalCommandCache relationalCommandCache, + IReadOnlyList? readerColumns, + bool detailedErrorsEnabled, + CancellationToken cancellationToken) + { + var relationalCommand = relationalCommandCache.RentAndPopulateRelationalCommand(queryContext); + + return await relationalCommand.ExecuteReaderAsync( + new RelationalCommandParameterObject( + queryContext.Connection, + queryContext.ParameterValues, + readerColumns, + queryContext.Context, + queryContext.CommandLogger, + detailedErrorsEnabled, + CommandSource.LinqQuery), + cancellationToken) + .ConfigureAwait(false); + } + + resultCoordinator.SetDataReader(collectionId, dataReader); + } + + var splitQueryCollectionContext = resultCoordinator.Collections[collectionId]!; + var dataReaderContext = resultCoordinator.DataReaders[collectionId]!; + var dbDataReader = dataReaderContext.DataReader.DbDataReader; + if (splitQueryCollectionContext.Parent is TIncludingEntity entity) + { + while (dataReaderContext.HasNext ?? await dbDataReader.ReadAsync(queryContext.CancellationToken).ConfigureAwait(false)) + { + if (!CompareIdentifiers( + identifierValueComparers, + splitQueryCollectionContext.ParentIdentifier, childIdentifier(queryContext, dbDataReader))) + { + dataReaderContext.HasNext = true; + + return; + } + + dataReaderContext.HasNext = null; + splitQueryCollectionContext.ResultContext.Values = null; + + innerShaper(queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); + if (relatedDataLoaders != null) + { + await relatedDataLoaders(queryContext, executionStrategy, resultCoordinator).ConfigureAwait(false); + } + + var relatedEntity = innerShaper( + queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); + + if (!trackingQuery) + { + fixup(entity, relatedEntity); + inverseNavigation?.SetIsLoadedWhenNoTracking(relatedEntity); + } + } + + dataReaderContext.HasNext = false; + } + } + + private static TCollection InitializeCollection( + int collectionId, + QueryContext queryContext, + DbDataReader dbDataReader, + SingleQueryResultCoordinator resultCoordinator, + Func parentIdentifier, + Func outerIdentifier, + IClrCollectionAccessor? clrCollectionAccessor) + where TCollection : class, ICollection + { + var collection = clrCollectionAccessor?.Create() ?? new List(); + + var parentKey = parentIdentifier(queryContext, dbDataReader); + var outerKey = outerIdentifier(queryContext, dbDataReader); + + var collectionMaterializationContext = new SingleQueryCollectionContext(null, collection, parentKey, outerKey); + + resultCoordinator.SetSingleQueryCollectionContext(collectionId, collectionMaterializationContext); + + return (TCollection)collection; + } + + private static void PopulateCollection( + int collectionId, + QueryContext queryContext, + DbDataReader dbDataReader, + SingleQueryResultCoordinator resultCoordinator, + Func parentIdentifier, + Func outerIdentifier, + Func selfIdentifier, + IReadOnlyList parentIdentifierValueComparers, + IReadOnlyList outerIdentifierValueComparers, + IReadOnlyList selfIdentifierValueComparers, + Func innerShaper) + where TRelatedEntity : TElement + where TCollection : class, ICollection + { + var collectionMaterializationContext = resultCoordinator.Collections[collectionId]!; + if (collectionMaterializationContext.Collection is null) + { + // nothing to materialize since no collection created + return; + } + + if (resultCoordinator.HasNext == false) + { + // Outer Enumerator has ended + GenerateCurrentElementIfPending(); + return; + } + + if (!CompareIdentifiers( + outerIdentifierValueComparers, + outerIdentifier(queryContext, dbDataReader), collectionMaterializationContext.OuterIdentifier)) + { + // Outer changed so collection has ended. Materialize last element. + GenerateCurrentElementIfPending(); + // If parent also changed then this row is now pointing to element of next collection + if (!CompareIdentifiers( + parentIdentifierValueComparers, + parentIdentifier(queryContext, dbDataReader), collectionMaterializationContext.ParentIdentifier)) + { + resultCoordinator.HasNext = true; + } + + return; + } + + var innerKey = selfIdentifier(queryContext, dbDataReader); + if (innerKey.Length > 0 && innerKey.All(e => e == null)) + { + // No correlated element + return; + } + + if (collectionMaterializationContext.SelfIdentifier != null) + { + if (CompareIdentifiers( + selfIdentifierValueComparers, + innerKey, collectionMaterializationContext.SelfIdentifier)) + { + // repeated row for current element + // If it is pending materialization then it may have nested elements + if (collectionMaterializationContext.ResultContext.Values != null) + { + ProcessCurrentElementRow(); + } + + resultCoordinator.ResultReady = false; + return; + } + + // Row for new element which is not first element + // So materialize the element + GenerateCurrentElementIfPending(); + resultCoordinator.HasNext = null; + collectionMaterializationContext.UpdateSelfIdentifier(innerKey); + } + else + { + // First row for current element + collectionMaterializationContext.UpdateSelfIdentifier(innerKey); + } + + ProcessCurrentElementRow(); + resultCoordinator.ResultReady = false; + + void ProcessCurrentElementRow() + { + var previousResultReady = resultCoordinator.ResultReady; + resultCoordinator.ResultReady = true; + var element = innerShaper( + queryContext, dbDataReader, collectionMaterializationContext.ResultContext, resultCoordinator); + if (resultCoordinator.ResultReady) + { + // related element is materialized + collectionMaterializationContext.ResultContext.Values = null; + ((TCollection)collectionMaterializationContext.Collection).Add(element); + } + + resultCoordinator.ResultReady &= previousResultReady; + } + + void GenerateCurrentElementIfPending() + { + if (collectionMaterializationContext.ResultContext.Values != null) + { + resultCoordinator.HasNext = false; + ProcessCurrentElementRow(); + } + + collectionMaterializationContext.UpdateSelfIdentifier(null); + } + } + + private static TCollection InitializeSplitCollection( + int collectionId, + QueryContext queryContext, + DbDataReader parentDataReader, + SplitQueryResultCoordinator resultCoordinator, + Func parentIdentifier, + IClrCollectionAccessor? clrCollectionAccessor) + where TCollection : class, ICollection + { + var collection = clrCollectionAccessor?.Create() ?? new List(); + var parentKey = parentIdentifier(queryContext, parentDataReader); + var splitQueryCollectionContext = new SplitQueryCollectionContext(null, collection, parentKey); + + resultCoordinator.SetSplitQueryCollectionContext(collectionId, splitQueryCollectionContext); + + return (TCollection)collection; + } + + private static void PopulateSplitCollection( + int collectionId, + RelationalQueryContext queryContext, + IExecutionStrategy executionStrategy, + RelationalCommandCache relationalCommandCache, + IReadOnlyList? readerColumns, + bool detailedErrorsEnabled, + SplitQueryResultCoordinator resultCoordinator, + Func childIdentifier, + IReadOnlyList identifierValueComparers, + Func innerShaper, + Action? relatedDataLoaders) + where TRelatedEntity : TElement + where TCollection : class, ICollection + { + if (resultCoordinator.DataReaders.Count <= collectionId + || resultCoordinator.DataReaders[collectionId] == null) + { + // Execute and fetch data reader + var dataReader = executionStrategy.Execute( + (queryContext, relationalCommandCache, readerColumns, detailedErrorsEnabled), + ((RelationalQueryContext, RelationalCommandCache, IReadOnlyList?, bool) tup) + => InitializeReader(tup.Item1, tup.Item2, tup.Item3, tup.Item4), + verifySucceeded: null); + + static RelationalDataReader InitializeReader( + RelationalQueryContext queryContext, + RelationalCommandCache relationalCommandCache, + IReadOnlyList? readerColumns, + bool detailedErrorsEnabled) + { + var relationalCommand = relationalCommandCache.RentAndPopulateRelationalCommand(queryContext); + + return relationalCommand.ExecuteReader( + new RelationalCommandParameterObject( + queryContext.Connection, + queryContext.ParameterValues, + readerColumns, + queryContext.Context, + queryContext.CommandLogger, + detailedErrorsEnabled, CommandSource.LinqQuery)); + } + + resultCoordinator.SetDataReader(collectionId, dataReader); + } + + var splitQueryCollectionContext = resultCoordinator.Collections[collectionId]!; + var dataReaderContext = resultCoordinator.DataReaders[collectionId]!; + var dbDataReader = dataReaderContext.DataReader.DbDataReader; + if (splitQueryCollectionContext.Collection is null) + { + // nothing to materialize since no collection created + return; + } + + while (dataReaderContext.HasNext ?? dbDataReader.Read()) + { + if (!CompareIdentifiers( + identifierValueComparers, + splitQueryCollectionContext.ParentIdentifier, childIdentifier(queryContext, dbDataReader))) + { + dataReaderContext.HasNext = true; + + return; + } + + dataReaderContext.HasNext = null; + splitQueryCollectionContext.ResultContext.Values = null; + + innerShaper(queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); + relatedDataLoaders?.Invoke(queryContext, executionStrategy, resultCoordinator); + var relatedElement = innerShaper( + queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); + ((TCollection)splitQueryCollectionContext.Collection).Add(relatedElement); + } + + dataReaderContext.HasNext = false; + } + + private static async Task PopulateSplitCollectionAsync( + int collectionId, + RelationalQueryContext queryContext, + IExecutionStrategy executionStrategy, + RelationalCommandCache relationalCommandCache, + IReadOnlyList? readerColumns, + bool detailedErrorsEnabled, + SplitQueryResultCoordinator resultCoordinator, + Func childIdentifier, + IReadOnlyList identifierValueComparers, + Func innerShaper, + Func? relatedDataLoaders) + where TRelatedEntity : TElement + where TCollection : class, ICollection + { + if (resultCoordinator.DataReaders.Count <= collectionId + || resultCoordinator.DataReaders[collectionId] == null) + { + // Execute and fetch data reader + var dataReader = await executionStrategy.ExecuteAsync( + (queryContext, relationalCommandCache, readerColumns, detailedErrorsEnabled), + ((RelationalQueryContext, RelationalCommandCache, IReadOnlyList?, bool) tup, CancellationToken cancellationToken) + => InitializeReaderAsync(tup.Item1, tup.Item2, tup.Item3, tup.Item4, cancellationToken), + verifySucceeded: null, + queryContext.CancellationToken) + .ConfigureAwait(false); + + static async Task InitializeReaderAsync( + RelationalQueryContext queryContext, + RelationalCommandCache relationalCommandCache, + IReadOnlyList? readerColumns, + bool detailedErrorsEnabled, + CancellationToken cancellationToken) + { + var relationalCommand = relationalCommandCache.RentAndPopulateRelationalCommand(queryContext); + + return await relationalCommand.ExecuteReaderAsync( + new RelationalCommandParameterObject( + queryContext.Connection, + queryContext.ParameterValues, + readerColumns, + queryContext.Context, + queryContext.CommandLogger, + detailedErrorsEnabled, + CommandSource.LinqQuery), + cancellationToken) + .ConfigureAwait(false); + } + + resultCoordinator.SetDataReader(collectionId, dataReader); + } + + var splitQueryCollectionContext = resultCoordinator.Collections[collectionId]!; + var dataReaderContext = resultCoordinator.DataReaders[collectionId]!; + var dbDataReader = dataReaderContext.DataReader.DbDataReader; + if (splitQueryCollectionContext.Collection is null) + { + // nothing to materialize since no collection created + return; + } + + while (dataReaderContext.HasNext ?? await dbDataReader.ReadAsync(queryContext.CancellationToken).ConfigureAwait(false)) + { + if (!CompareIdentifiers( + identifierValueComparers, + splitQueryCollectionContext.ParentIdentifier, childIdentifier(queryContext, dbDataReader))) + { + dataReaderContext.HasNext = true; + + return; + } + + dataReaderContext.HasNext = null; + splitQueryCollectionContext.ResultContext.Values = null; + + innerShaper(queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); + if (relatedDataLoaders != null) + { + await relatedDataLoaders(queryContext, executionStrategy, resultCoordinator).ConfigureAwait(false); + } + + var relatedElement = innerShaper( + queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); + ((TCollection)splitQueryCollectionContext.Collection).Add(relatedElement); + } + + dataReaderContext.HasNext = false; + } + + private static void IncludeJsonEntityReference( + QueryContext queryContext, + JsonElement? jsonElement, + object[] keyPropertyValues, + TIncludingEntity entity, + Func innerShaper, + Action fixup) + where TIncludingEntity : class + where TIncludedEntity : class + { + if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) + { + var included = innerShaper(queryContext, keyPropertyValues, jsonElement.Value); + fixup(entity, included); + } + } + + private static void IncludeJsonEntityCollection( + QueryContext queryContext, + JsonElement? jsonElement, + object[] keyPropertyValues, + TIncludingEntity entity, + Func innerShaper, + Action fixup) + where TIncludingEntity : class + where TIncludedCollectionElement : class + { + if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) + { + var newKeyPropertyValues = new object[keyPropertyValues.Length + 1]; + Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length); + + var i = 0; + foreach (var jsonArrayElement in jsonElement.Value.EnumerateArray()) + { + newKeyPropertyValues[^1] = ++i; + + var resultElement = innerShaper(queryContext, newKeyPropertyValues, jsonArrayElement); + + fixup(entity, resultElement); + } + } + } + + private static TEntity? MaterializeJsonEntity( + QueryContext queryContext, + JsonElement? jsonElement, + object[] keyPropertyValues, + bool nullable, + Func shaper) + where TEntity : class + { + if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) + { + var result = shaper(queryContext, keyPropertyValues, jsonElement.Value); + + return result; + } + + if (nullable) + { + return default(TEntity); + } + + throw new InvalidOperationException( + RelationalStrings.JsonRequiredEntityWithNullJson(typeof(TEntity).Name)); + } + + private static TResult? MaterializeJsonEntityCollection( + QueryContext queryContext, + JsonElement? jsonElement, + object[] keyPropertyValues, + INavigationBase navigation, + Func innerShaper) + where TEntity : class + where TResult : ICollection + { + if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) + { + var collectionAccessor = navigation.GetCollectionAccessor(); + var result = (TResult)collectionAccessor!.Create(); + + var newKeyPropertyValues = new object[keyPropertyValues.Length + 1]; + Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length); + + var i = 0; + foreach (var jsonArrayElement in jsonElement.Value.EnumerateArray()) + { + newKeyPropertyValues[^1] = ++i; + + var resultElement = innerShaper(queryContext, newKeyPropertyValues, jsonArrayElement); + + result.Add(resultElement); + } + + return result; + } + + return default(TResult); + } + + private static async Task TaskAwaiter(Func[] taskFactories) + { + for (var i = 0; i < taskFactories.Length; i++) + { + await taskFactories[i]().ConfigureAwait(false); + } + } + + private static bool CompareIdentifiers(IReadOnlyList valueComparers, object[] left, object[] right) + { + // Ignoring size check on all for perf as they should be same unless bug in code. + for (var i = 0; i < left.Length; i++) + { + if (!valueComparers[i].Equals(left[i], right[i])) + { + return false; + } + } + + return true; + } + } +} diff --git a/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.cs b/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.cs index 901f2981d5d..fc2995f97cc 100644 --- a/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.cs +++ b/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.cs @@ -12,7 +12,7 @@ namespace Microsoft.EntityFrameworkCore.Query; public partial class RelationalShapedQueryCompilingExpressionVisitor { - private sealed class ShaperProcessingExpressionVisitor : ExpressionVisitor + private sealed partial class ShaperProcessingExpressionVisitor : ExpressionVisitor { // Reading database values private static readonly MethodInfo IsDbNullMethod = @@ -21,12 +21,6 @@ private sealed class ShaperProcessingExpressionVisitor : ExpressionVisitor public static readonly MethodInfo GetFieldValueMethod = typeof(DbDataReader).GetRuntimeMethod(nameof(DbDataReader.GetFieldValue), new[] { typeof(int) })!; - private static readonly MethodInfo ThrowReadValueExceptionMethod = - typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(ThrowReadValueException))!; - - private static readonly MethodInfo ThrowExtractJsonPropertyExceptionMethod = - typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(ThrowExtractJsonPropertyException))!; - // Coordinating results private static readonly MemberInfo ResultContextValuesMemberInfo = typeof(ResultContext).GetMember(nameof(ResultContext.Values))[0]; @@ -34,61 +28,9 @@ private static readonly MemberInfo ResultContextValuesMemberInfo private static readonly MemberInfo SingleQueryResultCoordinatorResultReadyMemberInfo = typeof(SingleQueryResultCoordinator).GetMember(nameof(SingleQueryResultCoordinator.ResultReady))[0]; - // Performing collection materialization - private static readonly MethodInfo IncludeReferenceMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(IncludeReference))!; - - private static readonly MethodInfo InitializeIncludeCollectionMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(InitializeIncludeCollection))!; - - private static readonly MethodInfo PopulateIncludeCollectionMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateIncludeCollection))!; - - private static readonly MethodInfo InitializeSplitIncludeCollectionMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(InitializeSplitIncludeCollection))!; - - private static readonly MethodInfo PopulateSplitIncludeCollectionMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateSplitIncludeCollection))!; - - private static readonly MethodInfo PopulateSplitIncludeCollectionAsyncMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateSplitIncludeCollectionAsync))!; - - private static readonly MethodInfo InitializeCollectionMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(InitializeCollection))!; - - private static readonly MethodInfo PopulateCollectionMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateCollection))!; - - private static readonly MethodInfo InitializeSplitCollectionMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(InitializeSplitCollection))!; - - private static readonly MethodInfo PopulateSplitCollectionMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateSplitCollection))!; - - private static readonly MethodInfo PopulateSplitCollectionAsyncMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(PopulateSplitCollectionAsync))!; - - private static readonly MethodInfo TaskAwaiterMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(TaskAwaiter))!; - - private static readonly MethodInfo IncludeJsonEntityReferenceMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(IncludeJsonEntityReference))!; - - private static readonly MethodInfo IncludeJsonEntityCollectionMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(IncludeJsonEntityCollection))!; - - private static readonly MethodInfo MaterializeJsonEntityMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(MaterializeJsonEntity))!; - - private static readonly MethodInfo MaterializeJsonEntityCollectionMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(MaterializeJsonEntityCollection))!; - private static readonly MethodInfo CollectionAccessorAddMethodInfo = typeof(IClrCollectionAccessor).GetTypeInfo().GetDeclaredMethod(nameof(IClrCollectionAccessor.Add))!; - private static readonly MethodInfo ExtractJsonPropertyMethodInfo - = typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(ExtractJsonProperty))!; - private static readonly MethodInfo JsonElementGetPropertyMethod = typeof(JsonElement).GetMethod(nameof(JsonElement.GetProperty), new[] { typeof(string) })!; @@ -233,6 +175,35 @@ private ShaperProcessingExpressionVisitor( _selectExpression.ApplyTags(_tags); } + public LambdaExpression ProcessRelationalGroupingResult( + RelationalGroupByResultExpression relationalGroupByResultExpression, + out RelationalCommandCache relationalCommandCache, + out IReadOnlyList? readerColumns, + out LambdaExpression keySelector, + out LambdaExpression keyIdentifier, + out LambdaExpression? relatedDataLoaders, + ref int collectionId) + { + _inline = true; + keySelector = Expression.Lambda( + Visit(relationalGroupByResultExpression.KeyShaper), + QueryCompilationContext.QueryContextParameter, + _dataReaderParameter); + + keyIdentifier = Expression.Lambda( + Visit(relationalGroupByResultExpression.KeyIdentifier), + QueryCompilationContext.QueryContextParameter, + _dataReaderParameter); + + _inline = false; + + return ProcessShaper(relationalGroupByResultExpression.ElementShaper, + out relationalCommandCache!, + out readerColumns, + out relatedDataLoaders, + ref collectionId); + } + public LambdaExpression ProcessShaper( Expression shaperExpression, out RelationalCommandCache? relationalCommandCache, @@ -642,8 +613,8 @@ when collectionResultExpression.Navigation is INavigation navigation Expression.Constant(parentIdentifierLambda.Compile()), Expression.Constant(outerIdentifierLambda.Compile()), Expression.Constant(navigation), - Expression.Constant(navigation.IsShadowProperty() - ? null + Expression.Constant(navigation.IsShadowProperty() + ? null : navigation.GetCollectionAccessor(), typeof(IClrCollectionAccessor)), Expression.Constant(_isTracking), #pragma warning disable EF1001 // Internal EF Core API usage. @@ -1229,7 +1200,7 @@ private Expression CreateJsonShapers( } private (ParameterExpression, ParameterExpression) JsonShapingPreProcess( - ValueTuple, string[]> projectionIndex, + ValueTuple, string[]> projectionIndex, IEntityType entityType, bool isCollection) { @@ -1512,46 +1483,6 @@ Expression valueExpression return valueExpression; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static TValue ThrowReadValueException( - Exception exception, - object? value, - Type expectedType, - IPropertyBase? property = null) - { - var actualType = value?.GetType(); - - string message; - - if (property != null) - { - var entityType = property.DeclaringType.DisplayName(); - var propertyName = property.Name; - if (expectedType == typeof(object)) - { - expectedType = property.ClrType; - } - - message = exception is NullReferenceException - || Equals(value, DBNull.Value) - ? RelationalStrings.ErrorMaterializingPropertyNullReference(entityType, propertyName, expectedType) - : exception is InvalidCastException - ? CoreStrings.ErrorMaterializingPropertyInvalidCast(entityType, propertyName, expectedType, actualType) - : RelationalStrings.ErrorMaterializingProperty(entityType, propertyName); - } - else - { - message = exception is NullReferenceException - || Equals(value, DBNull.Value) - ? RelationalStrings.ErrorMaterializingValueNullReference(expectedType) - : exception is InvalidCastException - ? RelationalStrings.ErrorMaterializingValueInvalidCast(expectedType, actualType) - : RelationalStrings.ErrorMaterializingValue; - } - - throw new InvalidOperationException(message, exception); - } - private Expression CreateExtractJsonPropertyExpression( ParameterExpression jsonElementParameter, IProperty property) @@ -1602,885 +1533,6 @@ private Expression CreateExtractJsonPropertyExpression( return resultExpression; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static TValue ThrowExtractJsonPropertyException( - Exception exception, - IProperty property) - { - var entityType = property.DeclaringType.DisplayName(); - var propertyName = property.Name; - - throw new InvalidOperationException( - RelationalStrings.JsonErrorExtractingJsonProperty(entityType, propertyName), - exception); - } - - private static object? ExtractJsonProperty(JsonElement element, string propertyName, Type returnType) - { - var jsonElementProperty = element.GetProperty(propertyName); - - return jsonElementProperty.Deserialize(returnType); - } - - private static void IncludeReference( - QueryContext queryContext, - TEntity entity, - TIncludedEntity? relatedEntity, - INavigationBase navigation, - INavigationBase? inverseNavigation, - Action fixup, - bool trackingQuery) - where TEntity : class - where TIncludingEntity : class, TEntity - where TIncludedEntity : class - { - if (entity is TIncludingEntity includingEntity) - { - if (trackingQuery - && navigation.DeclaringEntityType.FindPrimaryKey() != null) - { - // For non-null relatedEntity StateManager will set the flag - if (relatedEntity == null) - { - queryContext.SetNavigationIsLoaded(includingEntity, navigation); - } - } - else - { - navigation.SetIsLoadedWhenNoTracking(includingEntity); - if (relatedEntity != null) - { - fixup(includingEntity, relatedEntity); - if (inverseNavigation != null - && !inverseNavigation.IsCollection) - { - inverseNavigation.SetIsLoadedWhenNoTracking(relatedEntity); - } - } - } - } - } - - private static void InitializeIncludeCollection( - int collectionId, - QueryContext queryContext, - DbDataReader dbDataReader, - SingleQueryResultCoordinator resultCoordinator, - TParent entity, - Func parentIdentifier, - Func outerIdentifier, - INavigationBase navigation, - IClrCollectionAccessor? clrCollectionAccessor, - bool trackingQuery, - bool setLoaded) - where TParent : class - where TNavigationEntity : class, TParent - { - object? collection = null; - if (entity is TNavigationEntity) - { - if (setLoaded) - { - if (trackingQuery) - { - queryContext.SetNavigationIsLoaded(entity, navigation); - } - else - { - navigation.SetIsLoadedWhenNoTracking(entity); - } - } - - collection = clrCollectionAccessor?.GetOrCreate(entity, forMaterialization: true); - } - - var parentKey = parentIdentifier(queryContext, dbDataReader); - var outerKey = outerIdentifier(queryContext, dbDataReader); - - var collectionMaterializationContext = new SingleQueryCollectionContext(entity, collection, parentKey, outerKey); - - resultCoordinator.SetSingleQueryCollectionContext(collectionId, collectionMaterializationContext); - } - - private static void PopulateIncludeCollection( - int collectionId, - QueryContext queryContext, - DbDataReader dbDataReader, - SingleQueryResultCoordinator resultCoordinator, - Func parentIdentifier, - Func outerIdentifier, - Func selfIdentifier, - IReadOnlyList parentIdentifierValueComparers, - IReadOnlyList outerIdentifierValueComparers, - IReadOnlyList selfIdentifierValueComparers, - Func innerShaper, - INavigationBase? inverseNavigation, - Action fixup, - bool trackingQuery) - where TIncludingEntity : class - where TIncludedEntity : class - { - var collectionMaterializationContext = resultCoordinator.Collections[collectionId]!; - if (collectionMaterializationContext.Parent is TIncludingEntity entity) - { - if (resultCoordinator.HasNext == false) - { - // Outer Enumerator has ended - GenerateCurrentElementIfPending(); - return; - } - - if (!CompareIdentifiers( - outerIdentifierValueComparers, - outerIdentifier(queryContext, dbDataReader), collectionMaterializationContext.OuterIdentifier)) - { - // Outer changed so collection has ended. Materialize last element. - GenerateCurrentElementIfPending(); - // If parent also changed then this row is now pointing to element of next collection - if (!CompareIdentifiers( - parentIdentifierValueComparers, - parentIdentifier(queryContext, dbDataReader), collectionMaterializationContext.ParentIdentifier)) - { - resultCoordinator.HasNext = true; - } - - return; - } - - var innerKey = selfIdentifier(queryContext, dbDataReader); - if (innerKey.All(e => e == null)) - { - // No correlated element - return; - } - - if (collectionMaterializationContext.SelfIdentifier != null) - { - if (CompareIdentifiers(selfIdentifierValueComparers, innerKey, collectionMaterializationContext.SelfIdentifier)) - { - // repeated row for current element - // If it is pending materialization then it may have nested elements - if (collectionMaterializationContext.ResultContext.Values != null) - { - ProcessCurrentElementRow(); - } - - resultCoordinator.ResultReady = false; - return; - } - - // Row for new element which is not first element - // So materialize the element - GenerateCurrentElementIfPending(); - resultCoordinator.HasNext = null; - collectionMaterializationContext.UpdateSelfIdentifier(innerKey); - } - else - { - // First row for current element - collectionMaterializationContext.UpdateSelfIdentifier(innerKey); - } - - ProcessCurrentElementRow(); - resultCoordinator.ResultReady = false; - } - - void ProcessCurrentElementRow() - { - var previousResultReady = resultCoordinator.ResultReady; - resultCoordinator.ResultReady = true; - var relatedEntity = innerShaper( - queryContext, dbDataReader, collectionMaterializationContext.ResultContext, resultCoordinator); - if (resultCoordinator.ResultReady) - { - // related entity is materialized - collectionMaterializationContext.ResultContext.Values = null; - if (!trackingQuery) - { - fixup(entity, relatedEntity); - if (inverseNavigation != null) - { - inverseNavigation.SetIsLoadedWhenNoTracking(relatedEntity); - } - } - } - - resultCoordinator.ResultReady &= previousResultReady; - } - - void GenerateCurrentElementIfPending() - { - if (collectionMaterializationContext.ResultContext.Values != null) - { - resultCoordinator.HasNext = false; - ProcessCurrentElementRow(); - } - - collectionMaterializationContext.UpdateSelfIdentifier(null); - } - } - - private static void InitializeSplitIncludeCollection( - int collectionId, - QueryContext queryContext, - DbDataReader parentDataReader, - SplitQueryResultCoordinator resultCoordinator, - TParent entity, - Func parentIdentifier, - INavigationBase navigation, - IClrCollectionAccessor clrCollectionAccessor, - bool trackingQuery, - bool setLoaded) - where TParent : class - where TNavigationEntity : class, TParent - { - object? collection = null; - if (entity is TNavigationEntity) - { - if (setLoaded) - { - if (trackingQuery) - { - queryContext.SetNavigationIsLoaded(entity, navigation); - } - else - { - navigation.SetIsLoadedWhenNoTracking(entity); - } - } - - collection = clrCollectionAccessor.GetOrCreate(entity, forMaterialization: true); - } - - var parentKey = parentIdentifier(queryContext, parentDataReader); - - var splitQueryCollectionContext = new SplitQueryCollectionContext(entity, collection, parentKey); - - resultCoordinator.SetSplitQueryCollectionContext(collectionId, splitQueryCollectionContext); - } - - private static void PopulateSplitIncludeCollection( - int collectionId, - RelationalQueryContext queryContext, - IExecutionStrategy executionStrategy, - RelationalCommandCache relationalCommandCache, - IReadOnlyList? readerColumns, - bool detailedErrorsEnabled, - SplitQueryResultCoordinator resultCoordinator, - Func childIdentifier, - IReadOnlyList identifierValueComparers, - Func innerShaper, - Action? relatedDataLoaders, - INavigationBase? inverseNavigation, - Action fixup, - bool trackingQuery) - where TIncludingEntity : class - where TIncludedEntity : class - { - if (resultCoordinator.DataReaders.Count <= collectionId - || resultCoordinator.DataReaders[collectionId] == null) - { - // Execute and fetch data reader - var dataReader = executionStrategy.Execute( - (queryContext, relationalCommandCache, readerColumns, detailedErrorsEnabled), - ((RelationalQueryContext, RelationalCommandCache, IReadOnlyList?, bool) tup) - => InitializeReader(tup.Item1, tup.Item2, tup.Item3, tup.Item4), - verifySucceeded: null); - - static RelationalDataReader InitializeReader( - RelationalQueryContext queryContext, - RelationalCommandCache relationalCommandCache, - IReadOnlyList? readerColumns, - bool detailedErrorsEnabled) - { - var relationalCommand = relationalCommandCache.RentAndPopulateRelationalCommand(queryContext); - - return relationalCommand.ExecuteReader( - new RelationalCommandParameterObject( - queryContext.Connection, - queryContext.ParameterValues, - readerColumns, - queryContext.Context, - queryContext.CommandLogger, - detailedErrorsEnabled, CommandSource.LinqQuery)); - } - - resultCoordinator.SetDataReader(collectionId, dataReader); - } - - var splitQueryCollectionContext = resultCoordinator.Collections[collectionId]!; - var dataReaderContext = resultCoordinator.DataReaders[collectionId]!; - var dbDataReader = dataReaderContext.DataReader.DbDataReader; - if (splitQueryCollectionContext.Parent is TIncludingEntity entity) - { - while (dataReaderContext.HasNext ?? dbDataReader.Read()) - { - if (!CompareIdentifiers( - identifierValueComparers, - splitQueryCollectionContext.ParentIdentifier, childIdentifier(queryContext, dbDataReader))) - { - dataReaderContext.HasNext = true; - - return; - } - - dataReaderContext.HasNext = null; - splitQueryCollectionContext.ResultContext.Values = null; - - innerShaper(queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); - relatedDataLoaders?.Invoke(queryContext, executionStrategy, resultCoordinator); - var relatedEntity = innerShaper( - queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); - - if (!trackingQuery) - { - fixup(entity, relatedEntity); - inverseNavigation?.SetIsLoadedWhenNoTracking(relatedEntity); - } - } - - dataReaderContext.HasNext = false; - } - } - - private static async Task PopulateSplitIncludeCollectionAsync( - int collectionId, - RelationalQueryContext queryContext, - IExecutionStrategy executionStrategy, - RelationalCommandCache relationalCommandCache, - IReadOnlyList? readerColumns, - bool detailedErrorsEnabled, - SplitQueryResultCoordinator resultCoordinator, - Func childIdentifier, - IReadOnlyList identifierValueComparers, - Func innerShaper, - Func? relatedDataLoaders, - INavigationBase? inverseNavigation, - Action fixup, - bool trackingQuery) - where TIncludingEntity : class - where TIncludedEntity : class - { - if (resultCoordinator.DataReaders.Count <= collectionId - || resultCoordinator.DataReaders[collectionId] == null) - { - // Execute and fetch data reader - var dataReader = await executionStrategy.ExecuteAsync( - (queryContext, relationalCommandCache, readerColumns, detailedErrorsEnabled), - ((RelationalQueryContext, RelationalCommandCache, IReadOnlyList?, bool) tup, CancellationToken cancellationToken) - => InitializeReaderAsync(tup.Item1, tup.Item2, tup.Item3, tup.Item4, cancellationToken), - verifySucceeded: null, - queryContext.CancellationToken) - .ConfigureAwait(false); - - static async Task InitializeReaderAsync( - RelationalQueryContext queryContext, - RelationalCommandCache relationalCommandCache, - IReadOnlyList? readerColumns, - bool detailedErrorsEnabled, - CancellationToken cancellationToken) - { - var relationalCommand = relationalCommandCache.RentAndPopulateRelationalCommand(queryContext); - - return await relationalCommand.ExecuteReaderAsync( - new RelationalCommandParameterObject( - queryContext.Connection, - queryContext.ParameterValues, - readerColumns, - queryContext.Context, - queryContext.CommandLogger, - detailedErrorsEnabled, - CommandSource.LinqQuery), - cancellationToken) - .ConfigureAwait(false); - } - - resultCoordinator.SetDataReader(collectionId, dataReader); - } - - var splitQueryCollectionContext = resultCoordinator.Collections[collectionId]!; - var dataReaderContext = resultCoordinator.DataReaders[collectionId]!; - var dbDataReader = dataReaderContext.DataReader.DbDataReader; - if (splitQueryCollectionContext.Parent is TIncludingEntity entity) - { - while (dataReaderContext.HasNext ?? await dbDataReader.ReadAsync(queryContext.CancellationToken).ConfigureAwait(false)) - { - if (!CompareIdentifiers( - identifierValueComparers, - splitQueryCollectionContext.ParentIdentifier, childIdentifier(queryContext, dbDataReader))) - { - dataReaderContext.HasNext = true; - - return; - } - - dataReaderContext.HasNext = null; - splitQueryCollectionContext.ResultContext.Values = null; - - innerShaper(queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); - if (relatedDataLoaders != null) - { - await relatedDataLoaders(queryContext, executionStrategy, resultCoordinator).ConfigureAwait(false); - } - - var relatedEntity = innerShaper( - queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); - - if (!trackingQuery) - { - fixup(entity, relatedEntity); - inverseNavigation?.SetIsLoadedWhenNoTracking(relatedEntity); - } - } - - dataReaderContext.HasNext = false; - } - } - - private static TCollection InitializeCollection( - int collectionId, - QueryContext queryContext, - DbDataReader dbDataReader, - SingleQueryResultCoordinator resultCoordinator, - Func parentIdentifier, - Func outerIdentifier, - IClrCollectionAccessor? clrCollectionAccessor) - where TCollection : class, ICollection - { - var collection = clrCollectionAccessor?.Create() ?? new List(); - - var parentKey = parentIdentifier(queryContext, dbDataReader); - var outerKey = outerIdentifier(queryContext, dbDataReader); - - var collectionMaterializationContext = new SingleQueryCollectionContext(null, collection, parentKey, outerKey); - - resultCoordinator.SetSingleQueryCollectionContext(collectionId, collectionMaterializationContext); - - return (TCollection)collection; - } - - private static void PopulateCollection( - int collectionId, - QueryContext queryContext, - DbDataReader dbDataReader, - SingleQueryResultCoordinator resultCoordinator, - Func parentIdentifier, - Func outerIdentifier, - Func selfIdentifier, - IReadOnlyList parentIdentifierValueComparers, - IReadOnlyList outerIdentifierValueComparers, - IReadOnlyList selfIdentifierValueComparers, - Func innerShaper) - where TRelatedEntity : TElement - where TCollection : class, ICollection - { - var collectionMaterializationContext = resultCoordinator.Collections[collectionId]!; - if (collectionMaterializationContext.Collection is null) - { - // nothing to materialize since no collection created - return; - } - - if (resultCoordinator.HasNext == false) - { - // Outer Enumerator has ended - GenerateCurrentElementIfPending(); - return; - } - - if (!CompareIdentifiers( - outerIdentifierValueComparers, - outerIdentifier(queryContext, dbDataReader), collectionMaterializationContext.OuterIdentifier)) - { - // Outer changed so collection has ended. Materialize last element. - GenerateCurrentElementIfPending(); - // If parent also changed then this row is now pointing to element of next collection - if (!CompareIdentifiers( - parentIdentifierValueComparers, - parentIdentifier(queryContext, dbDataReader), collectionMaterializationContext.ParentIdentifier)) - { - resultCoordinator.HasNext = true; - } - - return; - } - - var innerKey = selfIdentifier(queryContext, dbDataReader); - if (innerKey.Length > 0 && innerKey.All(e => e == null)) - { - // No correlated element - return; - } - - if (collectionMaterializationContext.SelfIdentifier != null) - { - if (CompareIdentifiers( - selfIdentifierValueComparers, - innerKey, collectionMaterializationContext.SelfIdentifier)) - { - // repeated row for current element - // If it is pending materialization then it may have nested elements - if (collectionMaterializationContext.ResultContext.Values != null) - { - ProcessCurrentElementRow(); - } - - resultCoordinator.ResultReady = false; - return; - } - - // Row for new element which is not first element - // So materialize the element - GenerateCurrentElementIfPending(); - resultCoordinator.HasNext = null; - collectionMaterializationContext.UpdateSelfIdentifier(innerKey); - } - else - { - // First row for current element - collectionMaterializationContext.UpdateSelfIdentifier(innerKey); - } - - ProcessCurrentElementRow(); - resultCoordinator.ResultReady = false; - - void ProcessCurrentElementRow() - { - var previousResultReady = resultCoordinator.ResultReady; - resultCoordinator.ResultReady = true; - var element = innerShaper( - queryContext, dbDataReader, collectionMaterializationContext.ResultContext, resultCoordinator); - if (resultCoordinator.ResultReady) - { - // related element is materialized - collectionMaterializationContext.ResultContext.Values = null; - ((TCollection)collectionMaterializationContext.Collection).Add(element); - } - - resultCoordinator.ResultReady &= previousResultReady; - } - - void GenerateCurrentElementIfPending() - { - if (collectionMaterializationContext.ResultContext.Values != null) - { - resultCoordinator.HasNext = false; - ProcessCurrentElementRow(); - } - - collectionMaterializationContext.UpdateSelfIdentifier(null); - } - } - - private static TCollection InitializeSplitCollection( - int collectionId, - QueryContext queryContext, - DbDataReader parentDataReader, - SplitQueryResultCoordinator resultCoordinator, - Func parentIdentifier, - IClrCollectionAccessor? clrCollectionAccessor) - where TCollection : class, ICollection - { - var collection = clrCollectionAccessor?.Create() ?? new List(); - var parentKey = parentIdentifier(queryContext, parentDataReader); - var splitQueryCollectionContext = new SplitQueryCollectionContext(null, collection, parentKey); - - resultCoordinator.SetSplitQueryCollectionContext(collectionId, splitQueryCollectionContext); - - return (TCollection)collection; - } - - private static void PopulateSplitCollection( - int collectionId, - RelationalQueryContext queryContext, - IExecutionStrategy executionStrategy, - RelationalCommandCache relationalCommandCache, - IReadOnlyList? readerColumns, - bool detailedErrorsEnabled, - SplitQueryResultCoordinator resultCoordinator, - Func childIdentifier, - IReadOnlyList identifierValueComparers, - Func innerShaper, - Action? relatedDataLoaders) - where TRelatedEntity : TElement - where TCollection : class, ICollection - { - if (resultCoordinator.DataReaders.Count <= collectionId - || resultCoordinator.DataReaders[collectionId] == null) - { - // Execute and fetch data reader - var dataReader = executionStrategy.Execute( - (queryContext, relationalCommandCache, readerColumns, detailedErrorsEnabled), - ((RelationalQueryContext, RelationalCommandCache, IReadOnlyList?, bool) tup) - => InitializeReader(tup.Item1, tup.Item2, tup.Item3, tup.Item4), - verifySucceeded: null); - - static RelationalDataReader InitializeReader( - RelationalQueryContext queryContext, - RelationalCommandCache relationalCommandCache, - IReadOnlyList? readerColumns, - bool detailedErrorsEnabled) - { - var relationalCommand = relationalCommandCache.RentAndPopulateRelationalCommand(queryContext); - - return relationalCommand.ExecuteReader( - new RelationalCommandParameterObject( - queryContext.Connection, - queryContext.ParameterValues, - readerColumns, - queryContext.Context, - queryContext.CommandLogger, - detailedErrorsEnabled, CommandSource.LinqQuery)); - } - - resultCoordinator.SetDataReader(collectionId, dataReader); - } - - var splitQueryCollectionContext = resultCoordinator.Collections[collectionId]!; - var dataReaderContext = resultCoordinator.DataReaders[collectionId]!; - var dbDataReader = dataReaderContext.DataReader.DbDataReader; - if (splitQueryCollectionContext.Collection is null) - { - // nothing to materialize since no collection created - return; - } - - while (dataReaderContext.HasNext ?? dbDataReader.Read()) - { - if (!CompareIdentifiers( - identifierValueComparers, - splitQueryCollectionContext.ParentIdentifier, childIdentifier(queryContext, dbDataReader))) - { - dataReaderContext.HasNext = true; - - return; - } - - dataReaderContext.HasNext = null; - splitQueryCollectionContext.ResultContext.Values = null; - - innerShaper(queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); - relatedDataLoaders?.Invoke(queryContext, executionStrategy, resultCoordinator); - var relatedElement = innerShaper( - queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); - ((TCollection)splitQueryCollectionContext.Collection).Add(relatedElement); - } - - dataReaderContext.HasNext = false; - } - - private static async Task PopulateSplitCollectionAsync( - int collectionId, - RelationalQueryContext queryContext, - IExecutionStrategy executionStrategy, - RelationalCommandCache relationalCommandCache, - IReadOnlyList? readerColumns, - bool detailedErrorsEnabled, - SplitQueryResultCoordinator resultCoordinator, - Func childIdentifier, - IReadOnlyList identifierValueComparers, - Func innerShaper, - Func? relatedDataLoaders) - where TRelatedEntity : TElement - where TCollection : class, ICollection - { - if (resultCoordinator.DataReaders.Count <= collectionId - || resultCoordinator.DataReaders[collectionId] == null) - { - // Execute and fetch data reader - var dataReader = await executionStrategy.ExecuteAsync( - (queryContext, relationalCommandCache, readerColumns, detailedErrorsEnabled), - ((RelationalQueryContext, RelationalCommandCache, IReadOnlyList?, bool) tup, CancellationToken cancellationToken) - => InitializeReaderAsync(tup.Item1, tup.Item2, tup.Item3, tup.Item4, cancellationToken), - verifySucceeded: null, - queryContext.CancellationToken) - .ConfigureAwait(false); - - static async Task InitializeReaderAsync( - RelationalQueryContext queryContext, - RelationalCommandCache relationalCommandCache, - IReadOnlyList? readerColumns, - bool detailedErrorsEnabled, - CancellationToken cancellationToken) - { - var relationalCommand = relationalCommandCache.RentAndPopulateRelationalCommand(queryContext); - - return await relationalCommand.ExecuteReaderAsync( - new RelationalCommandParameterObject( - queryContext.Connection, - queryContext.ParameterValues, - readerColumns, - queryContext.Context, - queryContext.CommandLogger, - detailedErrorsEnabled, - CommandSource.LinqQuery), - cancellationToken) - .ConfigureAwait(false); - } - - resultCoordinator.SetDataReader(collectionId, dataReader); - } - - var splitQueryCollectionContext = resultCoordinator.Collections[collectionId]!; - var dataReaderContext = resultCoordinator.DataReaders[collectionId]!; - var dbDataReader = dataReaderContext.DataReader.DbDataReader; - if (splitQueryCollectionContext.Collection is null) - { - // nothing to materialize since no collection created - return; - } - - while (dataReaderContext.HasNext ?? await dbDataReader.ReadAsync(queryContext.CancellationToken).ConfigureAwait(false)) - { - if (!CompareIdentifiers( - identifierValueComparers, - splitQueryCollectionContext.ParentIdentifier, childIdentifier(queryContext, dbDataReader))) - { - dataReaderContext.HasNext = true; - - return; - } - - dataReaderContext.HasNext = null; - splitQueryCollectionContext.ResultContext.Values = null; - - innerShaper(queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); - if (relatedDataLoaders != null) - { - await relatedDataLoaders(queryContext, executionStrategy, resultCoordinator).ConfigureAwait(false); - } - - var relatedElement = innerShaper( - queryContext, dbDataReader, splitQueryCollectionContext.ResultContext, resultCoordinator); - ((TCollection)splitQueryCollectionContext.Collection).Add(relatedElement); - } - - dataReaderContext.HasNext = false; - } - - private static void IncludeJsonEntityReference( - QueryContext queryContext, - JsonElement? jsonElement, - object[] keyPropertyValues, - TIncludingEntity entity, - Func innerShaper, - Action fixup) - where TIncludingEntity : class - where TIncludedEntity : class - { - if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) - { - var included = innerShaper(queryContext, keyPropertyValues, jsonElement.Value); - fixup(entity, included); - } - } - - private static void IncludeJsonEntityCollection( - QueryContext queryContext, - JsonElement? jsonElement, - object[] keyPropertyValues, - TIncludingEntity entity, - Func innerShaper, - Action fixup) - where TIncludingEntity : class - where TIncludedCollectionElement : class - { - if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) - { - var newKeyPropertyValues = new object[keyPropertyValues.Length + 1]; - Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length); - - var i = 0; - foreach (var jsonArrayElement in jsonElement.Value.EnumerateArray()) - { - newKeyPropertyValues[^1] = ++i; - - var resultElement = innerShaper(queryContext, newKeyPropertyValues, jsonArrayElement); - - fixup(entity, resultElement); - } - } - } - - private static TEntity? MaterializeJsonEntity( - QueryContext queryContext, - JsonElement? jsonElement, - object[] keyPropertyValues, - bool nullable, - Func shaper) - where TEntity : class - { - if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) - { - var result = shaper(queryContext, keyPropertyValues, jsonElement.Value); - - return result; - } - - if (nullable) - { - return default(TEntity); - } - - throw new InvalidOperationException( - RelationalStrings.JsonRequiredEntityWithNullJson(typeof(TEntity).Name)); - } - - private static TResult? MaterializeJsonEntityCollection( - QueryContext queryContext, - JsonElement? jsonElement, - object[] keyPropertyValues, - INavigationBase navigation, - Func innerShaper) - where TEntity : class - where TResult : ICollection - { - if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null) - { - var collectionAccessor = navigation.GetCollectionAccessor(); - var result = (TResult)collectionAccessor!.Create(); - - var newKeyPropertyValues = new object[keyPropertyValues.Length + 1]; - Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length); - - var i = 0; - foreach (var jsonArrayElement in jsonElement.Value.EnumerateArray()) - { - newKeyPropertyValues[^1] = ++i; - - var resultElement = innerShaper(queryContext, newKeyPropertyValues, jsonArrayElement); - - result.Add(resultElement); - } - - return result; - } - - return default(TResult); - } - - private static async Task TaskAwaiter(Func[] taskFactories) - { - for (var i = 0; i < taskFactories.Length; i++) - { - await taskFactories[i]().ConfigureAwait(false); - } - } - - private static bool CompareIdentifiers(IReadOnlyList valueComparers, object[] left, object[] right) - { - // Ignoring size check on all for perf as they should be same unless bug in code. - for (var i = 0; i < left.Length; i++) - { - if (!valueComparers[i].Equals(left[i], right[i])) - { - return false; - } - } - - return true; - } - private sealed class CollectionShaperFindingExpressionVisitor : ExpressionVisitor { private bool _containsCollection; diff --git a/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.cs b/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.cs index 103c641f8ae..7607b9aaa76 100644 --- a/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.cs +++ b/src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.cs @@ -242,74 +242,141 @@ protected override Expression VisitShapedQuery(ShapedQueryExpression shapedQuery var selectExpression = (SelectExpression)shapedQueryExpression.QueryExpression; VerifyNoClientConstant(shapedQueryExpression.ShaperExpression); - var nonComposedFromSql = selectExpression.IsNonComposedFromSql(); var querySplittingBehavior = ((RelationalQueryCompilationContext)QueryCompilationContext).QuerySplittingBehavior; var splitQuery = querySplittingBehavior == QuerySplittingBehavior.SplitQuery; var collectionCount = 0; - var shaper = new ShaperProcessingExpressionVisitor(this, selectExpression, _tags, splitQuery, nonComposedFromSql).ProcessShaper( - shapedQueryExpression.ShaperExpression, - out var relationalCommandCache, out var readerColumns, out var relatedDataLoaders, ref collectionCount); - - if (querySplittingBehavior == null - && collectionCount > 1) + if (shapedQueryExpression.ShaperExpression is RelationalGroupByResultExpression relationalGroupByResultExpression) { - QueryCompilationContext.Logger.MultipleCollectionIncludeWarning(); - } + var elementSelector = new ShaperProcessingExpressionVisitor(this, selectExpression, selectExpression.Tags, splitQuery, false) + .ProcessRelationalGroupingResult(relationalGroupByResultExpression, + out var relationalCommandCache, + out var readerColumns, + out var keySelector, + out var keyIdentifier, + out var relatedDataLoaders, + ref collectionCount); + + if (querySplittingBehavior == null + && collectionCount > 1) + { + QueryCompilationContext.Logger.MultipleCollectionIncludeWarning(); + } + if (splitQuery) + { + var relatedDataLoadersParameter = Expression.Constant( + QueryCompilationContext.IsAsync ? null : relatedDataLoaders?.Compile(), + typeof(Action)); + + var relatedDataLoadersAsyncParameter = Expression.Constant( + QueryCompilationContext.IsAsync ? relatedDataLoaders?.Compile() : null, + typeof(Func)); + + return Expression.New( + typeof(GroupBySplitQueryingEnumerable<,>).MakeGenericType( + keySelector.ReturnType, + elementSelector.ReturnType).GetConstructors()[0], + Expression.Convert(QueryCompilationContext.QueryContextParameter, typeof(RelationalQueryContext)), + Expression.Constant(relationalCommandCache), + Expression.Constant(readerColumns, typeof(IReadOnlyList)), + Expression.Constant(keySelector.Compile()), + Expression.Constant(keyIdentifier.Compile()), + Expression.Constant(relationalGroupByResultExpression.KeyIdentifierValueComparers, typeof(IReadOnlyList)), + Expression.Constant(elementSelector.Compile()), + relatedDataLoadersParameter, + relatedDataLoadersAsyncParameter, + Expression.Constant(_contextType), + Expression.Constant( + QueryCompilationContext.QueryTrackingBehavior == QueryTrackingBehavior.NoTrackingWithIdentityResolution), + Expression.Constant(_detailedErrorsEnabled), + Expression.Constant(_threadSafetyChecksEnabled)); + } - if (nonComposedFromSql) - { return Expression.New( - typeof(FromSqlQueryingEnumerable<>).MakeGenericType(shaper.ReturnType).GetConstructors()[0], + typeof(GroupBySingleQueryingEnumerable<,>).MakeGenericType( + keySelector.ReturnType, + elementSelector.ReturnType).GetConstructors()[0], Expression.Convert(QueryCompilationContext.QueryContextParameter, typeof(RelationalQueryContext)), Expression.Constant(relationalCommandCache), Expression.Constant(readerColumns, typeof(IReadOnlyList)), - Expression.Constant( - selectExpression.Projection.Select(pe => ((ColumnExpression)pe.Expression).Name).ToList(), - typeof(IReadOnlyList)), - Expression.Constant(shaper.Compile()), + Expression.Constant(keySelector.Compile()), + Expression.Constant(keyIdentifier.Compile()), + Expression.Constant(relationalGroupByResultExpression.KeyIdentifierValueComparers, typeof(IReadOnlyList)), + Expression.Constant(elementSelector.Compile()), Expression.Constant(_contextType), Expression.Constant( QueryCompilationContext.QueryTrackingBehavior == QueryTrackingBehavior.NoTrackingWithIdentityResolution), Expression.Constant(_detailedErrorsEnabled), Expression.Constant(_threadSafetyChecksEnabled)); } - - if (splitQuery) + else { - var relatedDataLoadersParameter = Expression.Constant( - QueryCompilationContext.IsAsync ? null : relatedDataLoaders?.Compile(), - typeof(Action)); - var relatedDataLoadersAsyncParameter = Expression.Constant( - QueryCompilationContext.IsAsync ? relatedDataLoaders?.Compile() : null, - typeof(Func)); + var nonComposedFromSql = selectExpression.IsNonComposedFromSql(); + var shaper = new ShaperProcessingExpressionVisitor(this, selectExpression, _tags, splitQuery, nonComposedFromSql).ProcessShaper( + shapedQueryExpression.ShaperExpression, + out var relationalCommandCache, out var readerColumns, out var relatedDataLoaders, ref collectionCount); + + if (querySplittingBehavior == null + && collectionCount > 1) + { + QueryCompilationContext.Logger.MultipleCollectionIncludeWarning(); + } + + if (nonComposedFromSql) + { + return Expression.New( + typeof(FromSqlQueryingEnumerable<>).MakeGenericType(shaper.ReturnType).GetConstructors()[0], + Expression.Convert(QueryCompilationContext.QueryContextParameter, typeof(RelationalQueryContext)), + Expression.Constant(relationalCommandCache), + Expression.Constant(readerColumns, typeof(IReadOnlyList)), + Expression.Constant( + selectExpression.Projection.Select(pe => ((ColumnExpression)pe.Expression).Name).ToList(), + typeof(IReadOnlyList)), + Expression.Constant(shaper.Compile()), + Expression.Constant(_contextType), + Expression.Constant( + QueryCompilationContext.QueryTrackingBehavior == QueryTrackingBehavior.NoTrackingWithIdentityResolution), + Expression.Constant(_detailedErrorsEnabled), + Expression.Constant(_threadSafetyChecksEnabled)); + } + + if (splitQuery) + { + var relatedDataLoadersParameter = Expression.Constant( + QueryCompilationContext.IsAsync ? null : relatedDataLoaders?.Compile(), + typeof(Action)); + + var relatedDataLoadersAsyncParameter = Expression.Constant( + QueryCompilationContext.IsAsync ? relatedDataLoaders?.Compile() : null, + typeof(Func)); + + return Expression.New( + typeof(SplitQueryingEnumerable<>).MakeGenericType(shaper.ReturnType).GetConstructors().Single(), + Expression.Convert(QueryCompilationContext.QueryContextParameter, typeof(RelationalQueryContext)), + Expression.Constant(relationalCommandCache), + Expression.Constant(readerColumns, typeof(IReadOnlyList)), + Expression.Constant(shaper.Compile()), + relatedDataLoadersParameter, + relatedDataLoadersAsyncParameter, + Expression.Constant(_contextType), + Expression.Constant( + QueryCompilationContext.QueryTrackingBehavior == QueryTrackingBehavior.NoTrackingWithIdentityResolution), + Expression.Constant(_detailedErrorsEnabled), + Expression.Constant(_threadSafetyChecksEnabled)); + } return Expression.New( - typeof(SplitQueryingEnumerable<>).MakeGenericType(shaper.ReturnType).GetConstructors().Single(), + typeof(SingleQueryingEnumerable<>).MakeGenericType(shaper.ReturnType).GetConstructors()[0], Expression.Convert(QueryCompilationContext.QueryContextParameter, typeof(RelationalQueryContext)), Expression.Constant(relationalCommandCache), Expression.Constant(readerColumns, typeof(IReadOnlyList)), Expression.Constant(shaper.Compile()), - relatedDataLoadersParameter, - relatedDataLoadersAsyncParameter, Expression.Constant(_contextType), Expression.Constant( QueryCompilationContext.QueryTrackingBehavior == QueryTrackingBehavior.NoTrackingWithIdentityResolution), Expression.Constant(_detailedErrorsEnabled), Expression.Constant(_threadSafetyChecksEnabled)); } - - return Expression.New( - typeof(SingleQueryingEnumerable<>).MakeGenericType(shaper.ReturnType).GetConstructors()[0], - Expression.Convert(QueryCompilationContext.QueryContextParameter, typeof(RelationalQueryContext)), - Expression.Constant(relationalCommandCache), - Expression.Constant(readerColumns, typeof(IReadOnlyList)), - Expression.Constant(shaper.Compile()), - Expression.Constant(_contextType), - Expression.Constant( - QueryCompilationContext.QueryTrackingBehavior == QueryTrackingBehavior.NoTrackingWithIdentityResolution), - Expression.Constant(_detailedErrorsEnabled), - Expression.Constant(_threadSafetyChecksEnabled)); } } diff --git a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.Helper.cs b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.Helper.cs index 957f0070119..d43f374c690 100644 --- a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.Helper.cs +++ b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.Helper.cs @@ -760,6 +760,16 @@ public ClientProjectionRemappingExpressionVisitor(List clientProjectionI throw new InvalidOperationException(); } + if (expression is RelationalGroupByResultExpression relationalGroupByResultExpression) + { + // Only element shaper needs remapping + return new RelationalGroupByResultExpression( + relationalGroupByResultExpression.KeyIdentifier, + relationalGroupByResultExpression.KeyIdentifierValueComparers, + relationalGroupByResultExpression.KeyShaper, + Visit(relationalGroupByResultExpression.ElementShaper)); + } + return base.Visit(expression); } } diff --git a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs index 174e54dfc6c..9869311d2b5 100644 --- a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs +++ b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Runtime.CompilerServices; -using System.Text.Json; using Microsoft.EntityFrameworkCore.Metadata.Internal; using Microsoft.EntityFrameworkCore.Query.Internal; @@ -60,6 +59,13 @@ public sealed partial class SelectExpression : TableExpressionBase private SortedDictionary? _annotations; + // We need to remember identfiers before GroupBy in case it is final GroupBy and element selector has a colection + // This state doesn't need to propagate + // It should be only at top-level otherwise GroupBy won't be final operator. + // Cloning skips it altogether (we don't clone top level with GroupBy) + // Pushdown should null it out as if GroupBy was present was pushed down. + private List<(ColumnExpression Column, ValueComparer Comparer)>? _preGroupByIdenifier; + #if DEBUG private List? _removedAliases; #endif @@ -762,6 +768,33 @@ public Expression ApplyProjection( } _mutable = false; + if (shaperExpression is RelationalGroupByShaperExpression relationalGroupByShaperExpression) + { + // This is final GroupBy operation + Check.DebugAssert(_groupBy.Count > 0, "The selectExpression doesn't have grouping terms."); + + if (_clientProjections.Count == 0) + { + // Force client projection because we would be injecting keys and client-side key comparison + var mapping = ConvertProjectionMappingToClientProjections(_projectionMapping); + var innerShaperExpression = new ProjectionMemberToIndexConvertingExpressionVisitor(this, mapping).Visit( + relationalGroupByShaperExpression.ElementSelector); + shaperExpression = new RelationalGroupByShaperExpression( + relationalGroupByShaperExpression.KeySelector, + innerShaperExpression, + relationalGroupByShaperExpression.GroupingEnumerable); + } + + // Convert GroupBy to OrderBy + foreach (var groupingTerm in _groupBy) + { + AppendOrdering(new OrderingExpression(groupingTerm, ascending: true)); + } + _groupBy.Clear(); + // We do processing of adding key terms to projection when applying projection so we can move offsets for other + // projections correctly + } + if (_clientProjections.Count > 0) { EntityShaperNullableMarkingExpressionVisitor? entityShaperNullableMarkingExpressionVisitor = null; @@ -797,6 +830,7 @@ public Expression ApplyProjection( || (querySplittingBehavior == QuerySplittingBehavior.SingleQuery && containsCollection)) { // Pushdown outer since we will be adding join to this + // For grouping query pushown will not occur since we don't allow this terms to compose (yet!). if (Limit != null || Offset != null || IsDistinct @@ -809,14 +843,124 @@ public Expression ApplyProjection( entityShaperNullableMarkingExpressionVisitor = new EntityShaperNullableMarkingExpressionVisitor(); } - if (containsSingleResult || containsCollection) + if (querySplittingBehavior == QuerySplittingBehavior.SplitQuery + && (containsSingleResult || containsCollection)) { + // SingleResult can lift collection from inner cloningExpressionVisitor = new CloningExpressionVisitor(); } + var jsonClientProjectionDeduplicationMap = BuildJsonProjectionDeduplicationMap(_clientProjections.OfType()); + var earlierClientProjectionCount = _clientProjections.Count; + var newClientProjections = new List(); + var clientProjectionIndexMap = new List(); + var remappingRequired = false; + + if (shaperExpression is RelationalGroupByShaperExpression groupByShaper) + { + // We need to add key to projection and generate key selector in terms of projectionBindings + var projectionBindingMap = new Dictionary(); + var keySelector = AddGroupByKeySelectorToProjection( + this, newClientProjections, projectionBindingMap, groupByShaper.KeySelector); + var (keyIdentifier, keyIdentifierValueComparers) = GetIdentifierAccessor(projectionBindingMap, _identifier); + _identifier.Clear(); + _identifier.AddRange(_preGroupByIdenifier!); + _preGroupByIdenifier!.Clear(); + + static Expression AddGroupByKeySelectorToProjection( + SelectExpression selectExpression, + List clientProjectionList, + Dictionary projectionBindingMap, + Expression keySelector) + { + switch (keySelector) + { + case SqlExpression sqlExpression: + var index = selectExpression.AddToProjection(sqlExpression); + var clientProjectionToAdd = Constant(index); + var existingIndex = clientProjectionList.FindIndex( + e => ExpressionEqualityComparer.Instance.Equals(e, clientProjectionToAdd)); + if (existingIndex == -1) + { + clientProjectionList.Add(Constant(index)); + existingIndex = clientProjectionList.Count - 1; + } + + var projectionBindingExpression = new ProjectionBindingExpression( + selectExpression, existingIndex, sqlExpression.Type.MakeNullable()); + projectionBindingMap[sqlExpression] = projectionBindingExpression; + return projectionBindingExpression; + + case NewExpression newExpression: + var newArguments = new Expression[newExpression.Arguments.Count]; + for (var i = 0; i < newExpression.Arguments.Count; i++) + { + var newArgument = AddGroupByKeySelectorToProjection( + selectExpression, clientProjectionList, projectionBindingMap, newExpression.Arguments[i]); + newArguments[i] = newExpression.Arguments[i].Type != newArgument.Type + ? Convert(newArgument, newExpression.Arguments[i].Type) + : newArgument; + } + + return newExpression.Update(newArguments); + + case MemberInitExpression memberInitExpression: + var updatedNewExpression = AddGroupByKeySelectorToProjection( + selectExpression, clientProjectionList, projectionBindingMap, memberInitExpression.NewExpression); + var newBindings = new MemberBinding[memberInitExpression.Bindings.Count]; + for (var i = 0; i < newBindings.Length; i++) + { + var memberAssignment = (MemberAssignment)memberInitExpression.Bindings[i]; + var newAssignmentExpression = AddGroupByKeySelectorToProjection( + selectExpression, clientProjectionList, projectionBindingMap, memberAssignment.Expression); + newBindings[i] = memberAssignment.Update( + memberAssignment.Expression.Type != newAssignmentExpression.Type + ? Convert(newAssignmentExpression, memberAssignment.Expression.Type) + : newAssignmentExpression); + } + + return memberInitExpression.Update((NewExpression)updatedNewExpression, newBindings); + + case UnaryExpression unaryExpression + when unaryExpression.NodeType == ExpressionType.Convert + || unaryExpression.NodeType == ExpressionType.ConvertChecked: + return unaryExpression.Update( + AddGroupByKeySelectorToProjection( + selectExpression, clientProjectionList, projectionBindingMap, unaryExpression.Operand)); + + default: + throw new InvalidOperationException( + RelationalStrings.InvalidKeySelectorForGroupBy(keySelector, keySelector.GetType())); + } + } + + static (Expression, IReadOnlyList) GetIdentifierAccessor( + Dictionary projectionBindingMap, + IEnumerable<(ColumnExpression Column, ValueComparer Comparer)> identifyingProjection) + { + var updatedExpressions = new List(); + var comparers = new List(); + foreach (var (column, comparer) in identifyingProjection) + { + var projectionBindingExpression = projectionBindingMap[column]; + updatedExpressions.Add( + projectionBindingExpression.Type.IsValueType + ? Convert(projectionBindingExpression, typeof(object)) + : projectionBindingExpression); + comparers.Add(comparer); + } + + return (NewArrayInit(typeof(object), updatedExpressions), comparers); + } + remappingRequired = true; + shaperExpression = new RelationalGroupByResultExpression( + keyIdentifier, keyIdentifierValueComparers, keySelector, groupByShaper.ElementSelector); + } + SelectExpression? baseSelectExpression = null; if (querySplittingBehavior == QuerySplittingBehavior.SplitQuery && containsCollection) { + // Needs to happen after converting final GroupBy so we clone correct form. baseSelectExpression = (SelectExpression)cloningExpressionVisitor!.Visit(this); // We mark this as mutable because the split query will combine into this and take it over. baseSelectExpression._mutable = true; @@ -845,11 +989,6 @@ static void UpdateLimit(SelectExpression selectExpression) } } - var jsonClientProjectionDeduplicationMap = BuildJsonProjectionDeduplicationMap(_clientProjections.OfType()); - var earlierClientProjectionCount = _clientProjections.Count; - var newClientProjections = new List(); - var clientProjectionIndexMap = new List(); - var remappingRequired = false; for (var i = 0; i < _clientProjections.Count; i++) { if (i == earlierClientProjectionCount) @@ -857,9 +996,12 @@ static void UpdateLimit(SelectExpression selectExpression) // Since we lift nested client projections for single results up, we may need to re-clone the baseSelectExpression // again so it does contain the single result subquery too. We erase projections for it since it would be non-empty. earlierClientProjectionCount = _clientProjections.Count; - baseSelectExpression = (SelectExpression)cloningExpressionVisitor!.Visit(this); - baseSelectExpression._mutable = true; - baseSelectExpression._projection.Clear(); + if (cloningExpressionVisitor != null) + { + baseSelectExpression = (SelectExpression)cloningExpressionVisitor.Visit(this); + baseSelectExpression._mutable = true; + baseSelectExpression._projection.Clear(); + } //since we updated the client projections, we also need updated deduplication map jsonClientProjectionDeduplicationMap = BuildJsonProjectionDeduplicationMap( @@ -1362,7 +1504,7 @@ Expression CopyProjectionToOuter(SelectExpression innerSelectExpression, Express { result[projectionMember] = expression switch { - EntityProjectionExpression entityProjection => AddEntityProjection(entityProjection), + EntityProjectionExpression entityProjection => AddEntityProjection(entityProjection), JsonQueryExpression jsonQueryExpression => AddJsonProjection(jsonQueryExpression, jsonProjectionDeduplicationMap[jsonQueryExpression]), _ => Constant(AddToProjection((SqlExpression)expression, projectionMember.Last?.Name)) }; @@ -1712,7 +1854,7 @@ public void ApplyGrouping(Expression keySelector) var groupByAliases = new List(); PopulateGroupByTerms(keySelector, groupByTerms, groupByAliases, "Key"); - if (groupByTerms.Any(e => e is SqlConstantExpression || e is SqlParameterExpression || e is ScalarSubqueryExpression)) + if (groupByTerms.Any(e => e is not ColumnExpression)) { var sqlRemappingVisitor = PushdownIntoSubqueryInternal(); var newGroupByTerms = new List(groupByTerms.Count); @@ -1805,6 +1947,7 @@ public RelationalGroupByShaperExpression ApplyGrouping( if (!_identifier.All(e => _groupBy.Contains(e.Column))) { + _preGroupByIdenifier = _identifier.ToList(); _identifier.Clear(); if (_groupBy.All(e => e is ColumnExpression)) { @@ -3200,6 +3343,7 @@ private SqlRemappingVisitor PushdownIntoSubqueryInternal() Having = null; Offset = null; Limit = null; + _preGroupByIdenifier = null; subquery._removableJoinTables.AddRange(_removableJoinTables); _removableJoinTables.Clear(); foreach (var kvp in _tpcDiscriminatorValues) diff --git a/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.ExpressionVisitors.cs b/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.ExpressionVisitors.cs index 7774753bf32..5b26061de7a 100644 --- a/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.ExpressionVisitors.cs +++ b/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.ExpressionVisitors.cs @@ -1132,7 +1132,7 @@ public GroupingElementReplacingExpressionVisitor( GroupByNavigationExpansionExpression groupByNavigationExpansionExpression) { _parameterExpression = parameterExpression; - _navigationExpansionExpression = (NavigationExpansionExpression)groupByNavigationExpansionExpression.GroupingEnumerable; + _navigationExpansionExpression = groupByNavigationExpansionExpression.GroupingEnumerable; _keyAccessExpression = Expression.MakeMemberAccess( groupByNavigationExpansionExpression.CurrentParameter, groupByNavigationExpansionExpression.CurrentParameter.Type.GetTypeInfo().GetDeclaredProperty( diff --git a/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.Expressions.cs b/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.Expressions.cs index 664c2685935..5aff0d78046 100644 --- a/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.Expressions.cs +++ b/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.Expressions.cs @@ -330,7 +330,7 @@ public GroupByNavigationExpansionExpression( public ParameterExpression CurrentParameter { get; } - public Expression GroupingEnumerable { get; } + public NavigationExpansionExpression GroupingEnumerable { get; } public Type SourceElementType => CurrentParameter.Type; diff --git a/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.cs b/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.cs index 6a0c4368445..930dcfa1ea9 100644 --- a/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.cs +++ b/src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.cs @@ -101,14 +101,68 @@ public virtual Expression Expand(Expression query) { var result = Visit(query); - if (result is GroupByNavigationExpansionExpression) + if (result is GroupByNavigationExpansionExpression groupByNavigationExpansionExpression) { - // This indicates that GroupBy was not condensed out of grouping operator. - throw new InvalidOperationException(CoreStrings.TranslationFailed(query.Print())); - } + if (!(groupByNavigationExpansionExpression.Source is MethodCallExpression methodCallExpression + && methodCallExpression.Method.IsGenericMethod + && methodCallExpression.Method.GetGenericMethodDefinition() == QueryableMethods.GroupByWithKeySelector)) + { + // If the final operator is not GroupBy in source then GroupBy is not final operator. We throw exception. + throw new InvalidOperationException(CoreStrings.TranslationFailed(query.Print())); + } + + var groupingQueryable = groupByNavigationExpansionExpression.GroupingEnumerable.Source; + var innerEnumerable = new PendingSelectorExpandingExpressionVisitor(this, _extensibilityHelper, applyIncludes: true).Visit( + groupByNavigationExpansionExpression.GroupingEnumerable); + innerEnumerable = Reduce(innerEnumerable); - result = new PendingSelectorExpandingExpressionVisitor(this, _extensibilityHelper, applyIncludes: true).Visit(result); - result = Reduce(result); + if (innerEnumerable is MethodCallExpression selectMethodCall + && selectMethodCall.Method.IsGenericMethod + && selectMethodCall.Method.GetGenericMethodDefinition() == QueryableMethods.Select) + { + var elementSelector = selectMethodCall.Arguments[1]; + var elementSelectorLambda = elementSelector.UnwrapLambdaFromQuote(); + + // We do have element selector to inject which may have potentially expanded navigations + var oldKeySelectorLambda = methodCallExpression.Arguments[1].UnwrapLambdaFromQuote(); + var newSource = ReplacingExpressionVisitor.Replace( + groupingQueryable, methodCallExpression.Arguments[0], selectMethodCall.Arguments[0]); + + var oldKeySelectorParameterType = oldKeySelectorLambda.Parameters[0].Type; + var keySelectorParameter = Expression.Parameter(elementSelectorLambda.Parameters[0].Type); + var keySelectorMemberAccess = (Expression)keySelectorParameter; + while (keySelectorMemberAccess.Type != oldKeySelectorParameterType) + { + keySelectorMemberAccess = Expression.MakeMemberAccess( + keySelectorMemberAccess, + keySelectorMemberAccess.Type.GetTypeInfo().GetDeclaredField("Outer")!); + } + + var keySelectorLambda = Expression.Lambda( + ReplacingExpressionVisitor.Replace( + oldKeySelectorLambda.Parameters[0], keySelectorMemberAccess, oldKeySelectorLambda.Body), + keySelectorParameter); + + result = Expression.Call( + QueryableMethods.GroupByWithKeyElementSelector.MakeGenericMethod( + newSource.Type.GetSequenceType(), + keySelectorLambda.ReturnType, + elementSelectorLambda.ReturnType), + newSource, + Expression.Quote(keySelectorLambda), + elementSelector); + } + else + { + // Pending selector was identity so nothing to apply, we can return source as-is. + result = groupByNavigationExpansionExpression.Source; + } + } + else + { + result = new PendingSelectorExpandingExpressionVisitor(this, _extensibilityHelper, applyIncludes: true).Visit(result); + result = Reduce(result); + } var dbContextOnQueryContextPropertyAccess = Expression.Convert( @@ -1081,7 +1135,7 @@ private NavigationExpansionExpression ProcessInclude( { return (currentExpression, default); } - + if (!methodCallExpression.Method.IsGenericMethod || !SupportedFilteredIncludeOperations.Contains(methodCallExpression.Method.GetGenericMethodDefinition())) { @@ -2083,9 +2137,9 @@ when methodCallExpression.TryGetEFPropertyArguments(out var entityExpression, ou } throw new InvalidOperationException(CoreStrings.InvalidIncludeExpression(expression)); - + bool TryExtractIncludeTreeNode( - Expression innerExpression, + Expression innerExpression, string propertyName, [NotNullWhen(true)] out IncludeTreeNode? addedNode) { @@ -2106,7 +2160,7 @@ bool TryExtractIncludeTreeNode( var navigation = (INavigationBase?)entityType.FindNavigation(propertyName) ?? entityType.FindSkipNavigation(propertyName); - + if (navigation != null) { addedNode = innerIncludeTreeNode.AddNavigation(navigation, setLoaded); diff --git a/test/EFCore.InMemory.FunctionalTests/Query/ComplexNavigationsCollectionsQueryInMemoryTest.cs b/test/EFCore.InMemory.FunctionalTests/Query/ComplexNavigationsCollectionsQueryInMemoryTest.cs index d6599b32aa5..20504752774 100644 --- a/test/EFCore.InMemory.FunctionalTests/Query/ComplexNavigationsCollectionsQueryInMemoryTest.cs +++ b/test/EFCore.InMemory.FunctionalTests/Query/ComplexNavigationsCollectionsQueryInMemoryTest.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.InMemory.Internal; + namespace Microsoft.EntityFrameworkCore.Query; public class ComplexNavigationsCollectionsQueryInMemoryTest @@ -13,4 +15,39 @@ public ComplexNavigationsCollectionsQueryInMemoryTest( { //TestLoggerFactory.TestOutputHelper = testOutputHelper; } + + public override Task Final_GroupBy_property_entity_Include_collection(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_collection_nested(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection_nested(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_collection_reference(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection_reference(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_collection_multiple(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection_multiple(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_collection_reference_same_level(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection_reference_same_level(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_reference(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_reference(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_reference_multiple(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_reference_multiple(async), + InMemoryStrings.UncomposedGroupByNotSupported); } diff --git a/test/EFCore.InMemory.FunctionalTests/Query/ComplexNavigationsCollectionsSharedTypeQueryInMemoryTest.cs b/test/EFCore.InMemory.FunctionalTests/Query/ComplexNavigationsCollectionsSharedTypeQueryInMemoryTest.cs index 0ab251441f3..debbdc7debd 100644 --- a/test/EFCore.InMemory.FunctionalTests/Query/ComplexNavigationsCollectionsSharedTypeQueryInMemoryTest.cs +++ b/test/EFCore.InMemory.FunctionalTests/Query/ComplexNavigationsCollectionsSharedTypeQueryInMemoryTest.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.InMemory.Internal; + namespace Microsoft.EntityFrameworkCore.Query; public class ComplexNavigationsCollectionsSharedTypeQueryInMemoryTest @@ -13,4 +15,39 @@ public ComplexNavigationsCollectionsSharedTypeQueryInMemoryTest( { //TestLoggerFactory.TestOutputHelper = testOutputHelper; } + + public override Task Final_GroupBy_property_entity_Include_collection(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_collection_nested(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection_nested(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_collection_reference(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection_reference(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_collection_multiple(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection_multiple(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_collection_reference_same_level(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection_reference_same_level(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_reference(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_reference(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_reference_multiple(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_reference_multiple(async), + InMemoryStrings.UncomposedGroupByNotSupported); } diff --git a/test/EFCore.InMemory.FunctionalTests/Query/NorthwindGroupByQueryInMemoryTest.cs b/test/EFCore.InMemory.FunctionalTests/Query/NorthwindGroupByQueryInMemoryTest.cs index 2ab27f40dc6..8d9a829b8a3 100644 --- a/test/EFCore.InMemory.FunctionalTests/Query/NorthwindGroupByQueryInMemoryTest.cs +++ b/test/EFCore.InMemory.FunctionalTests/Query/NorthwindGroupByQueryInMemoryTest.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.EntityFrameworkCore.InMemory.Internal; + namespace Microsoft.EntityFrameworkCore.Query; public class NorthwindGroupByQueryInMemoryTest : NorthwindGroupByQueryTestBase> @@ -12,4 +14,54 @@ public NorthwindGroupByQueryInMemoryTest( { //TestLoggerFactory.TestOutputHelper = testOutputHelper; } + + public override Task Final_GroupBy_property_entity(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_anonymous_type(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_anonymous_type(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_multiple_properties_entity(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_multiple_properties_entity(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_complex_key_entity(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_complex_key_entity(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_nominal_type_entity(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_nominal_type_entity(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_anonymous_type_element_selector(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_anonymous_type_element_selector(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_Include_collection(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_Include_collection(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_projecting_collection(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_projecting_collection(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_projecting_collection_composed(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_projecting_collection_composed(async), + InMemoryStrings.UncomposedGroupByNotSupported); + + public override Task Final_GroupBy_property_entity_projecting_collection_and_single_result(bool async) + => AssertTranslationFailedWithDetails( + () => base.Final_GroupBy_property_entity_projecting_collection_and_single_result(async), + InMemoryStrings.UncomposedGroupByNotSupported); } diff --git a/test/EFCore.Specification.Tests/Query/ComplexNavigationsCollectionsQueryTestBase.cs b/test/EFCore.Specification.Tests/Query/ComplexNavigationsCollectionsQueryTestBase.cs index b6e38da3a54..c5d52f46f2d 100644 --- a/test/EFCore.Specification.Tests/Query/ComplexNavigationsCollectionsQueryTestBase.cs +++ b/test/EFCore.Specification.Tests/Query/ComplexNavigationsCollectionsQueryTestBase.cs @@ -2415,4 +2415,85 @@ public virtual Task Projecting_collection_with_group_by_after_optional_reference AssertCollection(e.Collection, a.Collection, elementSorter: ee => ee.Key); } }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_Include_collection(bool async) + => AssertQuery( + async, + ss => ss.Set().Include(l1 => l1.OneToMany_Optional1).GroupBy(l1 => l1.Name), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementAsserter: (ee, aa) => AssertInclude(ee, aa, new ExpectedInclude(i => i.OneToMany_Optional1)))); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_Include_collection_nested(bool async) + => AssertQuery( + async, + ss => ss.Set().Include(l1 => l1.OneToMany_Optional1).ThenInclude(l2 => l2.OneToMany_Optional2).GroupBy(l1 => l1.Name), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementAsserter: (ee, aa) => AssertInclude(ee, aa, + new ExpectedInclude(i => i.OneToMany_Optional1), + new ExpectedInclude(l2 => l2.OneToMany_Optional2, "OneToManyOptional1")))); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_Include_collection_reference(bool async) + => AssertQuery( + async, + ss => ss.Set().Include(l1 => l1.OneToMany_Optional1).ThenInclude(l2 => l2.OneToOne_Optional_FK2).GroupBy(l1 => l1.Name), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementAsserter: (ee, aa) => AssertInclude(ee, aa, + new ExpectedInclude(i => i.OneToMany_Optional1), + new ExpectedInclude(l2 => l2.OneToOne_Optional_FK2, "OneToManyOptional1")))); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_Include_collection_multiple(bool async) + => AssertQuery( + async, + ss => ss.Set().Include(l1 => l1.OneToMany_Optional1).Include(l1 => l1.OneToMany_Required1).GroupBy(l1 => l1.Name), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementAsserter: (ee, aa) => AssertInclude(ee, aa, + new ExpectedInclude(i => i.OneToMany_Optional1), + new ExpectedInclude(l2 => l2.OneToMany_Required1)))); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_Include_collection_reference_same_level(bool async) + => AssertQuery( + async, + ss => ss.Set().Include(l1 => l1.OneToMany_Optional1).Include(l1 => l1.OneToOne_Optional_FK1).GroupBy(l1 => l1.Name), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementAsserter: (ee, aa) => AssertInclude(ee, aa, + new ExpectedInclude(i => i.OneToMany_Optional1), + new ExpectedInclude(l2 => l2.OneToOne_Optional_FK1)))); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_Include_reference(bool async) + => AssertQuery( + async, + ss => ss.Set().Include(l1 => l1.OneToOne_Optional_FK1).GroupBy(l1 => l1.Name), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementAsserter: (ee, aa) => AssertInclude(ee, aa, + new ExpectedInclude(l2 => l2.OneToOne_Optional_FK1)))); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_Include_reference_multiple(bool async) + => AssertQuery( + async, + ss => ss.Set().Include(l1 => l1.OneToOne_Optional_FK1).Include(l1 => l1.OneToOne_Required_FK1).GroupBy(l1 => l1.Name), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementAsserter: (ee, aa) => AssertInclude(ee, aa, + new ExpectedInclude(l2 => l2.OneToOne_Optional_FK1), + new ExpectedInclude(l2 => l2.OneToOne_Required_FK1)))); } diff --git a/test/EFCore.Specification.Tests/Query/NorthwindGroupByQueryTestBase.cs b/test/EFCore.Specification.Tests/Query/NorthwindGroupByQueryTestBase.cs index 965477edae0..06929beada0 100644 --- a/test/EFCore.Specification.Tests/Query/NorthwindGroupByQueryTestBase.cs +++ b/test/EFCore.Specification.Tests/Query/NorthwindGroupByQueryTestBase.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; using Microsoft.EntityFrameworkCore.TestModels.Northwind; namespace Microsoft.EntityFrameworkCore.Query; @@ -2562,7 +2563,8 @@ public virtual Task GroupBy_group_Distinct_Select_Distinct_aggregate(bool async) g => new { - g.Key, Max = g.Distinct().Select(e => e.OrderDate).Distinct().Max(), + g.Key, + Max = g.Distinct().Select(e => e.OrderDate).Distinct().Max(), }), elementSorter: e => e.Key); @@ -2577,21 +2579,183 @@ public virtual Task GroupBy_group_Where_Select_Distinct_aggregate(bool async) g => new { - g.Key, Max = g.Where(e => e.OrderDate.HasValue).Select(e => e.OrderDate).Distinct().Max(), + g.Key, + Max = g.Where(e => e.OrderDate.HasValue).Select(e => e.OrderDate).Distinct().Max(), }), elementSorter: e => e.Key); #endregion - #region GroupByWithoutAggregate + #region FinalGroupBy [ConditionalTheory] [MemberData(nameof(IsAsyncData))] - public virtual Task GroupBy_as_final_operator(bool async) - => AssertTranslationFailed( - () => AssertQuery( - async, - ss => ss.Set().GroupBy(c => c.City))); + public virtual Task Final_GroupBy_property_entity(bool async) + => AssertQuery( + async, + ss => ss.Set().GroupBy(c => c.City), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a), + entryCount: 91); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_anonymous_type(bool async) + => AssertQuery( + async, + ss => ss.Set().Select(e => new { e.City, e.ContactName, e.ContactTitle }).GroupBy(c => c.City), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementSorter: i => (i.ContactName, i.ContactTitle), + elementAsserter: (ee, aa) => + { + AssertEqual(ee.City, aa.City); + AssertEqual(ee.ContactName, aa.ContactName); + AssertEqual(ee.ContactTitle, aa.ContactTitle); + }), + entryCount: 0); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_multiple_properties_entity(bool async) + => AssertQuery( + async, + ss => ss.Set().GroupBy(c => new { c.City, c.Region }), + elementSorter: e => (e.Key.City, e.Key.Region), + elementAsserter: (e, a) => AssertGrouping(e, a, + keyAsserter: (ee, aa) => + { + AssertEqual(ee.City, aa.City); + AssertEqual(ee.Region, aa.Region); + }), + entryCount: 91); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_complex_key_entity(bool async) + => AssertQuery( + async, + ss => ss.Set().GroupBy(c => new { c.City, Inner = new { c.Region, Constant = 1 } }), + elementSorter: e => (e.Key.City, e.Key.Inner.Region), + elementAsserter: (e, a) => AssertGrouping(e, a, + keyAsserter: (ee, aa) => + { + AssertEqual(ee.City, aa.City); + AssertEqual(ee.Inner.Region, aa.Inner.Region); + AssertEqual(ee.Inner.Constant, aa.Inner.Constant); + }), + entryCount: 91); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_nominal_type_entity(bool async) + => AssertQuery( + async, + ss => ss.Set().GroupBy(c => new RandomClass { City = c.City, Constant = 1 }), + ss => ss.Set().GroupBy(c => new RandomClass { City = c.City, Constant = 1 }, new RandomClassEqualityComparer()), + elementSorter: e => e.Key.City, + elementAsserter: (e, a) => AssertGrouping(e, a, keyAsserter: (ee, aa) => AssertEqual(ee.City, aa.City)), + entryCount: 91); + + protected class RandomClass + { + public string City { get; set; } + public int Constant { get; set; } + } + + protected class RandomClassEqualityComparer : IEqualityComparer + { + public bool Equals(RandomClass x, RandomClass y) => x.City == y.City && x.Constant == y.Constant; + public int GetHashCode([DisallowNull] RandomClass obj) => HashCode.Combine(obj.City, obj.Constant); + } + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_anonymous_type_element_selector(bool async) + => AssertQuery( + async, + ss => ss.Set().GroupBy(c => c.City, e => new { e.ContactName, e.ContactTitle }), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementSorter: i => (i.ContactName, i.ContactTitle), + elementAsserter: (ee, aa) => + { + AssertEqual(ee.ContactName, aa.ContactName); + AssertEqual(ee.ContactTitle, aa.ContactTitle); + }), + entryCount: 0); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_Include_collection(bool async) + => AssertQuery( + async, + ss => ss.Set().Where(c => c.Country == "USA").Include(c => c.Orders).GroupBy(c => c.City), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementAsserter: (ee, aa) => AssertInclude(ee, aa, new ExpectedInclude(c => c.Orders))), + entryCount: 135); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_projecting_collection(bool async) + => AssertQuery( + async, + ss => ss.Set().Where(c => c.Country == "USA").Select(c => new { c.City, c.Orders }).GroupBy(c => c.City), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementSorter: ee => ee.City, + elementAsserter: (ee, aa) => + { + AssertEqual(ee.City, aa.City); + AssertCollection(ee.Orders, aa.Orders); + }), + entryCount: 122); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_projecting_collection_composed(bool async) + => AssertQuery( + async, + ss => ss.Set().Where(c => c.Country == "USA") + .Select(c => new { c.City, Orders = c.Orders.Where(o => o.OrderID < 11000) }) + .GroupBy(c => c.City), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementSorter: ee => ee.City, + elementAsserter: (ee, aa) => + { + AssertEqual(ee.City, aa.City); + AssertCollection(ee.Orders, aa.Orders); + }), + entryCount: 108); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Final_GroupBy_property_entity_projecting_collection_and_single_result(bool async) + => AssertQuery( + async, + ss => ss.Set().Where(c => c.Country == "USA") + .Select(c => new + { + c.City, + Orders = c.Orders.Where(o => o.OrderID < 11000), + LastOrder = c.Orders.OrderByDescending(o => o.OrderDate).FirstOrDefault() }) + .GroupBy(c => c.City), + elementSorter: e => e.Key, + elementAsserter: (e, a) => AssertGrouping(e, a, + elementSorter: ee => ee.City, + elementAsserter: (ee, aa) => + { + AssertEqual(ee.City, aa.City); + AssertCollection(ee.Orders, aa.Orders); + AssertEqual(ee.LastOrder, aa.LastOrder); + }), + entryCount: 115); + + #endregion + + #region GroupByWithoutAggregate [ConditionalTheory] [MemberData(nameof(IsAsyncData))] diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsQuerySqlServerTest.cs index 2f62ff199f7..d630d688fbf 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsQuerySqlServerTest.cs @@ -2565,6 +2565,94 @@ WHERE [l1].[Name] <> N'Foo' OR [l1].[Name] IS NULL ORDER BY [l].[Id], [l0].[Id], [t0].[OneToMany_Optional_Inverse3Id], [t0].[Name]"); } + public override async Task Final_GroupBy_property_entity_Include_collection(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id] +FROM [LevelOne] AS [l] +LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_nested(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_nested(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [t].[Id], [t].[Date], [t].[Level1_Optional_Id], [t].[Level1_Required_Id], [t].[Name], [t].[OneToMany_Optional_Inverse2Id], [t].[OneToMany_Optional_Self_Inverse2Id], [t].[OneToMany_Required_Inverse2Id], [t].[OneToMany_Required_Self_Inverse2Id], [t].[OneToOne_Optional_PK_Inverse2Id], [t].[OneToOne_Optional_Self2Id], [t].[Id0], [t].[Level2_Optional_Id], [t].[Level2_Required_Id], [t].[Name0], [t].[OneToMany_Optional_Inverse3Id], [t].[OneToMany_Optional_Self_Inverse3Id], [t].[OneToMany_Required_Inverse3Id], [t].[OneToMany_Required_Self_Inverse3Id], [t].[OneToOne_Optional_PK_Inverse3Id], [t].[OneToOne_Optional_Self3Id] +FROM [LevelOne] AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l1].[Id] AS [Id0], [l1].[Level2_Optional_Id], [l1].[Level2_Required_Id], [l1].[Name] AS [Name0], [l1].[OneToMany_Optional_Inverse3Id], [l1].[OneToMany_Optional_Self_Inverse3Id], [l1].[OneToMany_Required_Inverse3Id], [l1].[OneToMany_Required_Self_Inverse3Id], [l1].[OneToOne_Optional_PK_Inverse3Id], [l1].[OneToOne_Optional_Self3Id] + FROM [LevelTwo] AS [l0] + LEFT JOIN [LevelThree] AS [l1] ON [l0].[Id] = [l1].[OneToMany_Optional_Inverse3Id] +) AS [t] ON [l].[Id] = [t].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [t].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_reference(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_reference(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [t].[Id], [t].[Date], [t].[Level1_Optional_Id], [t].[Level1_Required_Id], [t].[Name], [t].[OneToMany_Optional_Inverse2Id], [t].[OneToMany_Optional_Self_Inverse2Id], [t].[OneToMany_Required_Inverse2Id], [t].[OneToMany_Required_Self_Inverse2Id], [t].[OneToOne_Optional_PK_Inverse2Id], [t].[OneToOne_Optional_Self2Id], [t].[Id0], [t].[Level2_Optional_Id], [t].[Level2_Required_Id], [t].[Name0], [t].[OneToMany_Optional_Inverse3Id], [t].[OneToMany_Optional_Self_Inverse3Id], [t].[OneToMany_Required_Inverse3Id], [t].[OneToMany_Required_Self_Inverse3Id], [t].[OneToOne_Optional_PK_Inverse3Id], [t].[OneToOne_Optional_Self3Id] +FROM [LevelOne] AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l1].[Id] AS [Id0], [l1].[Level2_Optional_Id], [l1].[Level2_Required_Id], [l1].[Name] AS [Name0], [l1].[OneToMany_Optional_Inverse3Id], [l1].[OneToMany_Optional_Self_Inverse3Id], [l1].[OneToMany_Required_Inverse3Id], [l1].[OneToMany_Required_Self_Inverse3Id], [l1].[OneToOne_Optional_PK_Inverse3Id], [l1].[OneToOne_Optional_Self3Id] + FROM [LevelTwo] AS [l0] + LEFT JOIN [LevelThree] AS [l1] ON [l0].[Id] = [l1].[Level2_Optional_Id] +) AS [t] ON [l].[Id] = [t].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [t].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_multiple(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_multiple(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l1].[Id], [l1].[Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Optional_Self_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToMany_Required_Self_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id], [l1].[OneToOne_Optional_Self2Id] +FROM [LevelOne] AS [l] +LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[OneToMany_Optional_Inverse2Id] +LEFT JOIN [LevelTwo] AS [l1] ON [l].[Id] = [l1].[OneToMany_Required_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [l0].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_reference_same_level(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_reference_same_level(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l0].[Id], [l1].[Id], [l1].[Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Optional_Self_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToMany_Required_Self_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id], [l1].[OneToOne_Optional_Self2Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id] +FROM [LevelOne] AS [l] +LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[Level1_Optional_Id] +LEFT JOIN [LevelTwo] AS [l1] ON [l].[Id] = [l1].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [l0].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_reference(bool async) + { + await base.Final_GroupBy_property_entity_Include_reference(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id] +FROM [LevelOne] AS [l] +LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[Level1_Optional_Id] +ORDER BY [l].[Name]"); + } + + public override async Task Final_GroupBy_property_entity_Include_reference_multiple(bool async) + { + await base.Final_GroupBy_property_entity_Include_reference_multiple(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l1].[Id], [l1].[Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Optional_Self_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToMany_Required_Self_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id], [l1].[OneToOne_Optional_Self2Id] +FROM [LevelOne] AS [l] +LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[Level1_Optional_Id] +LEFT JOIN [LevelTwo] AS [l1] ON [l].[Id] = [l1].[Level1_Required_Id] +ORDER BY [l].[Name]"); + } + private void AssertSql(params string[] expected) => Fixture.TestSqlLoggerFactory.AssertBaseline(expected); } diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsSharedTypeQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsSharedTypeQuerySqlServerTest.cs index 8bb0537cb63..c325d176c8a 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsSharedTypeQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsSharedTypeQuerySqlServerTest.cs @@ -3447,6 +3447,140 @@ WHEN [t].[OneToOne_Required_PK_Date] IS NOT NULL AND [t].[Level1_Required_Id] IS ORDER BY [l].[Id], [t].[Id], [t0].[OneToMany_Optional_Inverse3Id], [t0].[Level3_Name]"); } + public override async Task Final_GroupBy_property_entity_Include_collection(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [t].[Id], [t].[OneToOne_Required_PK_Date], [t].[Level1_Optional_Id], [t].[Level1_Required_Id], [t].[Level2_Name], [t].[OneToMany_Optional_Inverse2Id], [t].[OneToMany_Required_Inverse2Id], [t].[OneToOne_Optional_PK_Inverse2Id] +FROM [Level1] AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[OneToOne_Required_PK_Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Level2_Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id] + FROM [Level1] AS [l0] + WHERE [l0].[OneToOne_Required_PK_Date] IS NOT NULL AND [l0].[Level1_Required_Id] IS NOT NULL AND [l0].[OneToMany_Required_Inverse2Id] IS NOT NULL +) AS [t] ON [l].[Id] = [t].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_nested(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_nested(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [t0].[Id], [t0].[OneToOne_Required_PK_Date], [t0].[Level1_Optional_Id], [t0].[Level1_Required_Id], [t0].[Level2_Name], [t0].[OneToMany_Optional_Inverse2Id], [t0].[OneToMany_Required_Inverse2Id], [t0].[OneToOne_Optional_PK_Inverse2Id], [t0].[Id0], [t0].[Level2_Optional_Id], [t0].[Level2_Required_Id], [t0].[Level3_Name], [t0].[OneToMany_Optional_Inverse3Id], [t0].[OneToMany_Required_Inverse3Id], [t0].[OneToOne_Optional_PK_Inverse3Id] +FROM [Level1] AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[OneToOne_Required_PK_Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Level2_Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [t].[Id] AS [Id0], [t].[Level2_Optional_Id], [t].[Level2_Required_Id], [t].[Level3_Name], [t].[OneToMany_Optional_Inverse3Id], [t].[OneToMany_Required_Inverse3Id], [t].[OneToOne_Optional_PK_Inverse3Id] + FROM [Level1] AS [l0] + LEFT JOIN ( + SELECT [l1].[Id], [l1].[Level2_Optional_Id], [l1].[Level2_Required_Id], [l1].[Level3_Name], [l1].[OneToMany_Optional_Inverse3Id], [l1].[OneToMany_Required_Inverse3Id], [l1].[OneToOne_Optional_PK_Inverse3Id] + FROM [Level1] AS [l1] + WHERE [l1].[Level2_Required_Id] IS NOT NULL AND [l1].[OneToMany_Required_Inverse3Id] IS NOT NULL + ) AS [t] ON CASE + WHEN [l0].[OneToOne_Required_PK_Date] IS NOT NULL AND [l0].[Level1_Required_Id] IS NOT NULL AND [l0].[OneToMany_Required_Inverse2Id] IS NOT NULL THEN [l0].[Id] + END = [t].[OneToMany_Optional_Inverse3Id] + WHERE [l0].[OneToOne_Required_PK_Date] IS NOT NULL AND [l0].[Level1_Required_Id] IS NOT NULL AND [l0].[OneToMany_Required_Inverse2Id] IS NOT NULL +) AS [t0] ON [l].[Id] = [t0].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [t0].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_reference(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_reference(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [t0].[Id], [t0].[OneToOne_Required_PK_Date], [t0].[Level1_Optional_Id], [t0].[Level1_Required_Id], [t0].[Level2_Name], [t0].[OneToMany_Optional_Inverse2Id], [t0].[OneToMany_Required_Inverse2Id], [t0].[OneToOne_Optional_PK_Inverse2Id], [t0].[Id0], [t0].[Level2_Optional_Id], [t0].[Level2_Required_Id], [t0].[Level3_Name], [t0].[OneToMany_Optional_Inverse3Id], [t0].[OneToMany_Required_Inverse3Id], [t0].[OneToOne_Optional_PK_Inverse3Id] +FROM [Level1] AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[OneToOne_Required_PK_Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Level2_Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [t].[Id] AS [Id0], [t].[Level2_Optional_Id], [t].[Level2_Required_Id], [t].[Level3_Name], [t].[OneToMany_Optional_Inverse3Id], [t].[OneToMany_Required_Inverse3Id], [t].[OneToOne_Optional_PK_Inverse3Id] + FROM [Level1] AS [l0] + LEFT JOIN ( + SELECT [l1].[Id], [l1].[Level2_Optional_Id], [l1].[Level2_Required_Id], [l1].[Level3_Name], [l1].[OneToMany_Optional_Inverse3Id], [l1].[OneToMany_Required_Inverse3Id], [l1].[OneToOne_Optional_PK_Inverse3Id] + FROM [Level1] AS [l1] + WHERE [l1].[Level2_Required_Id] IS NOT NULL AND [l1].[OneToMany_Required_Inverse3Id] IS NOT NULL + ) AS [t] ON CASE + WHEN [l0].[OneToOne_Required_PK_Date] IS NOT NULL AND [l0].[Level1_Required_Id] IS NOT NULL AND [l0].[OneToMany_Required_Inverse2Id] IS NOT NULL THEN [l0].[Id] + END = [t].[Level2_Optional_Id] + WHERE [l0].[OneToOne_Required_PK_Date] IS NOT NULL AND [l0].[Level1_Required_Id] IS NOT NULL AND [l0].[OneToMany_Required_Inverse2Id] IS NOT NULL +) AS [t0] ON [l].[Id] = [t0].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [t0].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_multiple(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_multiple(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [t].[Id], [t].[OneToOne_Required_PK_Date], [t].[Level1_Optional_Id], [t].[Level1_Required_Id], [t].[Level2_Name], [t].[OneToMany_Optional_Inverse2Id], [t].[OneToMany_Required_Inverse2Id], [t].[OneToOne_Optional_PK_Inverse2Id], [t0].[Id], [t0].[OneToOne_Required_PK_Date], [t0].[Level1_Optional_Id], [t0].[Level1_Required_Id], [t0].[Level2_Name], [t0].[OneToMany_Optional_Inverse2Id], [t0].[OneToMany_Required_Inverse2Id], [t0].[OneToOne_Optional_PK_Inverse2Id] +FROM [Level1] AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[OneToOne_Required_PK_Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Level2_Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id] + FROM [Level1] AS [l0] + WHERE [l0].[OneToOne_Required_PK_Date] IS NOT NULL AND [l0].[Level1_Required_Id] IS NOT NULL AND [l0].[OneToMany_Required_Inverse2Id] IS NOT NULL +) AS [t] ON [l].[Id] = [t].[OneToMany_Optional_Inverse2Id] +LEFT JOIN ( + SELECT [l1].[Id], [l1].[OneToOne_Required_PK_Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Level2_Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id] + FROM [Level1] AS [l1] + WHERE [l1].[OneToOne_Required_PK_Date] IS NOT NULL AND [l1].[Level1_Required_Id] IS NOT NULL AND [l1].[OneToMany_Required_Inverse2Id] IS NOT NULL +) AS [t0] ON [l].[Id] = [t0].[OneToMany_Required_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [t].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_reference_same_level(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_reference_same_level(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [t].[Id], [t0].[Id], [t0].[OneToOne_Required_PK_Date], [t0].[Level1_Optional_Id], [t0].[Level1_Required_Id], [t0].[Level2_Name], [t0].[OneToMany_Optional_Inverse2Id], [t0].[OneToMany_Required_Inverse2Id], [t0].[OneToOne_Optional_PK_Inverse2Id], [t].[OneToOne_Required_PK_Date], [t].[Level1_Optional_Id], [t].[Level1_Required_Id], [t].[Level2_Name], [t].[OneToMany_Optional_Inverse2Id], [t].[OneToMany_Required_Inverse2Id], [t].[OneToOne_Optional_PK_Inverse2Id] +FROM [Level1] AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[OneToOne_Required_PK_Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Level2_Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id] + FROM [Level1] AS [l0] + WHERE [l0].[OneToOne_Required_PK_Date] IS NOT NULL AND [l0].[Level1_Required_Id] IS NOT NULL AND [l0].[OneToMany_Required_Inverse2Id] IS NOT NULL +) AS [t] ON [l].[Id] = [t].[Level1_Optional_Id] +LEFT JOIN ( + SELECT [l1].[Id], [l1].[OneToOne_Required_PK_Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Level2_Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id] + FROM [Level1] AS [l1] + WHERE [l1].[OneToOne_Required_PK_Date] IS NOT NULL AND [l1].[Level1_Required_Id] IS NOT NULL AND [l1].[OneToMany_Required_Inverse2Id] IS NOT NULL +) AS [t0] ON [l].[Id] = [t0].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [t].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_reference(bool async) + { + await base.Final_GroupBy_property_entity_Include_reference(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [t].[Id], [t].[OneToOne_Required_PK_Date], [t].[Level1_Optional_Id], [t].[Level1_Required_Id], [t].[Level2_Name], [t].[OneToMany_Optional_Inverse2Id], [t].[OneToMany_Required_Inverse2Id], [t].[OneToOne_Optional_PK_Inverse2Id] +FROM [Level1] AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[OneToOne_Required_PK_Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Level2_Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id] + FROM [Level1] AS [l0] + WHERE [l0].[OneToOne_Required_PK_Date] IS NOT NULL AND [l0].[Level1_Required_Id] IS NOT NULL AND [l0].[OneToMany_Required_Inverse2Id] IS NOT NULL +) AS [t] ON [l].[Id] = [t].[Level1_Optional_Id] +ORDER BY [l].[Name]"); + } + + public override async Task Final_GroupBy_property_entity_Include_reference_multiple(bool async) + { + await base.Final_GroupBy_property_entity_Include_reference_multiple(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [t].[Id], [t].[OneToOne_Required_PK_Date], [t].[Level1_Optional_Id], [t].[Level1_Required_Id], [t].[Level2_Name], [t].[OneToMany_Optional_Inverse2Id], [t].[OneToMany_Required_Inverse2Id], [t].[OneToOne_Optional_PK_Inverse2Id], [t0].[Id], [t0].[OneToOne_Required_PK_Date], [t0].[Level1_Optional_Id], [t0].[Level1_Required_Id], [t0].[Level2_Name], [t0].[OneToMany_Optional_Inverse2Id], [t0].[OneToMany_Required_Inverse2Id], [t0].[OneToOne_Optional_PK_Inverse2Id] +FROM [Level1] AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[OneToOne_Required_PK_Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Level2_Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id] + FROM [Level1] AS [l0] + WHERE [l0].[OneToOne_Required_PK_Date] IS NOT NULL AND [l0].[Level1_Required_Id] IS NOT NULL AND [l0].[OneToMany_Required_Inverse2Id] IS NOT NULL +) AS [t] ON [l].[Id] = [t].[Level1_Optional_Id] +LEFT JOIN ( + SELECT [l1].[Id], [l1].[OneToOne_Required_PK_Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Level2_Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id] + FROM [Level1] AS [l1] + WHERE [l1].[OneToOne_Required_PK_Date] IS NOT NULL AND [l1].[Level1_Required_Id] IS NOT NULL AND [l1].[OneToMany_Required_Inverse2Id] IS NOT NULL +) AS [t0] ON [l].[Id] = [t0].[Level1_Required_Id] +ORDER BY [l].[Name]"); + } + private void AssertSql(params string[] expected) => Fixture.TestSqlLoggerFactory.AssertBaseline(expected); } diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsSplitQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsSplitQuerySqlServerTest.cs index 793a61e4db2..b4f78d1bae8 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsSplitQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsCollectionsSplitQuerySqlServerTest.cs @@ -3607,6 +3607,121 @@ WHERE [l1].[Name] <> N'Foo' OR [l1].[Name] IS NULL ORDER BY [l].[Id], [l0].[Id], [t0].[OneToMany_Optional_Inverse3Id], [t0].[Name]"); } + public override async Task Final_GroupBy_property_entity_Include_collection(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id] +FROM [LevelOne] AS [l] +ORDER BY [l].[Name], [l].[Id]", + // + @"SELECT [l].[Name], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l].[Id] +FROM [LevelOne] AS [l] +INNER JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_nested(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_nested(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id] +FROM [LevelOne] AS [l] +ORDER BY [l].[Name], [l].[Id]", + // + @"SELECT [l].[Name], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l].[Id] +FROM [LevelOne] AS [l] +INNER JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [l0].[Id]", + // + @"SELECT [l].[Name], [l1].[Id], [l1].[Level2_Optional_Id], [l1].[Level2_Required_Id], [l1].[Name], [l1].[OneToMany_Optional_Inverse3Id], [l1].[OneToMany_Optional_Self_Inverse3Id], [l1].[OneToMany_Required_Inverse3Id], [l1].[OneToMany_Required_Self_Inverse3Id], [l1].[OneToOne_Optional_PK_Inverse3Id], [l1].[OneToOne_Optional_Self3Id], [l].[Id], [l0].[Id] +FROM [LevelOne] AS [l] +INNER JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[OneToMany_Optional_Inverse2Id] +INNER JOIN [LevelThree] AS [l1] ON [l0].[Id] = [l1].[OneToMany_Optional_Inverse3Id] +ORDER BY [l].[Name], [l].[Id], [l0].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_reference(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_reference(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id] +FROM [LevelOne] AS [l] +ORDER BY [l].[Name], [l].[Id]", + // + @"SELECT [l].[Name], [t].[Id], [t].[Date], [t].[Level1_Optional_Id], [t].[Level1_Required_Id], [t].[Name], [t].[OneToMany_Optional_Inverse2Id], [t].[OneToMany_Optional_Self_Inverse2Id], [t].[OneToMany_Required_Inverse2Id], [t].[OneToMany_Required_Self_Inverse2Id], [t].[OneToOne_Optional_PK_Inverse2Id], [t].[OneToOne_Optional_Self2Id], [t].[Id0], [t].[Level2_Optional_Id], [t].[Level2_Required_Id], [t].[Name0], [t].[OneToMany_Optional_Inverse3Id], [t].[OneToMany_Optional_Self_Inverse3Id], [t].[OneToMany_Required_Inverse3Id], [t].[OneToMany_Required_Self_Inverse3Id], [t].[OneToOne_Optional_PK_Inverse3Id], [t].[OneToOne_Optional_Self3Id], [l].[Id] +FROM [LevelOne] AS [l] +INNER JOIN ( + SELECT [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l1].[Id] AS [Id0], [l1].[Level2_Optional_Id], [l1].[Level2_Required_Id], [l1].[Name] AS [Name0], [l1].[OneToMany_Optional_Inverse3Id], [l1].[OneToMany_Optional_Self_Inverse3Id], [l1].[OneToMany_Required_Inverse3Id], [l1].[OneToMany_Required_Self_Inverse3Id], [l1].[OneToOne_Optional_PK_Inverse3Id], [l1].[OneToOne_Optional_Self3Id] + FROM [LevelTwo] AS [l0] + LEFT JOIN [LevelThree] AS [l1] ON [l0].[Id] = [l1].[Level2_Optional_Id] +) AS [t] ON [l].[Id] = [t].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_multiple(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_multiple(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id] +FROM [LevelOne] AS [l] +ORDER BY [l].[Name], [l].[Id]", + // + @"SELECT [l].[Name], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l].[Id] +FROM [LevelOne] AS [l] +INNER JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id]", + // + @"SELECT [l].[Name], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l].[Id] +FROM [LevelOne] AS [l] +INNER JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[OneToMany_Required_Inverse2Id] +ORDER BY [l].[Name], [l].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_reference_same_level(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_reference_same_level(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id] +FROM [LevelOne] AS [l] +LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[Level1_Optional_Id] +ORDER BY [l].[Name], [l].[Id], [l0].[Id]", + // + @"SELECT [l].[Name], [l1].[Id], [l1].[Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Optional_Self_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToMany_Required_Self_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id], [l1].[OneToOne_Optional_Self2Id], [l].[Id], [l0].[Id] +FROM [LevelOne] AS [l] +LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[Level1_Optional_Id] +INNER JOIN [LevelTwo] AS [l1] ON [l].[Id] = [l1].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [l0].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_reference(bool async) + { + await base.Final_GroupBy_property_entity_Include_reference(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id] +FROM [LevelOne] AS [l] +LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[Level1_Optional_Id] +ORDER BY [l].[Name]"); + } + + public override async Task Final_GroupBy_property_entity_Include_reference_multiple(bool async) + { + await base.Final_GroupBy_property_entity_Include_reference_multiple(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l1].[Id], [l1].[Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Optional_Self_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToMany_Required_Self_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id], [l1].[OneToOne_Optional_Self2Id] +FROM [LevelOne] AS [l] +LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[Level1_Optional_Id] +LEFT JOIN [LevelTwo] AS [l1] ON [l].[Id] = [l1].[Level1_Required_Id] +ORDER BY [l].[Name]"); + } + private void AssertSql(params string[] expected) => Fixture.TestSqlLoggerFactory.AssertBaseline(expected); } diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/NorthwindGroupByQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/NorthwindGroupByQuerySqlServerTest.cs index b38a1421882..d049655f39e 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/NorthwindGroupByQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/NorthwindGroupByQuerySqlServerTest.cs @@ -3119,11 +3119,134 @@ public override async Task GroupBy_aggregate_SelectMany(bool async) AssertSql(); } - public override async Task GroupBy_as_final_operator(bool async) + public override async Task Final_GroupBy_property_entity(bool async) { - await base.GroupBy_as_final_operator(async); + await base.Final_GroupBy_property_entity(async); - AssertSql(); + AssertSql( + @"SELECT [c].[City], [c].[CustomerID], [c].[Address], [c].[CompanyName], [c].[ContactName], [c].[ContactTitle], [c].[Country], [c].[Fax], [c].[Phone], [c].[PostalCode], [c].[Region] +FROM [Customers] AS [c] +ORDER BY [c].[City]"); + } + + public override async Task Final_GroupBy_property_anonymous_type(bool async) + { + await base.Final_GroupBy_property_anonymous_type(async); + + AssertSql( + @"SELECT [c].[City], [c].[ContactName], [c].[ContactTitle] +FROM [Customers] AS [c] +ORDER BY [c].[City]"); + } + + public override async Task Final_GroupBy_multiple_properties_entity(bool async) + { + await base.Final_GroupBy_multiple_properties_entity(async); + + AssertSql( + @"SELECT [c].[City], [c].[Region], [c].[CustomerID], [c].[Address], [c].[CompanyName], [c].[ContactName], [c].[ContactTitle], [c].[Country], [c].[Fax], [c].[Phone], [c].[PostalCode] +FROM [Customers] AS [c] +ORDER BY [c].[City], [c].[Region]"); + } + + public override async Task Final_GroupBy_complex_key_entity(bool async) + { + await base.Final_GroupBy_complex_key_entity(async); + + AssertSql( + @"SELECT [t].[City], [t].[Region], [t].[Constant], [t].[CustomerID], [t].[Address], [t].[CompanyName], [t].[ContactName], [t].[ContactTitle], [t].[Country], [t].[Fax], [t].[Phone], [t].[PostalCode] +FROM ( + SELECT [c].[CustomerID], [c].[Address], [c].[City], [c].[CompanyName], [c].[ContactName], [c].[ContactTitle], [c].[Country], [c].[Fax], [c].[Phone], [c].[PostalCode], [c].[Region], 1 AS [Constant] + FROM [Customers] AS [c] +) AS [t] +ORDER BY [t].[City], [t].[Region], [t].[Constant]"); + } + + public override async Task Final_GroupBy_nominal_type_entity(bool async) + { + await base.Final_GroupBy_nominal_type_entity(async); + + AssertSql( + @"SELECT [t].[City], [t].[Constant], [t].[CustomerID], [t].[Address], [t].[CompanyName], [t].[ContactName], [t].[ContactTitle], [t].[Country], [t].[Fax], [t].[Phone], [t].[PostalCode], [t].[Region] +FROM ( + SELECT [c].[CustomerID], [c].[Address], [c].[City], [c].[CompanyName], [c].[ContactName], [c].[ContactTitle], [c].[Country], [c].[Fax], [c].[Phone], [c].[PostalCode], [c].[Region], 1 AS [Constant] + FROM [Customers] AS [c] +) AS [t] +ORDER BY [t].[City], [t].[Constant]"); + } + + public override async Task Final_GroupBy_property_anonymous_type_element_selector(bool async) + { + await base.Final_GroupBy_property_anonymous_type_element_selector(async); + + AssertSql( + @"SELECT [c].[City], [c].[ContactName], [c].[ContactTitle] +FROM [Customers] AS [c] +ORDER BY [c].[City]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection(async); + + AssertSql( + @"SELECT [c].[City], [c].[CustomerID], [c].[Address], [c].[CompanyName], [c].[ContactName], [c].[ContactTitle], [c].[Country], [c].[Fax], [c].[Phone], [c].[PostalCode], [c].[Region], [o].[OrderID], [o].[CustomerID], [o].[EmployeeID], [o].[OrderDate] +FROM [Customers] AS [c] +LEFT JOIN [Orders] AS [o] ON [c].[CustomerID] = [o].[CustomerID] +WHERE [c].[Country] = N'USA' +ORDER BY [c].[City], [c].[CustomerID]"); + } + + public override async Task Final_GroupBy_property_entity_projecting_collection(bool async) + { + await base.Final_GroupBy_property_entity_projecting_collection(async); + + AssertSql( + @"SELECT [c].[City], [c].[CustomerID], [o].[OrderID], [o].[CustomerID], [o].[EmployeeID], [o].[OrderDate] +FROM [Customers] AS [c] +LEFT JOIN [Orders] AS [o] ON [c].[CustomerID] = [o].[CustomerID] +WHERE [c].[Country] = N'USA' +ORDER BY [c].[City], [c].[CustomerID]"); + } + + public override async Task Final_GroupBy_property_entity_projecting_collection_composed(bool async) + { + await base.Final_GroupBy_property_entity_projecting_collection_composed(async); + + AssertSql( + @"SELECT [c].[City], [c].[CustomerID], [t].[OrderID], [t].[CustomerID], [t].[EmployeeID], [t].[OrderDate] +FROM [Customers] AS [c] +LEFT JOIN ( + SELECT [o].[OrderID], [o].[CustomerID], [o].[EmployeeID], [o].[OrderDate] + FROM [Orders] AS [o] + WHERE [o].[OrderID] < 11000 +) AS [t] ON [c].[CustomerID] = [t].[CustomerID] +WHERE [c].[Country] = N'USA' +ORDER BY [c].[City], [c].[CustomerID]"); + } + + public override async Task Final_GroupBy_property_entity_projecting_collection_and_single_result(bool async) + { + await base.Final_GroupBy_property_entity_projecting_collection_and_single_result(async); + + AssertSql( + @"SELECT [c].[City], [c].[CustomerID], [t].[OrderID], [t].[CustomerID], [t].[EmployeeID], [t].[OrderDate], [t0].[OrderID], [t0].[CustomerID], [t0].[EmployeeID], [t0].[OrderDate] +FROM [Customers] AS [c] +LEFT JOIN ( + SELECT [o].[OrderID], [o].[CustomerID], [o].[EmployeeID], [o].[OrderDate] + FROM [Orders] AS [o] + WHERE [o].[OrderID] < 11000 +) AS [t] ON [c].[CustomerID] = [t].[CustomerID] +LEFT JOIN ( + SELECT [t1].[OrderID], [t1].[CustomerID], [t1].[EmployeeID], [t1].[OrderDate] + FROM ( + SELECT [o0].[OrderID], [o0].[CustomerID], [o0].[EmployeeID], [o0].[OrderDate], ROW_NUMBER() OVER(PARTITION BY [o0].[CustomerID] ORDER BY [o0].[OrderDate] DESC) AS [row] + FROM [Orders] AS [o0] + ) AS [t1] + WHERE [t1].[row] <= 1 +) AS [t0] ON [c].[CustomerID] = [t0].[CustomerID] +WHERE [c].[Country] = N'USA' +ORDER BY [c].[City], [c].[CustomerID]"); } public override async Task GroupBy_Where_with_grouping_result(bool async) diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/TemporalComplexNavigationsCollectionsQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/TemporalComplexNavigationsCollectionsQuerySqlServerTest.cs index d4d30f74d12..3fec739dfd9 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/TemporalComplexNavigationsCollectionsQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/TemporalComplexNavigationsCollectionsQuerySqlServerTest.cs @@ -2583,6 +2583,94 @@ WHERE [l1].[Name] <> N'Foo' OR [l1].[Name] IS NULL ORDER BY [l].[Id], [l0].[Id], [t0].[OneToMany_Optional_Inverse3Id], [t0].[Name]"); } + public override async Task Final_GroupBy_property_entity_Include_collection(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l].[PeriodEnd], [l].[PeriodStart], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l0].[PeriodEnd], [l0].[PeriodStart] +FROM [LevelOne] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l] +LEFT JOIN [LevelTwo] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l0] ON [l].[Id] = [l0].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_nested(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_nested(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l].[PeriodEnd], [l].[PeriodStart], [t].[Id], [t].[Date], [t].[Level1_Optional_Id], [t].[Level1_Required_Id], [t].[Name], [t].[OneToMany_Optional_Inverse2Id], [t].[OneToMany_Optional_Self_Inverse2Id], [t].[OneToMany_Required_Inverse2Id], [t].[OneToMany_Required_Self_Inverse2Id], [t].[OneToOne_Optional_PK_Inverse2Id], [t].[OneToOne_Optional_Self2Id], [t].[PeriodEnd], [t].[PeriodStart], [t].[Id0], [t].[Level2_Optional_Id], [t].[Level2_Required_Id], [t].[Name0], [t].[OneToMany_Optional_Inverse3Id], [t].[OneToMany_Optional_Self_Inverse3Id], [t].[OneToMany_Required_Inverse3Id], [t].[OneToMany_Required_Self_Inverse3Id], [t].[OneToOne_Optional_PK_Inverse3Id], [t].[OneToOne_Optional_Self3Id], [t].[PeriodEnd0], [t].[PeriodStart0] +FROM [LevelOne] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l0].[PeriodEnd], [l0].[PeriodStart], [l1].[Id] AS [Id0], [l1].[Level2_Optional_Id], [l1].[Level2_Required_Id], [l1].[Name] AS [Name0], [l1].[OneToMany_Optional_Inverse3Id], [l1].[OneToMany_Optional_Self_Inverse3Id], [l1].[OneToMany_Required_Inverse3Id], [l1].[OneToMany_Required_Self_Inverse3Id], [l1].[OneToOne_Optional_PK_Inverse3Id], [l1].[OneToOne_Optional_Self3Id], [l1].[PeriodEnd] AS [PeriodEnd0], [l1].[PeriodStart] AS [PeriodStart0] + FROM [LevelTwo] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l0] + LEFT JOIN [LevelThree] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l1] ON [l0].[Id] = [l1].[OneToMany_Optional_Inverse3Id] +) AS [t] ON [l].[Id] = [t].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [t].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_reference(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_reference(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l].[PeriodEnd], [l].[PeriodStart], [t].[Id], [t].[Date], [t].[Level1_Optional_Id], [t].[Level1_Required_Id], [t].[Name], [t].[OneToMany_Optional_Inverse2Id], [t].[OneToMany_Optional_Self_Inverse2Id], [t].[OneToMany_Required_Inverse2Id], [t].[OneToMany_Required_Self_Inverse2Id], [t].[OneToOne_Optional_PK_Inverse2Id], [t].[OneToOne_Optional_Self2Id], [t].[PeriodEnd], [t].[PeriodStart], [t].[Id0], [t].[Level2_Optional_Id], [t].[Level2_Required_Id], [t].[Name0], [t].[OneToMany_Optional_Inverse3Id], [t].[OneToMany_Optional_Self_Inverse3Id], [t].[OneToMany_Required_Inverse3Id], [t].[OneToMany_Required_Self_Inverse3Id], [t].[OneToOne_Optional_PK_Inverse3Id], [t].[OneToOne_Optional_Self3Id], [t].[PeriodEnd0], [t].[PeriodStart0] +FROM [LevelOne] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l] +LEFT JOIN ( + SELECT [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l0].[PeriodEnd], [l0].[PeriodStart], [l1].[Id] AS [Id0], [l1].[Level2_Optional_Id], [l1].[Level2_Required_Id], [l1].[Name] AS [Name0], [l1].[OneToMany_Optional_Inverse3Id], [l1].[OneToMany_Optional_Self_Inverse3Id], [l1].[OneToMany_Required_Inverse3Id], [l1].[OneToMany_Required_Self_Inverse3Id], [l1].[OneToOne_Optional_PK_Inverse3Id], [l1].[OneToOne_Optional_Self3Id], [l1].[PeriodEnd] AS [PeriodEnd0], [l1].[PeriodStart] AS [PeriodStart0] + FROM [LevelTwo] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l0] + LEFT JOIN [LevelThree] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l1] ON [l0].[Id] = [l1].[Level2_Optional_Id] +) AS [t] ON [l].[Id] = [t].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [t].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_multiple(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_multiple(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l].[PeriodEnd], [l].[PeriodStart], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l0].[PeriodEnd], [l0].[PeriodStart], [l1].[Id], [l1].[Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Optional_Self_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToMany_Required_Self_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id], [l1].[OneToOne_Optional_Self2Id], [l1].[PeriodEnd], [l1].[PeriodStart] +FROM [LevelOne] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l] +LEFT JOIN [LevelTwo] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l0] ON [l].[Id] = [l0].[OneToMany_Optional_Inverse2Id] +LEFT JOIN [LevelTwo] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l1] ON [l].[Id] = [l1].[OneToMany_Required_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [l0].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_collection_reference_same_level(bool async) + { + await base.Final_GroupBy_property_entity_Include_collection_reference_same_level(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l].[PeriodEnd], [l].[PeriodStart], [l0].[Id], [l1].[Id], [l1].[Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Optional_Self_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToMany_Required_Self_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id], [l1].[OneToOne_Optional_Self2Id], [l1].[PeriodEnd], [l1].[PeriodStart], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l0].[PeriodEnd], [l0].[PeriodStart] +FROM [LevelOne] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l] +LEFT JOIN [LevelTwo] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l0] ON [l].[Id] = [l0].[Level1_Optional_Id] +LEFT JOIN [LevelTwo] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l1] ON [l].[Id] = [l1].[OneToMany_Optional_Inverse2Id] +ORDER BY [l].[Name], [l].[Id], [l0].[Id]"); + } + + public override async Task Final_GroupBy_property_entity_Include_reference(bool async) + { + await base.Final_GroupBy_property_entity_Include_reference(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l].[PeriodEnd], [l].[PeriodStart], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l0].[PeriodEnd], [l0].[PeriodStart] +FROM [LevelOne] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l] +LEFT JOIN [LevelTwo] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l0] ON [l].[Id] = [l0].[Level1_Optional_Id] +ORDER BY [l].[Name]"); + } + + public override async Task Final_GroupBy_property_entity_Include_reference_multiple(bool async) + { + await base.Final_GroupBy_property_entity_Include_reference_multiple(async); + + AssertSql( + @"SELECT [l].[Name], [l].[Id], [l].[Date], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id], [l].[PeriodEnd], [l].[PeriodStart], [l0].[Id], [l0].[Date], [l0].[Level1_Optional_Id], [l0].[Level1_Required_Id], [l0].[Name], [l0].[OneToMany_Optional_Inverse2Id], [l0].[OneToMany_Optional_Self_Inverse2Id], [l0].[OneToMany_Required_Inverse2Id], [l0].[OneToMany_Required_Self_Inverse2Id], [l0].[OneToOne_Optional_PK_Inverse2Id], [l0].[OneToOne_Optional_Self2Id], [l0].[PeriodEnd], [l0].[PeriodStart], [l1].[Id], [l1].[Date], [l1].[Level1_Optional_Id], [l1].[Level1_Required_Id], [l1].[Name], [l1].[OneToMany_Optional_Inverse2Id], [l1].[OneToMany_Optional_Self_Inverse2Id], [l1].[OneToMany_Required_Inverse2Id], [l1].[OneToMany_Required_Self_Inverse2Id], [l1].[OneToOne_Optional_PK_Inverse2Id], [l1].[OneToOne_Optional_Self2Id], [l1].[PeriodEnd], [l1].[PeriodStart] +FROM [LevelOne] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l] +LEFT JOIN [LevelTwo] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l0] ON [l].[Id] = [l0].[Level1_Optional_Id] +LEFT JOIN [LevelTwo] FOR SYSTEM_TIME AS OF '2010-01-01T00:00:00.0000000' AS [l1] ON [l].[Id] = [l1].[Level1_Required_Id] +ORDER BY [l].[Name]"); + } + private void AssertSql(params string[] expected) => Fixture.TestSqlLoggerFactory.AssertBaseline(expected); }