Query: Add support for custom EntityQueryables

This makes look up for EntityQueryable in query tree bit more complex but it adds support for providers to add custom EntityQueryable to carry more metadata as needs to and allow rest of the pipeline to be transparent about it.
Current FromSql has be to converted during compilation phase since, we want to parameterize the arguments array in the methodCall.
If there is nothing to parameterize then providers can directly create custom Queryable.

Resolves #16326

src/EFCore.Relational/Infrastructure/EntityFrameworkRelationalServicesBuilder.cs

@@ -167,6 +167,7 @@ public override EntityFrameworkServicesBuilder TryAddCoreServices()
             TryAdd<IMethodCallTranslatorProvider, RelationalMethodCallTranslatorProvider>();
             TryAdd<IMemberTranslatorProvider, RelationalMemberTranslatorProvider>();
             TryAdd<IShapedQueryOptimizerFactory, RelationalShapedQueryOptimizerFactory>();
+            TryAdd<IQueryOptimizerFactory, RelationalQueryOptimizerFactory>();
             TryAdd<IRelationalSqlTranslatingExpressionVisitorFactory, RelationalSqlTranslatingExpressionVisitorFactory>();
             TryAdd<ISqlExpressionFactory, SqlExpressionFactory>();
 
@@ -188,6 +189,7 @@ public override EntityFrameworkServicesBuilder TryAddCoreServices()
                 .AddDependencySingleton<QuerySqlGeneratorDependencies>()
                 .AddDependencySingleton<RelationalShapedQueryCompilingExpressionVisitorDependencies>()
                 .AddDependencySingleton<RelationalShapedQueryOptimizerDependencies>()
+                .AddDependencySingleton<RelationalQueryOptimizerDependencies>()
                 .AddDependencyScoped<MigrationsSqlGeneratorDependencies>()
                 .AddDependencyScoped<RelationalConventionSetBuilderDependencies>()
                 .AddDependencyScoped<ModificationCommandBatchFactoryDependencies>()

src/EFCore.Relational/Query/Internal/FromSqlEntityQueryable.cs

@@ -0,0 +1,46 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Linq.Expressions;
+
+namespace Microsoft.EntityFrameworkCore.Query.Internal
+{
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public class FromSqlEntityQueryable<TResult> : EntityQueryable<TResult>
+    {
+        /// <summary>
+        ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+        ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+        ///     any release. You should only use it directly in your code with extreme caution and knowing that
+        ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+        /// </summary>
+        public FromSqlEntityQueryable(IAsyncQueryProvider queryProvider, string sql, Expression arguments)
+            : base(queryProvider)
+        {
+            Sql = sql;
+            Arguments = arguments;
+            Expression = Expression.Constant(this);
+        }
+
+        /// <summary>
+        ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+        ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+        ///     any release. You should only use it directly in your code with extreme caution and knowing that
+        ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+        /// </summary>
+        public virtual string Sql { get; }
+
+        /// <summary>
+        ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+        ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+        ///     any release. You should only use it directly in your code with extreme caution and knowing that
+        ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+        /// </summary>
+        public virtual Expression Arguments { get; }
+    }
+}

src/EFCore.Relational/Query/Internal/FromSqlEntityQueryableInjectingExpressionVisitor.cs

@@ -0,0 +1,42 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Linq;
+using System.Linq.Expressions;
+using System.Reflection;
+using JetBrains.Annotations;
+
+namespace Microsoft.EntityFrameworkCore.Query.Internal
+{
+    public class FromSqlEntityQueryableInjectingExpressionVisitor : ExpressionVisitor
+    {
+        protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression)
+        {
+            if (methodCallExpression.Method.DeclaringType == typeof(RelationalQueryableExtensions)
+                && methodCallExpression.Method.Name == nameof(RelationalQueryableExtensions.FromSqlOnQueryable))
+            {
+                var sql = (string)((ConstantExpression)methodCallExpression.Arguments[1]).Value;
+                var queryable = (IQueryable)((ConstantExpression)methodCallExpression.Arguments[0]).Value;
+
+                return Expression.Constant(
+                    _createFromSqlEntityQueryableMethod
+                        .MakeGenericMethod(queryable.ElementType)
+                        .Invoke(null, new object[] { queryable.Provider, sql, methodCallExpression.Arguments[2] }));
+            }
+
+            return base.VisitMethodCall(methodCallExpression);
+        }
+
+        private static readonly MethodInfo _createFromSqlEntityQueryableMethod
+            = typeof(FromSqlEntityQueryableInjectingExpressionVisitor)
+                .GetTypeInfo().GetDeclaredMethod(nameof(_CreateFromSqlEntityQueryable));
+
+        [UsedImplicitly]
+        // ReSharper disable once InconsistentNaming
+        private static FromSqlEntityQueryable<TResult> _CreateFromSqlEntityQueryable<TResult>(
+            IAsyncQueryProvider entityQueryProvider,
+            string sql,
+            Expression arguments)
+            => new FromSqlEntityQueryable<TResult>(entityQueryProvider, sql, arguments);
+    }
+}

src/EFCore.Relational/Query/Internal/RelationalQueryOptimizerFactory.cs

@@ -0,0 +1,27 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Microsoft.EntityFrameworkCore.Query.Internal
+{
+    public class RelationalQueryOptimizerFactory : IQueryOptimizerFactory
+    {
+        private readonly QueryOptimizerDependencies _dependencies;
+        private readonly RelationalQueryOptimizerDependencies _relationalDependencies;
+
+        public RelationalQueryOptimizerFactory(
+            QueryOptimizerDependencies dependencies,
+            RelationalQueryOptimizerDependencies relationalDependencies)
+        {
+            _dependencies = dependencies;
+            _relationalDependencies = relationalDependencies;
+        }
+
+        public QueryOptimizer Create(QueryCompilationContext queryCompilationContext)
+        {
+            return new RelationalQueryOptimizer(
+                _dependencies,
+                _relationalDependencies,
+                queryCompilationContext);
+        }
+    }
+}

src/EFCore.Relational/Query/RelationalQueryOptimizer.cs

@@ -0,0 +1,29 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Linq.Expressions;
+using Microsoft.EntityFrameworkCore.Query.Internal;
+
+namespace Microsoft.EntityFrameworkCore.Query
+{
+    public class RelationalQueryOptimizer : QueryOptimizer
+    {
+        public RelationalQueryOptimizer(
+            QueryOptimizerDependencies dependencies,
+            RelationalQueryOptimizerDependencies relationalDependencies,
+            QueryCompilationContext queryCompilationContext)
+            : base(dependencies, queryCompilationContext)
+        {
+            RelationalDependencies = relationalDependencies;
+        }
+
+        protected virtual RelationalQueryOptimizerDependencies RelationalDependencies { get; }
+
+        public override Expression Visit(Expression query)
+        {
+            query = new FromSqlEntityQueryableInjectingExpressionVisitor().Visit(query);
+
+            return base.Visit(query);
+        }
+    }
+}

src/EFCore.Relational/Query/RelationalQueryOptimizerDependencies.cs

@@ -0,0 +1,58 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Microsoft.EntityFrameworkCore.Query
+{
+    /// <summary>
+    ///     <para>
+    ///         Service dependencies parameter class for <see cref="RelationalShapedQueryOptimizer" />
+    ///     </para>
+    ///     <para>
+    ///         This type is typically used by database providers (and other extensions). It is generally
+    ///         not used in application code.
+    ///     </para>
+    ///     <para>
+    ///         Do not construct instances of this class directly from either provider or application code as the
+    ///         constructor signature may change as new dependencies are added. Instead, use this type in
+    ///         your constructor so that an instance will be created and injected automatically by the
+    ///         dependency injection container. To create an instance with some dependent services replaced,
+    ///         first resolve the object from the dependency injection container, then replace selected
+    ///         services using the 'With...' methods. Do not call the constructor at any point in this process.
+    ///     </para>
+    ///     <para>
+    ///         The service lifetime is <see cref="ServiceLifetime.Scoped"/>. This means that each
+    ///         <see cref="DbContext"/> instance will use its own instance of this service.
+    ///         The implementation may depend on other services registered with any lifetime.
+    ///         The implementation does not need to be thread-safe.
+    ///     </para>
+    /// </summary>
+    public sealed class RelationalQueryOptimizerDependencies
+    {
+        /// <summary>
+        ///     <para>
+        ///         Creates the service dependencies parameter object for a <see cref="RelationalQueryOptimizer" />.
+        ///     </para>
+        ///     <para>
+        ///         Do not call this constructor directly from either provider or application code as it may change
+        ///         as new dependencies are added. Instead, use this type in your constructor so that an instance
+        ///         will be created and injected automatically by the dependency injection container. To create
+        ///         an instance with some dependent services replaced, first resolve the object from the dependency
+        ///         injection container, then replace selected services using the 'With...' methods. Do not call
+        ///         the constructor at any point in this process.
+        ///     </para>
+        ///     <para>
+        ///         This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+        ///         the same compatibility standards as public APIs. It may be changed or removed without notice in
+        ///         any release. You should only use it directly in your code with extreme caution and knowing that
+        ///         doing so can result in application failures when updating to a new Entity Framework Core release.
+        ///     </para>
+        /// </summary>
+        [EntityFrameworkInternal]
+        public RelationalQueryOptimizerDependencies()
+        {
+        }
+    }
+}

src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs

@@ -61,17 +61,23 @@ private RelationalQueryableMethodTranslatingExpressionVisitor(
             _sqlExpressionFactory = sqlExpressionFactory;
         }
 
-        protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression)
+        protected override Expression VisitConstant(ConstantExpression constantExpression)
+            => constantExpression.Type.IsGenericType
+            && constantExpression.Type.GetGenericTypeDefinition() == typeof(FromSqlEntityQueryable<>)
+                ? (Expression)_createFromSqlShapedQueryExpressionMethodInfo
+                    .MakeGenericMethod(constantExpression.Type.GetGenericArguments()[0])
+                    .Invoke(this, new object[] { constantExpression })
+                : base.VisitConstant(constantExpression);
+
+        private static readonly MethodInfo _createFromSqlShapedQueryExpressionMethodInfo
+            = typeof(RelationalQueryableMethodTranslatingExpressionVisitor)
+                .GetTypeInfo().GetDeclaredMethod(nameof(CreateFromSqlShapedQueryExpression));
+
+        private Expression CreateFromSqlShapedQueryExpression<TResult>(ConstantExpression constantExpression)
         {
-            if (methodCallExpression.Method.DeclaringType == typeof(RelationalQueryableExtensions)
-                && methodCallExpression.Method.Name == nameof(RelationalQueryableExtensions.FromSqlOnQueryable))
-            {
-                var sql = (string)((ConstantExpression)methodCallExpression.Arguments[1]).Value;
-                var queryable = (IQueryable)((ConstantExpression)methodCallExpression.Arguments[0]).Value;
-                return CreateShapedQueryExpression(queryable.ElementType, sql, methodCallExpression.Arguments[2]);
-            }
-
-            return base.VisitMethodCall(methodCallExpression);
+            var fromSqlEntityQueryable = (FromSqlEntityQueryable<TResult>)constantExpression.Value;
+            return CreateShapedQueryExpression(
+                fromSqlEntityQueryable.ElementType, fromSqlEntityQueryable.Sql, fromSqlEntityQueryable.Arguments);
         }
 
         public override ShapedQueryExpression TranslateSubquery(Expression expression)

src/EFCore.Relational/Query/RelationalShapedQueryOptimizerDependencies.cs

@@ -56,8 +56,9 @@ public sealed class RelationalShapedQueryOptimizerDependencies
         public RelationalShapedQueryOptimizerDependencies(
             [NotNull] ISqlExpressionFactory sqlExpressionFactory)
         {
-            SqlExpressionFactory = sqlExpressionFactory;
             Check.NotNull(sqlExpressionFactory, nameof(sqlExpressionFactory));
+
+            SqlExpressionFactory = sqlExpressionFactory;
         }
 
         /// <summary>

src/EFCore/Extensions/Internal/ExpressionExtensions.cs

@@ -142,8 +142,24 @@ public static bool IsLogicalOperation([NotNull] this Expression expression)
         ///     doing so can result in application failures when updating to a new Entity Framework Core release.
         /// </summary>
         public static bool IsEntityQueryable([NotNull] this ConstantExpression constantExpression)
-            => constantExpression.Type.GetTypeInfo().IsGenericType
-               && constantExpression.Type.GetGenericTypeDefinition() == typeof(EntityQueryable<>);
+        {
+            if (constantExpression.Type.GetTypeInfo().IsGenericType)
+            {
+                if (constantExpression.Type.GetTypeInfo().GetGenericTypeDefinition() == typeof(EntityQueryable<>))
+                {
+                    return true;
+                }
+
+                var genericArguments = constantExpression.Type.GetTypeInfo().GetGenericArguments();
+                if (genericArguments.Length == 1)
+                {
+                    return typeof(EntityQueryable<>).MakeGenericType(genericArguments[0])
+                        .GetTypeInfo().IsAssignableFrom(constantExpression.Type);
+                }
+            }
+
+            return false;
+        }
 
         /// <summary>
         ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to

src/EFCore/Query/Internal/EntityQueryable`.cs

@@ -74,7 +74,7 @@ public EntityQueryable([NotNull] IAsyncQueryProvider queryProvider, [NotNull] Ex
         ///     any release. You should only use it directly in your code with extreme caution and knowing that
         ///     doing so can result in application failures when updating to a new Entity Framework Core release.
         /// </summary>
-        public virtual Expression Expression { get; }
+        public virtual Expression Expression { get; protected set; }
 
         /// <summary>
         ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to