Support primitive collections

Closes dotnet#29427
Closes dotnet#30426
Closes dotnet#13617

All.sln.DotSettings

@@ -308,9 +308,12 @@ The .NET Foundation licenses this file to you under the MIT license.
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=navigations/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=niladic/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=NOCASE/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=OPENJSON/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=pluralizer/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Poolable/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Postgre/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=pushdown/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=queryables/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=remapper/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=requiredness/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=retriable/@EntryIndexedValue">True</s:Boolean>

src/EFCore.Relational/Properties/RelationalStrings.resx

@@ -175,6 +175,9 @@
   <data name="ConflictingRowValuesSensitive" xml:space="preserve">
     <value>Instances of entity types '{firstEntityType}' and '{secondEntityType}' are mapped to the same row with the key value '{keyValue}', but have different property values '{firstConflictingValue}' and '{secondConflictingValue}' for the column '{column}'.</value>
   </data>
+  <data name="ConflictingTypeMappingsForPrimitiveCollection" xml:space="preserve">
+    <value>Store type '{storeType1}' was inferred for a primitive collection, but that primitive collection was previously inferred to have store type '{storeType2}'.</value>
+  </data>
   <data name="ConflictingSeedValues" xml:space="preserve">
     <value>A seed entity for entity type '{entityType}' has the same key value as another seed entity mapped to the same table '{table}', but have different values for the column '{column}'. Consider using 'DbContextOptionsBuilder.EnableSensitiveDataLogging' to see the conflicting values.</value>
   </data>
@@ -346,6 +349,9 @@
   <data name="EitherOfTwoValuesMustBeNull" xml:space="preserve">
     <value>Either {param1} or {param2} must be null.</value>
   </data>
+  <data name="EmptyCollectionNotSupportedAsConstantQueryRoot" xml:space="preserve">
+    <value>Empty constant collections are not supported as constant query roots.</value>
+  </data>
   <data name="EntityShortNameNotUnique" xml:space="preserve">
     <value>The short name for '{entityType1}' is '{discriminatorValue}' which is the same for '{entityType2}'. Every concrete entity type in the hierarchy must have a unique short name. Either rename one of the types or call modelBuilder.Entity&lt;TEntity&gt;().Metadata.SetDiscriminatorValue("NewShortName").</value>
   </data>

src/EFCore.Relational/Query/Internal/EqualsTranslator.cs

@@ -60,8 +60,10 @@ public EqualsTranslator(ISqlExpressionFactory sqlExpressionFactory)
             && right != null)
         {
             if (left.Type == right.Type
-                || (right.Type == typeof(object) && (right is SqlParameterExpression || right is SqlConstantExpression))
-                || (left.Type == typeof(object) && (left is SqlParameterExpression || left is SqlConstantExpression)))
+                || (right.Type == typeof(object)
+                    && right is SqlParameterExpression or SqlConstantExpression or ColumnExpression { TypeMapping: null })
+                || (left.Type == typeof(object)
+                    && left is SqlParameterExpression or SqlConstantExpression or ColumnExpression { TypeMapping: null }))
             {
                 return _sqlExpressionFactory.Equal(left, right);
             }

src/EFCore.Relational/Query/QuerySqlGenerator.cs

@@ -226,12 +226,7 @@ protected override Expression VisitSelect(SelectExpression selectExpression)
             subQueryIndent = _relationalCommandBuilder.Indent();
         }
 
-        if (IsNonComposedSetOperation(selectExpression))
-        {
-            // Naked set operation
-            GenerateSetOperation((SetOperationBase)selectExpression.Tables[0]);
-        }
-        else
+        if (!TryGenerateWithoutWrappingSelect(selectExpression))
         {
             _relationalCommandBuilder.Append("SELECT ");
 
@@ -300,6 +295,43 @@ protected override Expression VisitSelect(SelectExpression selectExpression)
         return selectExpression;
     }
 
+    /// <summary>
+    ///     If possible, generates the expression contained within the provided <paramref name="selectExpression" /> without the wrapping
+    ///     SELECT. This can be done for set operations and VALUES, which can appear as top-level statements without needing to be wrapped
+    ///     in SELECT.
+    /// </summary>
+    protected virtual bool TryGenerateWithoutWrappingSelect(SelectExpression selectExpression)
+    {
+        if (IsNonComposedSetOperation(selectExpression))
+        {
+            GenerateSetOperation((SetOperationBase)selectExpression.Tables[0]);
+            return true;
+        }
+
+        if (selectExpression is
+            {
+                Tables: [ValuesExpression valuesExpression],
+                Offset: null,
+                Limit: null,
+                IsDistinct: false,
+                Predicate: null,
+                Having: null,
+                Orderings.Count: 0,
+                GroupBy.Count: 0,
+            }
+            && selectExpression.Projection.Count == valuesExpression.ColumnNames.Count
+            && selectExpression.Projection.Select(
+                    (pe, index) => pe.Expression is ColumnExpression column
+                        && column.Name == valuesExpression.ColumnNames[index])
+                .All(e => e))
+        {
+            GenerateValues(valuesExpression);
+            return true;
+        }
+
+        return false;
+    }
+
     /// <summary>
     ///     Generates a pseudo FROM clause. Required by some providers when a query has no actual FROM clause.
     /// </summary>
@@ -371,16 +403,16 @@ protected override Expression VisitSqlFunction(SqlFunctionExpression sqlFunction
     /// <inheritdoc />
     protected override Expression VisitTableValuedFunction(TableValuedFunctionExpression tableValuedFunctionExpression)
     {
-        if (!string.IsNullOrEmpty(tableValuedFunctionExpression.StoreFunction.Schema))
+        if (!string.IsNullOrEmpty(tableValuedFunctionExpression.Schema))
         {
             _relationalCommandBuilder
-                .Append(_sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.StoreFunction.Schema))
+                .Append(_sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.Schema))
                 .Append(".");
         }
 
-        var name = tableValuedFunctionExpression.StoreFunction.IsBuiltIn
-            ? tableValuedFunctionExpression.StoreFunction.Name
-            : _sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.StoreFunction.Name);
+        var name = tableValuedFunctionExpression.IsBuiltIn
+            ? tableValuedFunctionExpression.Name
+            : _sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.Name);
 
         _relationalCommandBuilder
             .Append(name)
@@ -607,6 +639,7 @@ protected override Expression VisitSqlParameter(SqlParameterExpression sqlParame
     {
         var invariantName = sqlParameterExpression.Name;
         var parameterName = sqlParameterExpression.Name;
+        var typeMapping = sqlParameterExpression.TypeMapping!;
 
         // Try to see if a parameter already exists - if so, just integrate the same placeholder into the SQL instead of sending the same
         // data twice.
@@ -615,11 +648,10 @@ protected override Expression VisitSqlParameter(SqlParameterExpression sqlParame
         var parameter = _relationalCommandBuilder.Parameters.FirstOrDefault(
             p =>
                 p.InvariantName == parameterName
-                && p is TypeMappedRelationalParameter typeMappedRelationalParameter
-                && string.Equals(
-                    typeMappedRelationalParameter.RelationalTypeMapping.StoreType, sqlParameterExpression.TypeMapping!.StoreType,
-                    StringComparison.OrdinalIgnoreCase)
-                && typeMappedRelationalParameter.RelationalTypeMapping.Converter == sqlParameterExpression.TypeMapping!.Converter);
+                && p is TypeMappedRelationalParameter { RelationalTypeMapping: var existingTypeMapping }
+                && string.Equals(existingTypeMapping.StoreType, typeMapping.StoreType, StringComparison.OrdinalIgnoreCase)
+                && (existingTypeMapping.Converter is null && typeMapping.Converter is null
+                    || existingTypeMapping.Converter is not null && existingTypeMapping.Converter.Equals(typeMapping.Converter)));
 
         if (parameter is null)
         {
@@ -1132,6 +1164,28 @@ protected override Expression VisitRowNumber(RowNumberExpression rowNumberExpres
         return rowNumberExpression;
     }
 
+    /// <inheritdoc />
+    protected override Expression VisitRowValue(RowValueExpression rowValueExpression)
+    {
+        Sql.Append("(");
+
+        var values = rowValueExpression.Values;
+        var count = values.Count;
+        for (var i = 0; i < count; i++)
+        {
+            if (i > 0)
+            {
+                Sql.Append(", ");
+            }
+
+            Visit(values[i]);
+        }
+
+        Sql.Append(")");
+
+        return rowValueExpression;
+    }
+
     /// <summary>
     ///     Generates a set operation in the relational command.
     /// </summary>
@@ -1311,6 +1365,66 @@ void LiftPredicate(TableExpressionBase joinTable)
             RelationalStrings.ExecuteOperationWithUnsupportedOperatorInSqlGeneration(nameof(RelationalQueryableExtensions.ExecuteUpdate)));
     }
 
+    /// <inheritdoc />
+    protected override Expression VisitValues(ValuesExpression valuesExpression)
+    {
+        _relationalCommandBuilder.Append("(");
+
+        GenerateValues(valuesExpression);
+
+        _relationalCommandBuilder
+            .Append(")")
+            .Append(AliasSeparator)
+            .Append(_sqlGenerationHelper.DelimitIdentifier(valuesExpression.Alias));
+
+        return valuesExpression;
+    }
+
+    /// <summary>
+    ///     Generates a VALUES expression.
+    /// </summary>
+    protected virtual void GenerateValues(ValuesExpression valuesExpression)
+    {
+        var rowValues = valuesExpression.RowValues;
+
+        // Some databases support providing the names of columns projected out of VALUES, e.g.
+        // SQL Server/PG: (VALUES (1, 3), (2, 4)) AS x(a, b). Others unfortunately don't; so by default, we extract out the first row,
+        // and generate a SELECT for it with the names, and a UNION ALL over the rest of the values.
+        _relationalCommandBuilder.Append("SELECT ");
+
+        Check.DebugAssert(rowValues.Count > 0, "rowValues.Count > 0");
+        var firstRowValues = rowValues[0].Values;
+        for (var i = 0; i < firstRowValues.Count; i++)
+        {
+            if (i > 0)
+            {
+                _relationalCommandBuilder.Append(", ");
+            }
+
+            Visit(firstRowValues[i]);
+
+            _relationalCommandBuilder
+                .Append(AliasSeparator)
+                .Append(_sqlGenerationHelper.DelimitIdentifier(valuesExpression.ColumnNames[i]));
+        }
+
+        if (rowValues.Count > 1)
+        {
+            _relationalCommandBuilder.Append(" UNION ALL VALUES ");
+
+            for (var i = 1; i < rowValues.Count; i++)
+            {
+                // TODO: Do we want newlines here?
+                if (i > 1)
+                {
+                    _relationalCommandBuilder.Append(", ");
+                }
+
+                Visit(valuesExpression.RowValues[i]);
+            }
+        }
+    }
+
     /// <inheritdoc />
     protected override Expression VisitJsonScalar(JsonScalarExpression jsonScalarExpression)
         => throw new InvalidOperationException(

src/EFCore.Relational/Query/RelationalQueryRootProcessor.cs

@@ -0,0 +1,63 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.EntityFrameworkCore.Query.Internal;
+using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
+
+namespace Microsoft.EntityFrameworkCore.Query;
+
+/// <inheritdoc />
+public class RelationalQueryRootProcessor : QueryRootProcessor
+{
+    private readonly IModel _model;
+
+    /// <summary>
+    ///     Creates a new instance of the <see cref="RelationalQueryRootProcessor" /> class.
+    /// </summary>
+    /// <param name="dependencies">Parameter object containing dependencies for this class.</param>
+    /// <param name="relationalDependencies">Parameter object containing relational dependencies for this class.</param>
+    /// <param name="queryCompilationContext">The query compilation context object to use.</param>
+    public RelationalQueryRootProcessor(
+        QueryTranslationPreprocessorDependencies dependencies,
+        RelationalQueryTranslationPreprocessorDependencies relationalDependencies,
+        QueryCompilationContext queryCompilationContext)
+        : base(dependencies, queryCompilationContext)
+    {
+        _model = queryCompilationContext.Model;
+    }
+
+    /// <summary>
+    ///     Converts a <see cref="ConstantExpression" /> to a <see cref="ConstantQueryRootExpression" />, which will later be translated
+    ///     to a <see cref="ValuesExpression" />.
+    /// </summary>
+    /// <param name="constantExpression">The constant expression to convert to a query root.</param>
+    protected override Expression VisitQueryableConstant(ConstantExpression constantExpression)
+        => new ConstantQueryRootExpression(constantExpression);
+
+    /// <inheritdoc />
+    protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression)
+    {
+        // Create query root node for table-valued functions
+        if (_model.FindDbFunction(methodCallExpression.Method) is { IsScalar: false, StoreFunction: var storeFunction })
+        {
+            // See issue #19970
+            return new TableValuedFunctionQueryRootExpression(
+                storeFunction.EntityTypeMappings.Single().EntityType,
+                storeFunction,
+                methodCallExpression.Arguments);
+        }
+
+        return base.VisitMethodCall(methodCallExpression);
+    }
+
+    /// <inheritdoc />
+    protected override Expression VisitExtension(Expression node)
+        => node switch
+        {
+            // We skip FromSqlQueryRootExpression, since that contains the arguments as an object array parameter, and don't want to convert
+            // that to a query root
+            FromSqlQueryRootExpression e => e,
+
+            _ => base.VisitExtension(node)
+        };
+}

src/EFCore.Relational/Query/RelationalQueryTranslationPreprocessor.cs

@@ -36,9 +36,12 @@ public override Expression NormalizeQueryableMethod(Expression expression)
     {
         expression = new RelationalQueryMetadataExtractingExpressionVisitor(_relationalQueryCompilationContext).Visit(expression);
         expression = base.NormalizeQueryableMethod(expression);
-        expression = new TableValuedFunctionToQueryRootConvertingExpressionVisitor(QueryCompilationContext.Model).Visit(expression);
         expression = new CollectionIndexerToElementAtNormalizingExpressionVisitor().Visit(expression);
 
         return expression;
     }
+
+    /// <inheritdoc />
+    protected override Expression ProcessQueryRoots(Expression expression)
+        => new RelationalQueryRootProcessor(Dependencies, RelationalDependencies, QueryCompilationContext).Visit(expression);
 }

src/EFCore.Relational/Query/RelationalQueryTranslationPreprocessorDependencies.cs

@@ -45,7 +45,13 @@ public sealed record RelationalQueryTranslationPreprocessorDependencies
     ///     the constructor at any point in this process.
     /// </remarks>
     [EntityFrameworkInternal]
-    public RelationalQueryTranslationPreprocessorDependencies()
+    public RelationalQueryTranslationPreprocessorDependencies(IRelationalTypeMappingSource relationalTypeMappingSource)
     {
+        RelationalTypeMappingSource = relationalTypeMappingSource;
     }
+
+    /// <summary>
+    ///     The type mapping source.
+    /// </summary>
+    public IRelationalTypeMappingSource RelationalTypeMappingSource { get; init; }
 }