Support queryable array parameters and columns

Closes npgsql#2677
Closes npgsql#2726

src/EFCore.PG.NTS/Storage/Internal/NpgsqlGeometryTypeMapping.cs

@@ -7,14 +7,32 @@
 // ReSharper disable once CheckNamespace
 namespace Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal;
 
+/// <summary>
+///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+///     any release. You should only use it directly in your code with extreme caution and knowing that
+///     doing so can result in application failures when updating to a new Entity Framework Core release.
+/// </summary>
+public interface INpgsqlGeometryTypeMapping
+{
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    RelationalTypeMapping CloneWithElementTypeMapping(RelationalTypeMapping elementTypeMapping);
+}
+
 /// <summary>
 ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
 ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
 ///     any release. You should only use it directly in your code with extreme caution and knowing that
 ///     doing so can result in application failures when updating to a new Entity Framework Core release.
 /// </summary>
 [UsedImplicitly]
-public class NpgsqlGeometryTypeMapping<TGeometry> : RelationalGeometryTypeMapping<TGeometry, TGeometry>, INpgsqlTypeMapping
+public class NpgsqlGeometryTypeMapping<TGeometry> : RelationalGeometryTypeMapping<TGeometry, TGeometry>,
+    INpgsqlTypeMapping, INpgsqlGeometryTypeMapping
 {
     private readonly bool _isGeography;
 
@@ -49,6 +67,16 @@ protected NpgsqlGeometryTypeMapping(RelationalTypeMappingParameters parameters)
     protected override RelationalTypeMapping Clone(RelationalTypeMappingParameters parameters)
         => new NpgsqlGeometryTypeMapping<TGeometry>(parameters);
 
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    RelationalTypeMapping INpgsqlGeometryTypeMapping.CloneWithElementTypeMapping(RelationalTypeMapping elementTypeMapping)
+        => new NpgsqlGeometryTypeMapping<TGeometry>(
+            Parameters.WithCoreParameters(Parameters.CoreParameters.WithElementTypeMapping(elementTypeMapping)));
+
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
     ///     the same compatibility standards as public APIs. It may be changed or removed without notice in

src/EFCore.PG.NTS/Storage/Internal/NpgsqlNetTopologySuiteTypeMappingSourcePlugin.cs

@@ -57,9 +57,15 @@ public NpgsqlNetTopologySuiteTypeMappingSourcePlugin(INpgsqlNetTopologySuiteOpti
         var storeTypeName = mappingInfo.StoreTypeName;
         var isGeography = _options.IsGeographyDefault;
 
-        if (clrType is not null && !typeof(Geometry).IsAssignableFrom(clrType))
+        if (clrType is not null)
         {
-            return null;
+            if (!clrType.IsAssignableTo(typeof(Geometry)))
+            {
+                return null;
+            }
+
+            // TODO: if store type is null, consider setting it based on the CLR type, i.e. create GEOMETRY(Point) instead of Geometry when
+            // the CLR property is NTS Point.
         }
 
         if (storeTypeName is not null)
@@ -69,18 +75,34 @@ public NpgsqlNetTopologySuiteTypeMappingSourcePlugin(INpgsqlNetTopologySuiteOpti
                 return null;
             }
 
-            if (clrType is null)
-            {
-                clrType = parsedSubtype;
-            }
+            clrType ??= parsedSubtype;
+        }
+
+        storeTypeName ??= isGeography ? "geography" : "geometry";
+
+        Check.DebugAssert(clrType is not null, "clrType is not null");
+
+        var typeMapping = (RelationalTypeMapping)Activator.CreateInstance(
+            typeof(NpgsqlGeometryTypeMapping<>).MakeGenericType(clrType), storeTypeName, isGeography)!;
+
+        // TODO: Also restrict the element type mapping based on the user-specified store type?
+        var elementType = clrType == typeof(MultiPoint)
+            ? typeof(Point)
+            : clrType == typeof(MultiLineString)
+                ? typeof(LineString)
+                : clrType == typeof(MultiPolygon)
+                    ? typeof(Polygon)
+                    : clrType == typeof(GeometryCollection)
+                        ? typeof(Geometry)
+                        : null;
+
+        if (elementType is not null)
+        {
+            var elementTypeMapping = FindMapping(new() { ClrType = elementType })!;
+            typeMapping = ((INpgsqlGeometryTypeMapping)typeMapping).CloneWithElementTypeMapping(elementTypeMapping);
         }
 
-        return clrType is not null || storeTypeName is not null
-            ? (RelationalTypeMapping)Activator.CreateInstance(
-                typeof(NpgsqlGeometryTypeMapping<>).MakeGenericType(clrType ?? typeof(Geometry)),
-                storeTypeName ?? (isGeography ? "geography" : "geometry"),
-                isGeography)!
-            : null;
+        return typeMapping;
     }
 
     /// <summary>
@@ -98,7 +120,7 @@ public static bool TryParseStoreTypeName(
         storeTypeName = storeTypeName.Trim();
         subtypeName = storeTypeName;
         isGeography = false;
-        clrType = null;
+        clrType = typeof(Geometry);
         srid = -1;
         ordinates = Ordinates.AllOrdinates;
 

src/EFCore.PG.NodaTime/Storage/Internal/NpgsqlNodaTimeTypeMappingSourcePlugin.cs

@@ -350,14 +350,15 @@ public NpgsqlNodaTimeTypeMappingSourcePlugin(ISqlGenerationHelper sqlGenerationH
 
             // If no mapping was found for the element, there's no mapping for the array.
             // Also, arrays of arrays aren't supported (as opposed to multidimensional arrays) by PostgreSQL
-            if (elementMapping is null || elementMapping is NpgsqlArrayTypeMapping)
+            if (elementMapping is null or NpgsqlArrayTypeMapping)
             {
                 return null;
             }
 
-            return new NpgsqlArrayArrayTypeMapping(storeType, elementMapping);
+            return new NpgsqlArrayTypeMapping(storeType, clrType ?? elementMapping.ClrType.MakeArrayType(), elementMapping);
         }
 
+        // TODO: Clean this up, should not be needed
         if (clrType is null)
         {
             return null;
@@ -382,7 +383,7 @@ public NpgsqlNodaTimeTypeMappingSourcePlugin(ISqlGenerationHelper sqlGenerationH
                 return null;
             }
 
-            return new NpgsqlArrayArrayTypeMapping(clrType, elementMapping);
+            return new NpgsqlArrayTypeMapping(clrType, elementMapping);
         }
 
         if (clrType.IsGenericList())
@@ -402,7 +403,7 @@ public NpgsqlNodaTimeTypeMappingSourcePlugin(ISqlGenerationHelper sqlGenerationH
                 return null;
             }
 
-            return new NpgsqlArrayListTypeMapping(clrType, elementMapping);
+            return new NpgsqlArrayTypeMapping(clrType, elementMapping);
         }
 
         return null;

src/EFCore.PG/Extensions/NpgsqlServiceCollectionExtensions.cs

@@ -1,4 +1,3 @@
-using System.Data.Common;
 using Npgsql.EntityFrameworkCore.PostgreSQL.Diagnostics.Internal;
 using Npgsql.EntityFrameworkCore.PostgreSQL.Infrastructure;
 using Npgsql.EntityFrameworkCore.PostgreSQL.Infrastructure.Internal;
@@ -114,6 +113,7 @@ public static IServiceCollection AddEntityFrameworkNpgsql(this IServiceCollectio
             .TryAdd<IEvaluatableExpressionFilter, NpgsqlEvaluatableExpressionFilter>()
             .TryAdd<IQuerySqlGeneratorFactory, NpgsqlQuerySqlGeneratorFactory>()
             .TryAdd<IRelationalSqlTranslatingExpressionVisitorFactory, NpgsqlSqlTranslatingExpressionVisitorFactory>()
+            .TryAdd<IQueryTranslationPreprocessorFactory, NpgsqlQueryTranslationPreprocessorFactory>()
             .TryAdd<IQueryTranslationPostprocessorFactory, NpgsqlQueryTranslationPostprocessorFactory>()
             .TryAdd<IRelationalParameterBasedSqlProcessorFactory, NpgsqlParameterBasedSqlProcessorFactory>()
             .TryAdd<ISqlExpressionFactory, NpgsqlSqlExpressionFactory>()
@@ -130,4 +130,4 @@ public static IServiceCollection AddEntityFrameworkNpgsql(this IServiceCollectio
 
         return serviceCollection;
     }
-}
+}

src/EFCore.PG/Infrastructure/Internal/INpgsqlSingletonOptions.cs

@@ -13,9 +13,9 @@ public interface INpgsqlSingletonOptions : ISingletonOptions
     Version PostgresVersion { get; }
 
     /// <summary>
-    ///     The backend version to target, but returns <see langword="null" /> unless the user explicitly specified a version.
+    ///     Whether the user has explicitly set the backend version to target.
     /// </summary>
-    Version? PostgresVersionWithoutDefault { get; }
+    bool IsPostgresVersionSet { get; }
 
     /// <summary>
     ///     Whether to target Redshift.
@@ -41,4 +41,4 @@ public interface INpgsqlSingletonOptions : ISingletonOptions
     ///     The root service provider for the application, if available. />.
     /// </summary>
     IServiceProvider? ApplicationServiceProvider { get; }
-}
+}

src/EFCore.PG/Infrastructure/Internal/NpgsqlOptionsExtension.cs

@@ -32,8 +32,8 @@ public virtual Version PostgresVersion
     /// <summary>
     /// The backend version to target, but returns <see langword="null" /> unless the user explicitly specified a version.
     /// </summary>
-    public virtual Version? PostgresVersionWithoutDefault
-        => _postgresVersion;
+    public virtual bool IsPostgresVersionSet
+        => _postgresVersion is not null;
 
     /// <summary>
     ///     The <see cref="DbDataSource" />, or <see langword="null" /> if a connection string or <see cref="DbConnection" /> was used

src/EFCore.PG/Internal/NpgsqlSingletonOptions.cs

@@ -21,7 +21,7 @@ public class NpgsqlSingletonOptions : INpgsqlSingletonOptions
     ///     any release. You should only use it directly in your code with extreme caution and knowing that
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
-    public virtual Version? PostgresVersionWithoutDefault { get; private set; }
+    public virtual bool IsPostgresVersionSet { get; private set; }
 
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
@@ -79,7 +79,7 @@ public virtual void Initialize(IDbContextOptions options)
         var coreOptions = options.FindExtension<CoreOptionsExtension>() ?? new();
 
         PostgresVersion = npgsqlOptions.PostgresVersion;
-        PostgresVersionWithoutDefault = npgsqlOptions.PostgresVersionWithoutDefault;
+        IsPostgresVersionSet = npgsqlOptions.IsPostgresVersionSet;
         UseRedshift = npgsqlOptions.UseRedshift;
         ReverseNullOrderingEnabled = npgsqlOptions.ReverseNullOrdering;
         UserRangeDefinitions = npgsqlOptions.UserRangeDefinitions;

src/EFCore.PG/Query/ExpressionTranslators/Internal/NpgsqlArrayMethodTranslator.cs

@@ -0,0 +1,158 @@
+using Npgsql.EntityFrameworkCore.PostgreSQL.Internal;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal.Mapping;
+using static Npgsql.EntityFrameworkCore.PostgreSQL.Utilities.Statics;
+
+namespace Npgsql.EntityFrameworkCore.PostgreSQL.Query.ExpressionTranslators.Internal;
+
+/// <summary>
+/// Translates method and property calls on arrays/lists into their corresponding PostgreSQL operations.
+/// </summary>
+/// <remarks>
+/// https://www.postgresql.org/docs/current/static/functions-array.html
+/// </remarks>
+public class NpgsqlArrayMethodTranslator : IMethodCallTranslator
+{
+    #region Methods
+
+    // ReSharper disable InconsistentNaming
+    private static readonly MethodInfo Array_IndexOf1 =
+        typeof(Array).GetTypeInfo().GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly)
+            .Single(m => m.Name == nameof(Array.IndexOf) && m.IsGenericMethod && m.GetParameters().Length == 2);
+
+    private static readonly MethodInfo Array_IndexOf2 =
+        typeof(Array).GetTypeInfo().GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly)
+            .Single(m => m is { Name: nameof(Array.IndexOf), IsGenericMethod: true } && m.GetParameters().Length == 3);
+
+    private static readonly MethodInfo Enumerable_ElementAt =
+        typeof(Enumerable).GetTypeInfo().GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly)
+            .Single(m => m.Name == nameof(Enumerable.ElementAt) && m.GetParameters().Length == 2 && m.GetParameters()[1].ParameterType == typeof(int));
+
+    private static readonly MethodInfo Enumerable_SequenceEqual =
+        typeof(Enumerable).GetTypeInfo().GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.DeclaredOnly)
+            .Single(m => m.Name == nameof(Enumerable.SequenceEqual) && m.GetParameters().Length == 2);
+    // ReSharper restore InconsistentNaming
+
+    #endregion Methods
+
+    private readonly NpgsqlSqlExpressionFactory _sqlExpressionFactory;
+    private readonly NpgsqlJsonPocoTranslator _jsonPocoTranslator;
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public NpgsqlArrayMethodTranslator(NpgsqlSqlExpressionFactory sqlExpressionFactory, NpgsqlJsonPocoTranslator jsonPocoTranslator)
+    {
+        _sqlExpressionFactory = sqlExpressionFactory;
+        _jsonPocoTranslator = jsonPocoTranslator;
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public virtual SqlExpression? Translate(
+        SqlExpression? instance,
+        MethodInfo method,
+        IReadOnlyList<SqlExpression> arguments,
+        IDiagnosticsLogger<DbLoggerCategory.Query> logger)
+    {
+        // During preprocessing, ArrayIndex and List[] get normalized to ElementAt; so we handle indexing into array/list here
+        if (method.IsClosedFormOf(Enumerable_ElementAt))
+        {
+            // Indexing over bytea is special, we have to use function rather than subscript
+            if (arguments[0].TypeMapping is NpgsqlByteArrayTypeMapping)
+            {
+                return _sqlExpressionFactory.Function(
+                    "get_byte",
+                    new[] { arguments[0], arguments[1] },
+                    nullable: true,
+                    argumentsPropagateNullability: TrueArrays[2],
+                    typeof(byte));
+            }
+
+            // Try translating indexing inside JSON column
+            // Note that Length over PG arrays (not within JSON) gets translated by QueryableMethodTranslatingEV, since arrays are primitive
+            // collections
+            return _jsonPocoTranslator.TranslateMemberAccess(arguments[0], arguments[1], method.ReturnType);
+        }
+
+        if (method.IsClosedFormOf(Enumerable_SequenceEqual)
+            && arguments[0].Type.IsArrayOrGenericList() && !IsMappedToNonArray(arguments[0])
+            && arguments[1].Type.IsArrayOrGenericList() && !IsMappedToNonArray(arguments[1]))
+        {
+            return _sqlExpressionFactory.Equal(arguments[0], arguments[1]);
+        }
+
+        // Translate instance methods on List
+        if (instance is not null && instance.Type.IsGenericList() && !IsMappedToNonArray(instance))
+        {
+            return TranslateCommon(instance, arguments);
+        }
+
+        // Translate extension methods over array or List
+        if (instance is null && arguments.Count > 0 && arguments[0].Type.IsArrayOrGenericList() && !IsMappedToNonArray(arguments[0]))
+        {
+            return TranslateCommon(arguments[0], arguments.Slice(1));
+        }
+
+        return null;
+
+        // The array/list CLR type may be mapped to a non-array database type (e.g. byte[] to bytea, or just
+        // value converters) - we don't want to translate for those cases.
+        static bool IsMappedToNonArray(SqlExpression arrayOrList)
+            => arrayOrList.TypeMapping is { } and not (NpgsqlArrayTypeMapping or NpgsqlJsonTypeMapping);
+
+#pragma warning disable CS8321
+        SqlExpression? TranslateCommon(SqlExpression arrayOrList, IReadOnlyList<SqlExpression> arguments)
+#pragma warning restore CS8321
+        {
+            if (method.IsClosedFormOf(Array_IndexOf1)
+                ||
+                method.Name == nameof(List<int>.IndexOf)
+                && method.DeclaringType.IsGenericList()
+                && method.GetParameters().Length == 1)
+            {
+                var (item, array) = _sqlExpressionFactory.ApplyTypeMappingsOnItemAndArray(arguments[0], arrayOrList);
+
+                return _sqlExpressionFactory.Coalesce(
+                    _sqlExpressionFactory.Subtract(
+                        _sqlExpressionFactory.Function(
+                            "array_position",
+                            new[] { array, item },
+                            nullable: true,
+                            TrueArrays[2],
+                            arrayOrList.Type),
+                        _sqlExpressionFactory.Constant(1)),
+                    _sqlExpressionFactory.Constant(-1));
+            }
+
+            if (method.IsClosedFormOf(Array_IndexOf2)
+                ||
+                method.Name == nameof(List<int>.IndexOf)
+                && method.DeclaringType.IsGenericList()
+                && method.GetParameters().Length == 2)
+            {
+                var (item, array) = _sqlExpressionFactory.ApplyTypeMappingsOnItemAndArray(arguments[0], arrayOrList);
+                var startIndex = _sqlExpressionFactory.GenerateOneBasedIndexExpression(arguments[1]);
+
+                return _sqlExpressionFactory.Coalesce(
+                    _sqlExpressionFactory.Subtract(
+                        _sqlExpressionFactory.Function(
+                            "array_position",
+                            new[] { array, item, startIndex },
+                            nullable: true,
+                            TrueArrays[3],
+                            arrayOrList.Type),
+                        _sqlExpressionFactory.Constant(1)),
+                    _sqlExpressionFactory.Constant(-1));
+            }
+
+            return null;
+        }
+    }
+}