Cosmos querying over arrays of structural type (#33998)

Closes #16926
Closes #20441
Closes #29497
Closes #25700
Closes #25701
Closes #33032

src/EFCore.Cosmos/Query/Internal/CosmosProjectionBindingExpressionVisitor.cs

@@ -60,6 +60,7 @@ public virtual Expression Translate(SelectExpression selectExpression, Expressio
         _projectionMembers.Push(new ProjectionMember());
 
         var result = Visit(expression);
+
         if (result == QueryCompilationContext.NotTranslatedExpression)
         {
             _clientEval = true;
@@ -329,7 +330,7 @@ UnaryExpression unaryExpression
                     Expression.Convert(Expression.Convert(entityProjection, typeof(object)), typeof(ValueBuffer)),
                     nullable: true);
 
-            case ObjectArrayProjectionExpression objectArrayProjectionExpression:
+            case ObjectArrayAccessExpression objectArrayProjectionExpression:
             {
                 var innerShaperExpression = new StructuralTypeShaperExpression(
                     navigation.TargetEntityType,
@@ -487,6 +488,9 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
 
             var innerEntityProjection = shaperExpression.ValueBufferExpression switch
             {
+                EntityProjectionExpression entityProjection
+                    => entityProjection,
+
                 ProjectionBindingExpression innerProjectionBindingExpression
                     => (EntityProjectionExpression)_selectExpression.Projection[innerProjectionBindingExpression.Index!.Value].Expression,
 
@@ -518,26 +522,27 @@ UnaryExpression unaryExpression
 
             switch (navigationProjection)
             {
-                case EntityProjectionExpression entityProjection:
-                    return new StructuralTypeShaperExpression(
-                        navigation.TargetEntityType,
-                        Expression.Convert(Expression.Convert(entityProjection, typeof(object)), typeof(ValueBuffer)),
-                        nullable: true);
+                case StructuralTypeShaperExpression shaper when navigation.IsCollection:
+                    var objectArrayAccessExpression = shaper.ValueBufferExpression as ObjectArrayAccessExpression;
+                    Check.DebugAssert(objectArrayAccessExpression is not null, "Expected ObjectArrayAccessExpression");
 
-                case ObjectArrayProjectionExpression objectArrayProjectionExpression:
-                {
                     var innerShaperExpression = new StructuralTypeShaperExpression(
                         navigation.TargetEntityType,
                         Expression.Convert(
-                            Expression.Convert(objectArrayProjectionExpression.InnerProjection, typeof(object)), typeof(ValueBuffer)),
+                            Expression.Convert(objectArrayAccessExpression.InnerProjection, typeof(object)), typeof(ValueBuffer)),
                         nullable: true);
 
                     return new CollectionShaperExpression(
-                        objectArrayProjectionExpression,
+                        objectArrayAccessExpression,
                         innerShaperExpression,
                         navigation,
                         innerShaperExpression.StructuralType.ClrType);
-                }
+
+                case StructuralTypeShaperExpression shaper:
+                    return new StructuralTypeShaperExpression(
+                        shaper.StructuralType,
+                        Expression.Convert(Expression.Convert(shaper.ValueBufferExpression, typeof(object)), typeof(ValueBuffer)),
+                        shaper.IsNullable);
 
                 default:
                     throw new InvalidOperationException(CoreStrings.TranslationFailed(methodCallExpression.Print()));

src/EFCore.Cosmos/Query/Internal/CosmosQuerySqlGenerator.cs

@@ -49,7 +49,7 @@ public virtual CosmosSqlQuery GetSqlQuery(
     /// </summary>
     protected override Expression VisitEntityProjection(EntityProjectionExpression entityProjectionExpression)
     {
-        Visit(entityProjectionExpression.AccessExpression);
+        Visit(entityProjectionExpression.Object);
 
         return entityProjectionExpression;
     }
@@ -80,18 +80,52 @@ protected override Expression VisitExists(ExistsExpression existsExpression)
     ///     any release. You should only use it directly in your code with extreme caution and knowing that
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
-    protected override Expression VisitArray(ArrayExpression arrayExpression)
+    protected override Expression VisitObjectArray(ObjectArrayExpression objectArrayExpression)
+    {
+        GenerateArray(objectArrayExpression.Subquery);
+        return objectArrayExpression;
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    protected override Expression VisitScalarArray(ScalarArrayExpression scalarArrayExpression)
+    {
+        GenerateArray(scalarArrayExpression.Subquery);
+        return scalarArrayExpression;
+    }
+
+    private void GenerateArray(SelectExpression subquery)
     {
         _sqlBuilder.AppendLine("ARRAY(");
 
         using (_sqlBuilder.Indent())
         {
-            Visit(arrayExpression.Subquery);
+            Visit(subquery);
         }
 
         _sqlBuilder.Append(")");
+    }
 
-        return arrayExpression;
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    protected override Expression VisitObjectArrayAccess(ObjectArrayAccessExpression objectArrayAccessExpression)
+    {
+        Visit(objectArrayAccessExpression.Object);
+
+        _sqlBuilder
+            .Append("[\"")
+            .Append(objectArrayAccessExpression.PropertyName)
+            .Append("\"]");
+
+        return objectArrayAccessExpression;
     }
 
     /// <summary>
@@ -100,11 +134,14 @@ protected override Expression VisitArray(ArrayExpression arrayExpression)
     ///     any release. You should only use it directly in your code with extreme caution and knowing that
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
-    protected override Expression VisitObjectArrayProjection(ObjectArrayProjectionExpression objectArrayProjectionExpression)
+    protected override Expression VisitObjectArrayIndex(ObjectArrayIndexExpression objectArrayIndexExpression)
     {
-        _sqlBuilder.Append(objectArrayProjectionExpression.ToString());
+        Visit(objectArrayIndexExpression.Array);
+        _sqlBuilder.Append("[");
+        Visit(objectArrayIndexExpression.Index);
+        _sqlBuilder.Append("]");
 
-        return objectArrayProjectionExpression;
+        return objectArrayIndexExpression;
     }
 
     /// <summary>
@@ -113,11 +150,21 @@ protected override Expression VisitObjectArrayProjection(ObjectArrayProjectionEx
     ///     any release. You should only use it directly in your code with extreme caution and knowing that
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
-    protected override Expression VisitKeyAccess(KeyAccessExpression keyAccessExpression)
+    protected override Expression VisitScalarAccess(ScalarAccessExpression scalarAccessExpression)
     {
-        _sqlBuilder.Append(keyAccessExpression.ToString());
+        Visit(scalarAccessExpression.Object);
 
-        return keyAccessExpression;
+        // TODO: Remove check once __jObject is translated to the access root in a better fashion.
+        // See issue #17670 and related issue #14121.
+        if (scalarAccessExpression.PropertyName.Length > 0)
+        {
+            _sqlBuilder
+                .Append("[\"")
+                .Append(scalarAccessExpression.PropertyName)
+                .Append("\"]");
+        }
+
+        return scalarAccessExpression;
     }
 
     /// <summary>
@@ -128,7 +175,12 @@ protected override Expression VisitKeyAccess(KeyAccessExpression keyAccessExpres
     /// </summary>
     protected override Expression VisitObjectAccess(ObjectAccessExpression objectAccessExpression)
     {
-        _sqlBuilder.Append(objectAccessExpression.ToString());
+        Visit(objectAccessExpression.Object);
+
+        _sqlBuilder
+            .Append("[\"")
+            .Append(objectAccessExpression.PropertyName)
+            .Append("\"]");
 
         return objectAccessExpression;
     }
@@ -474,6 +526,30 @@ protected override Expression VisitSqlBinary(SqlBinaryExpression sqlBinaryExpres
         return sqlBinaryExpression;
     }
 
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    protected override Expression VisitObjectBinary(ObjectBinaryExpression objectBinaryExpression)
+    {
+        var op = objectBinaryExpression.OperatorType switch
+        {
+            ExpressionType.Coalesce => " ?? ",
+
+            _ => throw new UnreachableException($"Unsupported unary OperatorType: {objectBinaryExpression.OperatorType}")
+        };
+
+        _sqlBuilder.Append('(');
+        Visit(objectBinaryExpression.Left);
+        _sqlBuilder.Append(op);
+        Visit(objectBinaryExpression.Right);
+        _sqlBuilder.Append(')');
+
+        return objectBinaryExpression;
+    }
+
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
     ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
@@ -755,6 +831,18 @@ protected virtual void GenerateIn(InExpression inExpression, bool negated)
         _sqlBuilder.Append(')');
     }
 
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    protected override Expression VisitObjectFunction(ObjectFunctionExpression objectFunctionExpression)
+    {
+        GenerateFunction(objectFunctionExpression.Name, objectFunctionExpression.Arguments);
+        return objectFunctionExpression;
+    }
+
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
     ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
@@ -763,12 +851,16 @@ protected virtual void GenerateIn(InExpression inExpression, bool negated)
     /// </summary>
     protected override Expression VisitSqlFunction(SqlFunctionExpression sqlFunctionExpression)
     {
-        _sqlBuilder.Append(sqlFunctionExpression.Name);
+        GenerateFunction(sqlFunctionExpression.Name, sqlFunctionExpression.Arguments);
+        return sqlFunctionExpression;
+    }
+
+    private void GenerateFunction(string name, IReadOnlyList<Expression> arguments)
+    {
+        _sqlBuilder.Append(name);
         _sqlBuilder.Append('(');
-        GenerateList(sqlFunctionExpression.Arguments, e => Visit(e));
+        GenerateList(arguments, e => Visit(e));
         _sqlBuilder.Append(')');
-
-        return sqlFunctionExpression;
     }
 
     private sealed class ParameterNameGenerator