chore: DataFusion 52 migration

dev/diffs/4.0.1.diff

@@ -2960,6 +2960,30 @@ index 09ed6955a51..236a4e99824 100644
      )
    }
    test(s"parquet widening conversion $fromType -> $toType") {
+diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
+index 458b5dfc0f4..d209f3c85bc 100644
+--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
++++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
+@@ -26,7 +26,7 @@ import org.apache.parquet.hadoop.util.HadoopInputFile
+ import org.apache.parquet.schema.{LogicalTypeAnnotation, PrimitiveType}
+ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
+
+-import org.apache.spark.sql.{QueryTest, Row}
++import org.apache.spark.sql.{IgnoreCometSuite, QueryTest, Row}
+ import org.apache.spark.sql.internal.SQLConf
+ import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType
+ import org.apache.spark.sql.test.SharedSparkSession
+@@ -35,7 +35,9 @@ import org.apache.spark.unsafe.types.VariantVal
+ /**
+  * Test shredding Variant values in the Parquet reader/writer.
+  */
+-class ParquetVariantShreddingSuite extends QueryTest with ParquetTest with SharedSparkSession {
++class ParquetVariantShreddingSuite extends QueryTest with ParquetTest with SharedSparkSession
++    // TODO enable tests once https://github.com/apache/datafusion-comet/issues/2209 is fixed
++    with IgnoreCometSuite {
+
+   private def testWithTempDir(name: String)(block: File => Unit): Unit = test(name) {
+     withTempDir { dir =>
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
 index b8f3ea3c6f3..bbd44221288 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala