Skip to content

fix: unignore row index Spark SQL tests for native_datafusion #3414

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mbutrovich merged 1 commit into from
Feb 5, 2026
Merged

fix: unignore row index Spark SQL tests for native_datafusion #3414

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 0 additions & 43 deletions dev/diffs/3.5.8.diff
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2138,49 +2138,6 @@ index 5e01d3f447c..284d6657d4f 100644
withTempDir { dir =>
val readSchema =
new StructType()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileMetadataStructRowIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileMetadataStructRowIndexSuite.scala
index c10e1799702..ba6629abfd9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileMetadataStructRowIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileMetadataStructRowIndexSuite.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.execution.datasources.parquet

-import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest}
+import org.apache.spark.sql.{AnalysisException, DataFrame, IgnoreCometNativeDataFusion, QueryTest}
import org.apache.spark.sql.execution.datasources.FileFormat
import org.apache.spark.sql.functions.{col, lit}
import org.apache.spark.sql.internal.SQLConf
@@ -154,7 +154,8 @@ class ParquetFileMetadataStructRowIndexSuite extends QueryTest with SharedSparkS
}
}

- test(s"reading ${ROW_INDEX_TEMPORARY_COLUMN_NAME} - not present in a table") {
+ test(s"reading ${ROW_INDEX_TEMPORARY_COLUMN_NAME} - not present in a table",
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3317")) {
// File format supporting row index generation populates the column with row indexes.
withReadDataFrame("parquet", extraSchemaFields =
Seq(StructField(ROW_INDEX_TEMPORARY_COLUMN_NAME, LongType))) { df =>
@@ -172,7 +173,8 @@ class ParquetFileMetadataStructRowIndexSuite extends QueryTest with SharedSparkS
}
}

- test(s"reading ${ROW_INDEX_TEMPORARY_COLUMN_NAME} - present in a table") {
+ test(s"reading ${ROW_INDEX_TEMPORARY_COLUMN_NAME} - present in a table",
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3317")) {
withReadDataFrame("parquet", extraCol = ROW_INDEX_TEMPORARY_COLUMN_NAME) { df =>
// Values of ROW_INDEX_TEMPORARY_COLUMN_NAME column are always populated with
// generated row indexes, rather than read from the file.
@@ -189,7 +191,8 @@ class ParquetFileMetadataStructRowIndexSuite extends QueryTest with SharedSparkS
}
}

- test(s"reading ${ROW_INDEX_TEMPORARY_COLUMN_NAME} - as partition col") {
+ test(s"reading ${ROW_INDEX_TEMPORARY_COLUMN_NAME} - as partition col",
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3317")) {
withReadDataFrame("parquet", partitionCol = ROW_INDEX_TEMPORARY_COLUMN_NAME) { df =>
// Column values are set for each partition, rather than populated with generated row indexes.
assert(df
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 8e88049f51e..49f2001dc6b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
Expand Down
Loading