diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala index d8806f03443f..0af45d9472a1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala @@ -583,9 +583,8 @@ class MicroBatchExecution( case FileSourceMetadataAttribute(_) => true case _ => false } - val finalDataPlan = dataPlan match { + val finalDataPlan = dataPlan transformUp { case l: LogicalRelation if hasFileMetadata => l.withMetadataColumns() - case _ => dataPlan } // SPARK-40460: overwrite the entry with the new logicalPlan // because it might contain the _metadata column. It is a necessary change,