diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DatasetManager.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DatasetManager.scala index d8bc3913e5ad..0d393e307aaa 100644 --- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DatasetManager.scala +++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DatasetManager.scala @@ -179,13 +179,12 @@ object DatasetManager extends Logging { } // Wipe the data if we need to - val dropTable = (isFullRefresh || !table.isStreamingTable) && existingTableOpt.isDefined - if (dropTable) { - catalog.dropTable(identifier) + if ((isFullRefresh || !table.isStreamingTable) && existingTableOpt.isDefined) { + context.spark.sql(s"TRUNCATE TABLE ${table.identifier.quotedString}") } // Alter the table if we need to - if (existingTableOpt.isDefined && !dropTable) { + if (existingTableOpt.isDefined) { val existingSchema = v2ColumnsToStructType(existingTableOpt.get.columns()) val targetSchema = if (table.isStreamingTable && !isFullRefresh) { @@ -200,7 +199,7 @@ object DatasetManager extends Logging { } // Create the table if we need to - if (dropTable || existingTableOpt.isEmpty) { + if (existingTableOpt.isEmpty) { catalog.createTable( identifier, new TableInfo.Builder()