[CARMEL-2308][FOLLOWUP] update table statistics accurately (delta-io#87)

LantaoJin · Jul 30, 2020 · e3bc9af · e3bc9af
1 parent f40b326
commit e3bc9af
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 8 deletions.
diff --git a/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala b/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala
@@ -299,11 +299,6 @@ trait OptimisticTransactionImpl extends TransactionalWrite with SQLMetricsReport
         finalActions = commitInfo +: finalActions
       }
 
-      // the table stats may be not accurate
-      if (catalogTable.nonEmpty) {
-        CommandUtils.updateTableStats(spark, catalogTable.get)
-      }
-
       // Register post-commit hooks if any
       lazy val hasFileActions = finalActions.collect { case f: FileAction => f }.nonEmpty
       if (DeltaConfigs.SYMLINK_FORMAT_MANIFEST_ENABLED.fromMetaData(metadata) && hasFileActions) {
@@ -313,6 +308,10 @@ trait OptimisticTransactionImpl extends TransactionalWrite with SQLMetricsReport
       val commitVersion = doCommit(snapshot.version + 1, finalActions, 0, isolationLevelToUse)
       logInfo(s"Committed delta #$commitVersion to ${deltaLog.logPath}")
       postCommit(commitVersion, finalActions)
+
+      if (catalogTable.nonEmpty) {
+        CommandUtils.updateTableStats(spark, catalogTable.get)
+      }
       commitVersion
     } catch {
       case e: DeltaConcurrentModificationException =>

diff --git a/src/test/scala/org/apache/spark/sql/delta/services/SQLQueryTest.scala b/src/test/scala/org/apache/spark/sql/delta/services/SQLQueryTest.scala
@@ -113,7 +113,7 @@ class SQLQuerySuite extends QueryTest
         table = catalog.getTableMetadata(TableIdentifier("target"))
         assert(table.stats.isDefined)
         val size3 = table.stats.get.sizeInBytes
-        assert(size2 < size3)
+        assert(size2 == size3)
 
         sql(
           """
@@ -126,7 +126,7 @@ class SQLQuerySuite extends QueryTest
         table = catalog.getTableMetadata(TableIdentifier("target"))
         assert(table.stats.isDefined)
         val size4 = table.stats.get.sizeInBytes
-        assert(size3 < size4)
+        assert(size3 > size4)
 
         sql(
           """
@@ -139,7 +139,7 @@ class SQLQuerySuite extends QueryTest
         table = catalog.getTableMetadata(TableIdentifier("target"))
         assert(table.stats.isDefined)
         val size5 = table.stats.get.sizeInBytes
-        assert(size4 > size5)
+        assert(size4 == size5)
       }
     }
   }