update

Author: lzlfred

spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala

@@ -641,7 +641,7 @@ object DeltaLog extends DeltaLogging {
   /**
    * Helper to create delta log caches
    */
-  private[delta] def createCacheBuilder(conf: SQLConf): CacheBuilder[AnyRef, AnyRef] = {
+  private def createCacheBuilder(conf: SQLConf): CacheBuilder[AnyRef, AnyRef] = {
     val cacheRetention = conf.getConf(DeltaSQLConf.DELTA_LOG_CACHE_RETENTION_MINUTES)
     val cacheSize = conf
       .getConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE)
@@ -808,8 +808,8 @@ object DeltaLog extends DeltaLogging {
       // - Different `authority` (e.g., different user tokens in the path)
       // - Different mount point.
       try {
-        getOrCreateCache(spark.sessionState.conf).get(
-            path -> fileSystemOptions, () => {
+        getOrCreateCache(spark.sessionState.conf)
+          .get(path -> fileSystemOptions, () => {
             createDeltaLog()
           }
         )
@@ -864,9 +864,7 @@ object DeltaLog extends DeltaLogging {
   }
 
   def clearCache(): Unit = {
-    deltaLogCache.foreach { cache =>
-      cache.invalidateAll()
-    }
+    deltaLogCache.foreach(_.invalidateAll())
   }
 
   /** Unset the caches. Exposing for testing */

spark/src/test/scala/org/apache/spark/sql/delta/DeltaLogSuite.scala

@@ -655,10 +655,22 @@ class DeltaLogSuite extends QueryTest
       assert(DeltaLog.cacheSize === expected)
     }
     DeltaLog.unsetCache()
-    spark.sessionState.conf.setConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE, 2L)
-    assertCacheSize(2)
-    DeltaLog.unsetCache()
-    System.getProperties.setProperty("delta.log.cacheSize", "3")
-    assertCacheSize(3)
+    withSQLConf(DeltaSQLConf.DELTA_LOG_CACHE_SIZE.key -> "2") {
+      assertCacheSize(2)
+      DeltaLog.unsetCache()
+      // the larger of SQLConf and env var is adopted
+      try {
+        System.getProperties.setProperty("delta.log.cacheSize", "3")
+        assertCacheSize(3)
+      } finally {
+        System.getProperties.remove("delta.log.cacheSize")
+      }
+    }
+
+    // assert timeconf returns correct value
+    withSQLConf(DeltaSQLConf.DELTA_LOG_CACHE_RETENTION_MINUTES.key -> "100") {
+      assert(spark.sessionState.conf.getConf(
+        DeltaSQLConf.DELTA_LOG_CACHE_RETENTION_MINUTES) === 100)
+    }
   }
 }