Skip to content

Commit 473cd39

Browse files
committed
fix tests
1 parent 3fbd368 commit 473cd39

File tree

1 file changed

+4
-4
lines changed

1 file changed

+4
-4
lines changed

sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2305,7 +2305,7 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
23052305
}
23062306

23072307
// batch 5 will trigger list operation though the batch 4 should have 1 unseen file:
2308-
// 1 is smaller than the threshold (refer FileStreamOptions.discardCachedFilesRatio),
2308+
// 1 is smaller than the threshold (refer FileStreamOptions.discardCachedInputRatio),
23092309
// hence unseen files for batch 4 will be discarded.
23102310
val offsetBatch = source.latestOffset(FileStreamSourceOffset(-1L), ReadLimit.maxFiles(10))
23112311
.asInstanceOf[FileStreamSourceOffset]
@@ -2361,7 +2361,7 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
23612361
withCountListingLocalFileSystemAsLocalFileSystem {
23622362
withThreeTempDirs { case (src, meta, tmp) =>
23632363
val options = Map("latestFirst" -> "false", "maxFilesPerTrigger" -> "10",
2364-
"maxCachedFiles" -> "12", "discardCachedFilesRatio" -> "0.1")
2364+
"maxCachedFiles" -> "12", "discardCachedInputRatio" -> "0.1")
23652365
val scheme = CountListingLocalFileSystem.scheme
23662366
val source = new FileStreamSource(spark, s"$scheme:///${src.getCanonicalPath}/*/*", "text",
23672367
StructType(Nil), Seq.empty, meta.getCanonicalPath, options)
@@ -2406,7 +2406,7 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
24062406
// batch 0 processes 10 (12 cached)
24072407
// batch 1 processes 10 from cache (2 cached)
24082408
// batch 2 processes 2 from cache (0 cached) since
2409-
// discardCachedFilesRatio is less than threshold
2409+
// discardCachedInputRatio is less than threshold
24102410
val offsetBatch0 = source.latestOffset(FileStreamSourceOffset(-1L), ReadLimit.maxFiles(10))
24112411
.asInstanceOf[FileStreamSourceOffset]
24122412
verifyBatch(offsetBatch0, expectedBatchId = 0, inputFiles,
@@ -2424,7 +2424,7 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
24242424
// batch 3 processes 10 (12 cached)
24252425
// batch 4 processes 10 from cache (2 cached)
24262426
// batch 5 processes 2 from cache (0 cached) since
2427-
// discardCachedFilesRatio is less than threshold
2427+
// discardCachedInputRatio is less than threshold
24282428
val offsetBatch3 = source.latestOffset(FileStreamSourceOffset(-1L), ReadLimit.maxFiles(10))
24292429
.asInstanceOf[FileStreamSourceOffset]
24302430
verifyBatch(offsetBatch3, expectedBatchId = 3, inputFiles,

0 commit comments

Comments
 (0)