Skip to content

Commit b8a7958

Browse files
gatorsmilecloud-fan
authored andcommitted
[SPARK-16936][SQL] Case Sensitivity Support for Refresh Temp Table
### What changes were proposed in this pull request? Currently, the `refreshTable` API is always case sensitive. When users use the view name without the exact case match, the API silently ignores the call. Users might expect the command has been successfully completed. However, when users run the subsequent SQL commands, they might still get the exception, like ``` Job aborted due to stage failure: Task 1 in stage 4.0 failed 1 times, most recent failure: Lost task 1.0 in stage 4.0 (TID 7, localhost): java.io.FileNotFoundException: File file:/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-bd4b9ea6-9aec-49c5-8f05-01cff426211e/part-r-00000-0c84b915-c032-4f2e-abf5-1d48fdbddf38.snappy.parquet does not exist ``` This PR is to fix the issue. ### How was this patch tested? Added a test case. Author: gatorsmile <gatorsmile@gmail.com> Closes #14523 from gatorsmile/refreshTempTable. (cherry picked from commit 5959df2) Signed-off-by: Wenchen Fan <wenchen@databricks.com>
1 parent ca0c6e6 commit b8a7958

File tree

3 files changed

+28
-3
lines changed

3 files changed

+28
-3
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -495,7 +495,7 @@ class SessionCatalog(
495495
// If the database is defined, this is definitely not a temp table.
496496
// If the database is not defined, there is a good chance this is a temp table.
497497
if (name.database.isEmpty) {
498-
tempTables.get(name.table).foreach(_.refresh())
498+
tempTables.get(formatTableName(name.table)).foreach(_.refresh())
499499
}
500500
}
501501

@@ -512,7 +512,7 @@ class SessionCatalog(
512512
* For testing only.
513513
*/
514514
private[catalog] def getTempTable(name: String): Option[LogicalPlan] = synchronized {
515-
tempTables.get(name)
515+
tempTables.get(formatTableName(name))
516516
}
517517

518518
// ----------------------------------------------------------------------------

sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1209,7 +1209,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
12091209
*
12101210
* For example:
12111211
* {{{
1212-
* CREATE [TEMPORARY] VIEW [IF NOT EXISTS] [db_name.]view_name
1212+
* CREATE [OR REPLACE] [TEMPORARY] VIEW [IF NOT EXISTS] [db_name.]view_name
12131213
* [(column_name [COMMENT column_comment], ...) ]
12141214
* [COMMENT view_comment]
12151215
* [TBLPROPERTIES (property_name = property_value, ...)]

sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package org.apache.spark.sql
2020
import java.io.File
2121

2222
import org.apache.spark.SparkException
23+
import org.apache.spark.sql.internal.SQLConf
2324
import org.apache.spark.sql.test.SharedSQLContext
2425

2526
/**
@@ -85,4 +86,28 @@ class MetadataCacheSuite extends QueryTest with SharedSQLContext {
8586
assert(newCount > 0 && newCount < 100)
8687
}}
8788
}
89+
90+
test("case sensitivity support in temporary view refresh") {
91+
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
92+
withTempView("view_refresh") {
93+
withTempPath { (location: File) =>
94+
// Create a Parquet directory
95+
spark.range(start = 0, end = 100, step = 1, numPartitions = 3)
96+
.write.parquet(location.getAbsolutePath)
97+
98+
// Read the directory in
99+
spark.read.parquet(location.getAbsolutePath).createOrReplaceTempView("view_refresh")
100+
101+
// Delete a file
102+
deleteOneFileInDirectory(location)
103+
intercept[SparkException](sql("select count(*) from view_refresh").first())
104+
105+
// Refresh and we should be able to read it again.
106+
spark.catalog.refreshTable("vIeW_reFrEsH")
107+
val newCount = sql("select count(*) from view_refresh").first().getLong(0)
108+
assert(newCount > 0 && newCount < 100)
109+
}
110+
}
111+
}
112+
}
88113
}

0 commit comments

Comments
 (0)