Skip to content

[SPARK-19784][SQL][WIP]refresh table after alter the location #17119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
windpiger wants to merge 4 commits into from
Closed

[SPARK-19784][SQL][WIP]refresh table after alter the location #17119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3a7067e
[SPARK-19577][SQL]refresh table after alter the location
windpiger Mar 1, 2017
9050349
add hive table test
windpiger Mar 1, 2017
dccac9a
add more test
windpiger Mar 1, 2017
be98a0f
add more test
windpiger Mar 2, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -754,6 +754,8 @@ case class AlterTableSetLocationCommand(
// No partition spec is specified, so we set the location for the table itself
catalog.alterTable(table.withNewStorage(locationUri = Some(location)))
}

sparkSession.sessionState.catalog.refreshTable(table.identifier)
Seq.empty[Row]
}
}
Expand Down
24 changes: 24 additions & 0 deletions sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1952,4 +1952,28 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach {
}
}
}

Seq(true, false).foreach { shouldCache =>
val testName = if (shouldCache) "cached" else "non-cached"
test(s"refresh $testNames table after alter the location") {
withTable("t", "t1", "t2", "t3") {
withTempDir { dir =>
spark.sql(
"""
|CREATE TABLE t(a string)
|USING parquet
""".stripMargin)
spark.sql("INSERT INTO TABLE t SELECT 1")
if (shouldCache) {
spark.catalog.cacheTable("t")
}
checkAnswer(spark.table("t"), Row("1") :: Nil)
spark.sql(s"ALTER TABLE t SET LOCATION '$dir'")
checkAnswer(spark.table("t"), Nil)
}

// TODO: partition table tests
}
}
}
}
69 changes: 69 additions & 0 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1587,4 +1587,73 @@ class HiveDDLSuite
}
}
}

Seq(true, false).foreach { shouldCache =>
val testName = if (shouldCache) "cached" else "non-cached"
test(s"refresh $testNames table after alter the location") {
withTable("t", "t1", "t2", "t3") {
withTempDir { dir =>
spark.sql(
"""
|CREATE TABLE t(a string)
|USING parquet
""".stripMargin)
spark.sql("INSERT INTO TABLE t SELECT 1")
if (shouldCache) {
spark.catalog.cacheTable("t")
}
checkAnswer(spark.table("t"), Row("1") :: Nil)
spark.sql(s"ALTER TABLE t SET LOCATION '$dir'")
checkAnswer(spark.table("t"), Nil)
}

withTempDir { dir =>
spark.sql(
"""
|CREATE TABLE t1(a string, b string)
|USING parquet
|PARTITIONED BY(b)
""".stripMargin)
spark.sql("INSERT INTO TABLE t1 PARTITION(b=1) SELECT 2")
if (shouldCache) {
spark.catalog.cacheTable("t")
}
checkAnswer(spark.table("t1"), Row("2", "1") :: Nil)
spark.sql(s"ALTER TABLE t1 PARTITION(b=1)SET LOCATION '$dir'")
checkAnswer(spark.table("t1"), Nil)
}

withTempDir { dir =>
spark.sql(
"""
|CREATE TABLE t2(a string)
|USING hive
""".stripMargin)
spark.sql("INSERT INTO TABLE t2 SELECT 1")
if (shouldCache) {
spark.catalog.cacheTable("t")
}
checkAnswer(spark.table("t2"), Row("1") :: Nil)
spark.sql(s"ALTER TABLE t2 SET LOCATION '$dir'")
checkAnswer(spark.table("t2"), Nil)
}

withTempDir { dir =>
spark.sql(
"""
|CREATE TABLE t3(a string, b string)
|USING hive
|PARTITIONED BY(b)
""".stripMargin)
spark.sql("INSERT INTO TABLE t3 PARTITION(b=1) SELECT 2")
if (shouldCache) {
spark.catalog.cacheTable("t")
}
checkAnswer(spark.table("t3"), Row("2", "1") :: Nil)
spark.sql(s"ALTER TABLE t3 PARTITION(b=1)SET LOCATION '$dir'")
checkAnswer(spark.table("t3"), Nil)
}
}
}
}
}