Skip to content

Commit e573644

Browse files
committed
[SPARK-49054][SQL] Column default value should support current_* functions
### What changes were proposed in this pull request? This is a regression between Spark 3.5.0 and Spark 4. The following queries work on Spark 3.5.0 while fails on latest master branch: ``` CREATE TABLE test_current_user(i int, s string) USING parquet; ALTER TABLE test_current_user ALTER COLUMN s SET DEFAULT current_user() ``` ``` CREATE TABLE test_current_user(i int, s string default current_user()) USING parquet INSERT INTO test_current_user (i) VALUES ((0)); ``` This PR is to complete fixing this by eagerly executing finish-analysis and constant-folding rules before checking whether the expression is foldable and resolved. ### Why are the changes needed? Bug fix ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? New UTs ### Was this patch authored or co-authored using generative AI tooling? No Closes #47529 from gengliangwang/finishAnalysis. Authored-by: Gengliang Wang <gengliang@apache.org> Signed-off-by: Gengliang Wang <gengliang@apache.org>
1 parent 1ec211e commit e573644

File tree

2 files changed

+30
-2
lines changed

2 files changed

+30
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.analysis._
2828
import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, InMemoryCatalog, SessionCatalog}
2929
import org.apache.spark.sql.catalyst.expressions._
3030
import org.apache.spark.sql.catalyst.expressions.{Literal => ExprLiteral}
31-
import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, ReplaceExpressions}
31+
import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, Optimizer}
3232
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException}
3333
import org.apache.spark.sql.catalyst.plans.logical._
3434
import org.apache.spark.sql.catalyst.trees.TreePattern.PLAN_EXPRESSION
@@ -290,7 +290,9 @@ object ResolveDefaultColumns extends QueryErrorsBase
290290
val analyzer: Analyzer = DefaultColumnAnalyzer
291291
val analyzed = analyzer.execute(Project(Seq(Alias(parsed, colName)()), OneRowRelation()))
292292
analyzer.checkAnalysis(analyzed)
293-
ConstantFolding(ReplaceExpressions(analyzed))
293+
// Eagerly execute finish-analysis and constant-folding rules before checking whether the
294+
// expression is foldable and resolved.
295+
ConstantFolding(DefaultColumnOptimizer.FinishAnalysis(analyzed))
294296
} catch {
295297
case ex: AnalysisException =>
296298
throw QueryCompilationErrors.defaultValuesUnresolvedExprError(
@@ -517,6 +519,11 @@ object ResolveDefaultColumns extends QueryErrorsBase
517519
new CatalogManager(BuiltInFunctionCatalog, BuiltInFunctionCatalog.v1Catalog)) {
518520
}
519521

522+
/**
523+
* This is an Optimizer for convert default column expressions to foldable literals.
524+
*/
525+
object DefaultColumnOptimizer extends Optimizer(DefaultColumnAnalyzer.catalogManager)
526+
520527
/**
521528
* This is a FunctionCatalog for performing analysis using built-in functions only. It is a helper
522529
* for the DefaultColumnAnalyzer above.

sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,4 +287,25 @@ class ResolveDefaultColumnsSuite extends QueryTest with SharedSparkSession {
287287
checkAnswer(sql("select v from t"), sql("select parse_json('1')").collect())
288288
}
289289
}
290+
291+
test("SPARK-49054: Create table with current_user() default") {
292+
val tableName = "test_current_user"
293+
val user = spark.sparkContext.sparkUser
294+
withTable(tableName) {
295+
sql(s"CREATE TABLE $tableName(i int, s string default current_user()) USING parquet")
296+
sql(s"INSERT INTO $tableName (i) VALUES ((0))")
297+
checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(0, user)))
298+
}
299+
}
300+
301+
test("SPARK-49054: Alter table with current_user() default") {
302+
val tableName = "test_current_user"
303+
val user = spark.sparkContext.sparkUser
304+
withTable(tableName) {
305+
sql(s"CREATE TABLE $tableName(i int, s string) USING parquet")
306+
sql(s"ALTER TABLE $tableName ALTER COLUMN s SET DEFAULT current_user()")
307+
sql(s"INSERT INTO $tableName (i) VALUES ((0))")
308+
checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(0, user)))
309+
}
310+
}
290311
}

0 commit comments

Comments
 (0)