Skip to content

Commit

Permalink
[SC-72276] Add a config and test
Browse files Browse the repository at this point in the history
This PR adds a test so that we can detect #618

Author: Tathagata Das <tathagata.das1565@gmail.com>

GitOrigin-RevId: 1af03b03f64c607c8f61b41eef678f3a72355ad7
  • Loading branch information
tdas authored and mengtong-db committed Apr 6, 2021
1 parent 26e217e commit 7cb5115
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,18 @@ trait DeltaSQLConfBase {
.booleanConf
.createWithDefault(true)

val MERGE_SKIP_OSS_RESOLUTION_WITH_STAR =
buildConf("merge.skipOssResolutionWithStar")
.internal()
.doc(
"""
|If enabled, then any MERGE operation having UPDATE * / INSERT * will skip Apache
|Spark's resolution logic and use Delta's specific resolution logic. This is to avoid
|bug with star and temp views. See SC-72276 for details.
""".stripMargin)
.booleanConf
.createWithDefault(true)

val DELTA_LAST_COMMIT_VERSION_IN_SESSION =
buildConf("lastCommitVersionInSession")
.doc("The version of the last commit made in the SparkSession for any table.")
Expand Down
23 changes: 23 additions & 0 deletions src/test/scala/org/apache/spark/sql/delta/MergeIntoSQLSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -356,4 +356,27 @@ class MergeIntoSQLSuite extends MergeIntoSuiteBase with DeltaSQLCommandTest {
}
}
}

// This test is to capture the incorrect behavior caused by
// https://github.com/delta-io/delta/issues/618 .
// If this test fails then the issue has been fixed. Replace this test with a correct test
test("merge into a dataset temp views with star gives incorrect results") {
withTempView("v") {
withTempView("src") {
append(Seq((0, 0), (1, 1)).toDF("key", "value"))
readDeltaTable(tempPath).createOrReplaceTempView("v")
sql("CREATE TEMP VIEW src AS SELECT * FROM VALUES (10, 1) AS t(value, key)")
sql(s"""MERGE INTO v USING src
|ON src.key = v.key
|WHEN MATCHED THEN
| UPDATE SET *
|WHEN NOT MATCHED THEN
| INSERT *
|""".stripMargin)
val result = readDeltaTable(tempPath).as[(Long, Long)].collect().toSet
// This is expected to fail until the issue mentioned above is resolved.
assert(result != Set((0, 0), (1, 10)))
}
}
}
}

0 comments on commit 7cb5115

Please sign in to comment.