From ca37b4a123f9efd8429d078d940dd4e0987a1e78 Mon Sep 17 00:00:00 2001 From: Mike Dias Date: Fri, 25 Sep 2020 21:59:52 +0000 Subject: [PATCH] [DELTA-OSS-EXTERNAL] Parse the partition structure using the delta_log as the base path This allows converting a location that is under a path that looks like a partition value. For example, running the convert to delta command over a path like `s3://massive-events/year=2020/` would fail because the command will try to compare the partitions above the delta log base path. Signed-off-by: Mike Dias Closes delta-io/delta#510 Signed-off-by: liwensun Author: Mike Dias #12643 is resolved by liwensun/4w7hocg4. GitOrigin-RevId: 4dc2e55c97b47b3ce928a5a780b115324170e95d --- .../delta/commands/ConvertToDeltaCommand.scala | 2 +- .../spark/sql/delta/ConvertToDeltaSuiteBase.scala | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/scala/org/apache/spark/sql/delta/commands/ConvertToDeltaCommand.scala b/src/main/scala/org/apache/spark/sql/delta/commands/ConvertToDeltaCommand.scala index 0fc7521d998..72ed31a9ec9 100644 --- a/src/main/scala/org/apache/spark/sql/delta/commands/ConvertToDeltaCommand.scala +++ b/src/main/scala/org/apache/spark/sql/delta/commands/ConvertToDeltaCommand.scala @@ -395,7 +395,7 @@ abstract class ConvertToDeltaCommandBase( val (partitionOpt, _) = PartitionUtils.parsePartition( dir, typeInference = false, - basePaths = Set.empty, + basePaths = Set(basePath), userSpecifiedDataTypes = Map.empty, validatePartitionColumns = false, java.util.TimeZone.getDefault, diff --git a/src/test/scala/org/apache/spark/sql/delta/ConvertToDeltaSuiteBase.scala b/src/test/scala/org/apache/spark/sql/delta/ConvertToDeltaSuiteBase.scala index 8c2b949eb4c..d6fcfe08fba 100644 --- a/src/test/scala/org/apache/spark/sql/delta/ConvertToDeltaSuiteBase.scala +++ b/src/test/scala/org/apache/spark/sql/delta/ConvertToDeltaSuiteBase.scala @@ -1036,4 +1036,19 @@ trait ConvertToDeltaHiveTableTests extends ConvertToDeltaTestUtils with SQLTestU } } } + + test("can convert a partition-like table path") { + withTempDir { dir => + val path = dir.getCanonicalPath + writeFiles(path, simpleDF, partCols = Seq("key1", "key2")) + + val basePath = s"$path/key1=1/" + convertToDelta(s"parquet.`$basePath`", Some("key2 string")) + + checkAnswer( + sql(s"select id from delta.`$basePath` where key2 = '1'"), + simpleDF.filter("id % 2 == 1").filter("id % 3 == 1").select("id")) + } + } + }