From d9a5f9f9c64b5066467801917de158650ae091cb Mon Sep 17 00:00:00 2001 From: Tathagata Das Date: Fri, 16 Jun 2023 12:09:52 -0700 Subject: [PATCH] Re-enabled the iceberg module using iceberg-1.3 on spark 3.4 ## Description Re-enabled the iceberg module using iceberg-1.3 on spark 3.4 Existing unit tests. ## Does this PR introduce _any_ user-facing changes? No Closes delta-io/delta#1820 Signed-off-by: Scott Sandre GitOrigin-RevId: c48bfb7c3f4bc3d28b99f31d1d3d6d60afd2fbad --- build.sbt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index 7da55cf1d67..dedc3232ea9 100644 --- a/build.sbt +++ b/build.sbt @@ -242,9 +242,7 @@ lazy val storageS3DynamoDB = (project in file("storage-s3-dynamodb")) ) ) -// Requires iceberg release on 3.4 -/** -lazy val deltaIceberg = (project in file("delta-iceberg")) +lazy val iceberg = (project in file("iceberg")) .dependsOn(spark % "compile->compile;test->test;provided->provided") .settings ( name := "delta-iceberg", @@ -253,7 +251,7 @@ lazy val deltaIceberg = (project in file("delta-iceberg")) releaseSettings, libraryDependencies ++= Seq( { val (expMaj, expMin, _) = getMajorMinorPatch(sparkVersion) - ("org.apache.iceberg" % s"iceberg-spark-runtime-$expMaj.$expMin" % "1.1.0" % "provided") + ("org.apache.iceberg" % s"iceberg-spark-runtime-$expMaj.$expMin" % "1.3.0" % "provided") .cross(CrossVersion.binary) }, // Fix Iceberg's legacy java.lang.NoClassDefFoundError: scala/jdk/CollectionConverters$ error @@ -261,7 +259,6 @@ lazy val deltaIceberg = (project in file("delta-iceberg")) "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.1" ) ) -*/ /** * Get list of python files and return the mapping between source files and target paths @@ -291,7 +288,7 @@ val createTargetClassesDir = taskKey[Unit]("create target classes dir") // Don't use these groups for any other projects lazy val sparkGroup = project - .aggregate(spark, contribs, storage, storageS3DynamoDB) + .aggregate(spark, contribs, storage, storageS3DynamoDB, iceberg) .settings( // crossScalaVersions must be set to Nil on the aggregating project crossScalaVersions := Nil,