Additional refs update for iceberg 1.9.0 (#1491)

MonkeyCanCode · web-flow · commit 7c90731660e6 · 2025-05-05T10:44:08.000-05:00
* Additional refs update for iceberg 1.9.0

* Additional refs update for iceberg 1.9.0

* Additional refs update for iceberg 1.9.0
diff --git a/getting-started/eclipselink/docker-compose.yml b/getting-started/eclipselink/docker-compose.yml
@@ -73,7 +73,7 @@ services:
       retries: 15
     command: [
       /opt/spark/bin/spark-sql,
-      --packages, "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.7.0,org.apache.iceberg:iceberg-azure-bundle:1.7.0",
+      --packages, "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.9.0,org.apache.iceberg:iceberg-azure-bundle:1.9.0",
       --conf, "spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
       --conf, "spark.sql.catalog.quickstart_catalog=org.apache.iceberg.spark.SparkCatalog",
       --conf, "spark.sql.catalog.quickstart_catalog.type=rest",
diff --git a/getting-started/jdbc/docker-compose.yml b/getting-started/jdbc/docker-compose.yml
@@ -73,7 +73,7 @@ services:
       retries: 15
     command: [
       /opt/spark/bin/spark-sql,
-      --packages, "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.7.0,org.apache.iceberg:iceberg-azure-bundle:1.7.0",
+      --packages, "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.9.0,org.apache.iceberg:iceberg-azure-bundle:1.9.0",
       --conf, "spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
       --conf, "spark.sql.catalog.polaris=org.apache.iceberg.spark.SparkCatalog",
       --conf, "spark.sql.catalog.polaris.type=rest",
diff --git a/getting-started/spark/notebooks/SparkPolaris.ipynb b/getting-started/spark/notebooks/SparkPolaris.ipynb
@@ -256,7 +256,7 @@
     "\n",
     "spark = (SparkSession.builder\n",
     "  .config(\"spark.sql.catalog.spark_catalog\", \"org.apache.iceberg.spark.SparkSessionCatalog\")\n",
-    "  .config(\"spark.jars.packages\", \"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19\")\n",
+    "  .config(\"spark.jars.packages\", \"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19\")\n",
     "  .config('spark.sql.iceberg.vectorization.enabled', 'false')\n",
     "         \n",
     "  # Configure the 'polaris' catalog as an Iceberg rest catalog\n",
diff --git a/plugins/pluginlibs.versions.toml b/plugins/pluginlibs.versions.toml
@@ -18,7 +18,7 @@
 #
 
 [versions]
-iceberg = "1.8.1"
+iceberg = "1.9.0"
 spark35 = "3.5.5"
 scala212 = "2.12.19"
 scala213 = "2.13.15"
diff --git a/plugins/spark/README.md b/plugins/spark/README.md
@@ -6,9 +6,9 @@
   to you under the Apache License, Version 2.0 (the
   "License"); you may not use this file except in compliance
   with the License.  You may obtain a copy of the License at
- 
+
    http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing,
   software distributed under the License is distributed on an
   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,13 +20,13 @@
 # Polaris Spark Plugin
 
 The Polaris Spark plugin provides a SparkCatalog class, which communicates with the Polaris
-REST endpoints, and provides implementations for Apache Spark's 
-[TableCatalog](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java), 
-[SupportsNamespaces](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java), 
+REST endpoints, and provides implementations for Apache Spark's
+[TableCatalog](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java),
+[SupportsNamespaces](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java),
 [ViewCatalog](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/ViewCatalog.java) classes.
 
 Right now, the plugin only provides support for Spark 3.5, Scala version 2.12 and 2.13,
-and depends on iceberg-spark-runtime 1.8.1.
+and depends on iceberg-spark-runtime 1.9.0.
 
 # Build Plugin Jar
 A task createPolarisSparkJar is added to build a jar for the Polaris Spark plugin, the jar is named as:
@@ -87,7 +87,7 @@ bin/spark-shell \
 ```
 
 # Limitations
-The Polaris Spark client supports catalog management for both Iceberg and Delta tables, it routes all Iceberg table 
+The Polaris Spark client supports catalog management for both Iceberg and Delta tables, it routes all Iceberg table
 requests to the Iceberg REST endpoints, and routes all Delta table requests to the Generic Table REST endpoints.
 
 The Spark Client requires at least delta 3.2.1 to work with Delta tables, which requires at least Apache Spark 3.5.3.
diff --git a/plugins/spark/v3.5/spark/build.gradle.kts b/plugins/spark/v3.5/spark/build.gradle.kts
@@ -167,9 +167,11 @@ tasks.register<ShadowJar>("createPolarisSparkJar") {
   minimize {
     exclude(dependency("org.apache.iceberg:iceberg-spark-runtime-*.*"))
     exclude(dependency("org.apache.iceberg:iceberg-core*.*"))
+    exclude(dependency("org.apache.avro:avro*.*"))
   }
 
   relocate("com.fasterxml", "org.apache.polaris.shaded.com.fasterxml.jackson")
+  relocate("org.apache.avro", "org.apache.polaris.shaded.org.apache.avro")
 }
 
 tasks.withType(Jar::class).named("sourcesJar") { dependsOn("createPolarisSparkJar") }
diff --git a/site/content/in-dev/unreleased/getting-started/using-polaris.md b/site/content/in-dev/unreleased/getting-started/using-polaris.md
@@ -147,7 +147,7 @@ _Note: the credentials provided here are those for our principal, not the root c
 
 ```shell
 bin/spark-sql \
---packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.1,org.apache.hadoop:hadoop-aws:3.4.0 \
+--packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,org.apache.hadoop:hadoop-aws:3.4.0 \
 --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
 --conf spark.sql.catalog.quickstart_catalog.warehouse=quickstart_catalog \
 --conf spark.sql.catalog.quickstart_catalog.header.X-Iceberg-Access-Delegation=vended-credentials \

Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@`
`18`	`18`	`#`
`19`	`19`
`20`	`20`	`[versions]`
`21`		`-iceberg = "1.8.1"`
	`21`	`+iceberg = "1.9.0"`
`22`	`22`	`spark35 = "3.5.5"`
`23`	`23`	`scala212 = "2.12.19"`
`24`	`24`	`scala213 = "2.13.15"`
Original file line number	Diff line number	Diff line change
`@@ -167,9 +167,11 @@ tasks.register<ShadowJar>("createPolarisSparkJar") {`
`167`	`167`	`minimize {`
`168`	`168`	`exclude(dependency("org.apache.iceberg:iceberg-spark-runtime-."))`
`169`	`169`	`exclude(dependency("org.apache.iceberg:iceberg-core."))`
	`170`	`+ exclude(dependency("org.apache.avro:avro."))`
`170`	`171`	`}`
`171`	`172`
`172`	`173`	`relocate("com.fasterxml", "org.apache.polaris.shaded.com.fasterxml.jackson")`
	`174`	`+ relocate("org.apache.avro", "org.apache.polaris.shaded.org.apache.avro")`
`173`	`175`	`}`
`174`	`176`
`175`	`177`	`tasks.withType(Jar::class).named("sourcesJar") { dependsOn("createPolarisSparkJar") }`