Spark: Initial support for 3.2 (#3335)

Author: aokolnychyi

.github/workflows/spark-ci.yml

@@ -98,3 +98,30 @@ jobs:
         name: test logs
         path: |
           **/build/testlogs
+
+  spark-32-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        jvm: [8, 11]
+        spark: ['3.2']
+    env:
+      SPARK_LOCAL_IP: localhost
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.jvm }}
+      - uses: actions/cache@v2
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+      - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-3.2:check :iceberg-spark:iceberg-spark-3.2-extensions:check :iceberg-spark:iceberg-spark-3.2-runtime:check -Pquick=true -x javadoc
+      - uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: test logs
+          path: |
+            **/build/testlogs

.gitignore

@@ -30,6 +30,8 @@ spark/v2.4/spark2/benchmark/*
 !spark/v2.4/spark2/benchmark/.gitkeep
 spark/v3.0/spark3/benchmark/*
 !spark/v3.0/spark3/benchmark/.gitkeep
+spark/v3.2/spark/benchmark/*
+!spark/v3.2/spark/benchmark/.gitkeep
 
 __pycache__/
 *.py[cod]

build.gradle

@@ -64,10 +64,6 @@ allprojects {
     mavenCentral()
     mavenLocal()
   }
-  project.ext {
-    Spark30Version = '3.0.3'
-    Spark31Version = '3.1.1'
-  }
 }
 
 subprojects {
@@ -92,12 +88,6 @@ subprojects {
       exclude group: 'com.sun.jersey'
       exclude group: 'com.sun.jersey.contribs'
       exclude group: 'org.pentaho', module: 'pentaho-aggdesigner-algorithm'
-
-      resolutionStrategy {
-        force 'com.fasterxml.jackson.module:jackson-module-scala_2.11:2.11.4'
-        force 'com.fasterxml.jackson.module:jackson-module-scala_2.12:2.11.4'
-        force 'com.fasterxml.jackson.module:jackson-module-paranamer:2.11.4'
-      }
     }
 
     testArtifacts

gradle.properties

@@ -19,7 +19,7 @@ systemProp.defaultFlinkVersions=1.13
 systemProp.knownFlinkVersions=1.13
 systemProp.defaultHiveVersions=2,3
 systemProp.knownHiveVersions=2,3
-systemProp.defaultSparkVersions=2.4,3.0
-systemProp.knownSparkVersions=2.4,3.0
+systemProp.defaultSparkVersions=2.4,3.0,3.2
+systemProp.knownSparkVersions=2.4,3.0,3.2
 org.gradle.parallel=true
 org.gradle.jvmargs=-Xmx768m

jmh.gradle

@@ -32,6 +32,10 @@ if (sparkVersions.contains("3.0")) {
   jmhProjects.add(project(":iceberg-spark:iceberg-spark3"))
 }
 
+if (sparkVersions.contains("3.2")) {
+  jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.2"))
+}
+
 configure(jmhProjects) {
   apply plugin: 'me.champeau.gradle.jmh'
 

settings.gradle

@@ -91,6 +91,18 @@ if (sparkVersions.contains("3.0")) {
   project(':iceberg-spark:spark3-runtime').name = 'iceberg-spark3-runtime'
 }
 
+if (sparkVersions.contains("3.2")) {
+  include ':iceberg-spark:spark-3.2'
+  include ':iceberg-spark:spark-3.2-extensions'
+  include ':iceberg-spark:spark-3.2-runtime'
+  project(':iceberg-spark:spark-3.2').projectDir = file('spark/v3.2/spark')
+  project(':iceberg-spark:spark-3.2').name = 'iceberg-spark-3.2'
+  project(':iceberg-spark:spark-3.2-extensions').projectDir = file('spark/v3.2/spark-extensions')
+  project(':iceberg-spark:spark-3.2-extensions').name = 'iceberg-spark-3.2-extensions'
+  project(':iceberg-spark:spark-3.2-runtime').projectDir = file('spark/v3.2/spark-runtime')
+  project(':iceberg-spark:spark-3.2-runtime').name = 'iceberg-spark-3.2-runtime'
+}
+
 // hive 3 depends on hive 2, so always add hive 2 if hive3 is enabled
 if (hiveVersions.contains("2") || hiveVersions.contains("3")) {
   include 'mr'

spark/build.gradle

@@ -27,3 +27,7 @@ if (jdkVersion == '8' && sparkVersions.contains("2.4")) {
 if (sparkVersions.contains("3.0")) {
   apply from: file("$projectDir/v3.0/build.gradle")
 }
+
+if (sparkVersions.contains("3.2")) {
+  apply from: file("$projectDir/v3.2/build.gradle")
+}

spark/v2.4/build.gradle

@@ -21,6 +21,22 @@ if (jdkVersion != '8') {
   throw new GradleException("Spark 2.4 must be built with Java 8")
 }
 
+def sparkProjects = [
+    project(':iceberg-spark:iceberg-spark2'),
+    project(':iceberg-spark:iceberg-spark-runtime')
+]
+
+configure(sparkProjects) {
+  configurations {
+    all {
+      resolutionStrategy {
+        force 'com.fasterxml.jackson.module:jackson-module-scala_2.11:2.11.4'
+        force 'com.fasterxml.jackson.module:jackson-module-paranamer:2.11.4'
+      }
+    }
+  }
+}
+
 project(':iceberg-spark:iceberg-spark2') {
   configurations.all {
     resolutionStrategy {

spark/v3.0/build.gradle

@@ -17,6 +17,27 @@
  * under the License.
  */
 
+def sparkProjects = [
+    project(':iceberg-spark:iceberg-spark3'),
+    project(":iceberg-spark:iceberg-spark3-extensions"),
+    project(':iceberg-spark:iceberg-spark3-runtime')
+]
+
+configure(sparkProjects) {
+  project.ext {
+    sparkVersion = '3.0.3'
+  }
+
+  configurations {
+    all {
+      resolutionStrategy {
+        force 'com.fasterxml.jackson.module:jackson-module-scala_2.12:2.11.4'
+        force 'com.fasterxml.jackson.module:jackson-module-paranamer:2.11.4'
+      }
+    }
+  }
+}
+
 project(':iceberg-spark:iceberg-spark3') {
   apply plugin: 'scala'
 
@@ -40,7 +61,7 @@ project(':iceberg-spark:iceberg-spark3') {
 
     compileOnly "com.google.errorprone:error_prone_annotations"
     compileOnly "org.apache.avro:avro"
-    compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") {
+    compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.apache.arrow'
     }
@@ -109,7 +130,7 @@ project(":iceberg-spark:iceberg-spark3-extensions") {
     compileOnly project(':iceberg-spark')
     compileOnly project(':iceberg-spark:iceberg-spark3')
     compileOnly project(':iceberg-hive-metastore')
-    compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") {
+    compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.apache.arrow'
     }
@@ -176,7 +197,7 @@ project(':iceberg-spark:iceberg-spark3-runtime') {
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }
 
-    integrationImplementation "org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}"
+    integrationImplementation "org.apache.spark:spark-hive_2.12:${sparkVersion}"
     integrationImplementation 'org.junit.vintage:junit-vintage-engine'
     integrationImplementation 'org.slf4j:slf4j-simple'
     integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')