Merge remote-tracking branch 'origin/branch-23.08' into HEAD

NVIDIA · Jun 22, 2023 · 12e5164 · 12e5164
2 parents 9378dce + 870f9bf
commit 12e5164
Show file tree

Hide file tree

Showing 512 changed files with 17,109 additions and 5,550 deletions.
diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml
@@ -18,7 +18,7 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-    - branch-23.04
+    - branch-23.06
     types: [closed]
 
 jobs:
@@ -29,13 +29,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: branch-23.04 # force to fetch from latest upstream instead of PR ref
+          ref: branch-23.06 # force to fetch from latest upstream instead of PR ref
 
       - name: auto-merge job
         uses: ./.github/workflows/auto-merge
         env:
           OWNER: NVIDIA
           REPO_NAME: spark-rapids
-          HEAD: branch-23.04
-          BASE: branch-23.06
+          HEAD: branch-23.06
+          BASE: branch-23.08
           AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR
diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml
@@ -67,6 +67,7 @@ jobs:
       mattahrens,\
       YanxuanLiu,\
       cindyyuanjiang,\
+      thirtiseven,\
       ', format('{0},', github.actor)) && github.event.comment.body == 'build'
     steps:
       - name: Check if comment is issued by authorized person

diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml
@@ -23,13 +23,18 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  COMMON_MVN_FLAGS: >
+    -Ddist.jar.compress=false
+    -DskipTests
+    -Dskip
+
 jobs:
   get-shim-versions-from-dist:
     runs-on: ubuntu-latest
     outputs:
-      sparkHeadVersion: ${{ steps.allShimVersionsStep.outputs.headVersion }}
       sparkTailVersions: ${{ steps.allShimVersionsStep.outputs.tailVersions }}
-      sparkJDK11Versions: ${{ steps.allShimVersionsStep.outputs.jdk11Versions }}
+      sparkJDKVersions: ${{ steps.allShimVersionsStep.outputs.jdkVersions }}
     steps:
       - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
 
@@ -49,18 +54,24 @@ jobs:
           # do not add empty snapshot versions
           if [ ${#SPARK_SHIM_VERSIONS_SNAPSHOTS_ONLY[@]} -gt 0 ]; then
             svArrBodySnapshot=$(printf ",{\"spark-version\":\"%s\",\"isSnapshot\":true}" "${SPARK_SHIM_VERSIONS_SNAPSHOTS_ONLY[@]}")
+            svArrBodySnapshot=${svArrBodySnapshot:1}
+            svJsonStr=$(printf {\"include\":[%s]} $svArrBodyNoSnapshot,$svArrBodySnapshot)
+          else
+            svJsonStr=$(printf {\"include\":[%s]} $svArrBodyNoSnapshot)
           fi
 
-          # add snapshot versions which are not in snapshot property in pom file
-          svArrBodySnapshot+=$(printf ",{\"spark-version\":\"%s\",\"isSnapshot\":true}" 340)
-          svArrBodySnapshot=${svArrBodySnapshot:1}
-          svJsonStr=$(printf {\"include\":[%s]} $svArrBodyNoSnapshot,$svArrBodySnapshot)
-          echo "headVersion=$SPARK_BASE_SHIM_VERSION" >> $GITHUB_OUTPUT
           echo "tailVersions=$svJsonStr" >> $GITHUB_OUTPUT
-          jdkVersionArrBody=$(printf ",{\"spark-version\":\"%s\"}" "${SPARK_SHIM_VERSIONS_JDK11[@]}")
+          # default version
+          jdkHeadVersionArrBody=$(printf ",{\"spark-version\":\"%s\",\"java-version\":8}" "${SPARK_BASE_SHIM_VERSION}")
+          # jdk11
+          jdk11VersionArrBody=$(printf ",{\"spark-version\":\"%s\",\"java-version\":11}" "${SPARK_SHIM_VERSIONS_JDK11[@]}")
+          # jdk17
+          jdk17VersionArrBody=$(printf ",{\"spark-version\":\"%s\",\"java-version\":17}" "${SPARK_SHIM_VERSIONS_JDK17[@]}")
+          # jdk
+          jdkVersionArrBody=$jdkHeadVersionArrBody$jdk11VersionArrBody$jdk17VersionArrBody
           jdkVersionArrBody=${jdkVersionArrBody:1}
           jdkVersionJsonStr=$(printf {\"include\":[%s]} $jdkVersionArrBody)
-          echo "jdk11Versions=$jdkVersionJsonStr" >> $GITHUB_OUTPUT
+          echo "jdkVersions=$jdkVersionJsonStr" >> $GITHUB_OUTPUT
 
   package-tests:
     needs: get-shim-versions-from-dist
@@ -84,60 +95,27 @@ jobs:
           -pl integration_tests,tests -am
           -P 'individual,pre-merge'
           -Dbuildver=${{ matrix.spark-version }}
-          -DskipTests
-          -Dskip
-          -Dmaven.javadoc.skip
           -Dmaven.scalastyle.skip=true
           -Drat.skip=true
+          $COMMON_MVN_FLAGS
 
-
-  verify-all-modules-with-headSparkVersion:
-    needs: get-shim-versions-from-dist
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
-
-      - name: Setup Java and Maven Env
-        uses: actions/setup-java@v3
-        with:
-          distribution: adopt
-          java-version: 8
-
-      # includes RAT, code style and doc-gen checks of default shim
-      - name: verify all modules with lowest-supported Spark version
-        run: >
-          mvn -Dmaven.wagon.http.retryHandler.count=3 -B verify
-          -P 'individual,pre-merge'
-          -Dbuildver=${{ needs.get-shim-versions-from-dist.outputs.sparkHeadVersion }}
-          -DskipTests
-          -Dskip
-          -Dmaven.javadoc.skip
-
-  verify-modules-with-jdk11:
+  verify-all-modules:
     needs: get-shim-versions-from-dist
     runs-on: ubuntu-latest
     strategy:
-      matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkJDK11Versions) }}
+      matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkJDKVersions) }}
     steps:
       - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
 
       - name: Setup Java and Maven Env
         uses: actions/setup-java@v3
         with:
           distribution: adopt
-          java-version: 11
+          java-version: ${{ matrix.java-version }}
 
-      - name: Build JDK11
+      - name: Build JDK
         run: >
           mvn -Dmaven.wagon.http.retryHandler.count=3 -B verify
-          -P 'individual,pre-merge'
+          -P "individual,pre-merge,jdk${{ matrix.java-version }}"
           -Dbuildver=${{ matrix.spark-version }}
-          -Dmaven.compiler.release=11
-          -Dmaven.compiler.source=11
-          -Dmaven.compiler.target=11
-          -Dscala.plugin.version=4.6.1
-          -DallowConventionalDistJar=true
-          -DskipTests
-          -Dskip
-          -Dmaven.javadoc.skip
-          
+          $COMMON_MVN_FLAGS