Nessie / Iceberg #3143
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (C) 2021 The Authors of projectnessie | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: Nessie / Iceberg | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| schedule: | |
| # Run every 12 hours on week days | |
| - cron: '0 4/12 * * 1-5' | |
| workflow_dispatch: | |
| jobs: | |
| latest: | |
| name: Nessie main/Iceberg main | |
| runs-on: ubuntu-24.04 | |
| env: | |
| GIT_FETCH_DEPTH: 0 | |
| GIT_FETCH_OPTION: '' | |
| NESSIE_DIR: included-builds/nessie | |
| NESSIE_MAIN_REPOSITORY: projectnessie/nessie | |
| NESSIE_MAIN_BRANCH: main | |
| NESSIE_PATCH_REPOSITORY: '' | |
| NESSIE_PATCH_BRANCH: '' | |
| ICEBERG_DIR: included-builds/iceberg | |
| ICEBERG_MAIN_REPOSITORY: apache/iceberg | |
| ICEBERG_MAIN_BRANCH: main | |
| ICEBERG_PATCH_REPOSITORY: snazy/iceberg | |
| ICEBERG_PATCH_BRANCH: iceberg-nesqueit | |
| SPARK_LOCAL_IP: localhost | |
| steps: | |
| - name: Prepare Git | |
| run: | | |
| git config --global user.email "integrations-testing@projectnessie.org" | |
| git config --global user.name "Integrations Testing [Bot]" | |
| - name: Checkout Integrations Testing repo | |
| uses: actions/checkout@v4 | |
| - name: Free disk space | |
| uses: ./.github/actions/free-disk-space | |
| - name: Setup runner | |
| uses: ./.github/actions/setup-runner | |
| with: | |
| more-memory: 'true' | |
| - name: Checkout and patch Nessie | |
| uses: ./.github/actions/patch-git | |
| with: | |
| name: Nessie | |
| local-dir: ${{env.NESSIE_DIR}} | |
| main-repository: ${{env.NESSIE_MAIN_REPOSITORY}} | |
| main-branch: ${{env.NESSIE_MAIN_BRANCH}} | |
| patch-repository: ${{env.NESSIE_PATCH_REPOSITORY}} | |
| patch-branch: ${{env.NESSIE_PATCH_BRANCH}} | |
| work-branch: nessie_integtest | |
| - name: Checkout and patch Iceberg | |
| uses: ./.github/actions/patch-git | |
| with: | |
| name: Iceberg | |
| local-dir: ${{env.ICEBERG_DIR}} | |
| main-repository: ${{env.ICEBERG_MAIN_REPOSITORY}} | |
| main-branch: ${{env.ICEBERG_MAIN_BRANCH}} | |
| patch-repository: ${{env.ICEBERG_PATCH_REPOSITORY}} | |
| patch-branch: ${{env.ICEBERG_PATCH_BRANCH}} | |
| work-branch: iceberg_integtest | |
| # Setup Gradle properties, heap requirements are for the "Integration test w/ Nessie". | |
| - name: Setup gradle.properties | |
| run: | | |
| mkdir -p ~/.gradle | |
| echo "org.gradle.jvmargs=-Xms2g -Xmx4g -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8" >> ~/.gradle/gradle.properties | |
| echo "org.gradle.vfs.watch=false" >> ~/.gradle/gradle.properties | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| # Java 17 required for Nessie build, Java 11 required for Flink & Presto | |
| java-version: | | |
| 11 | |
| 17 | |
| 21 | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v4 | |
| with: | |
| cache-read-only: ${{ github.event_name != 'push' || github.ref != 'refs/heads/main' }} | |
| validate-wrappers: false | |
| - name: Spotless | |
| run: ./gradlew spotlessCheck --scan | |
| - name: Checkstyle | |
| run: ./gradlew checkstyleMain checkstyleTest --scan | |
| - name: Iceberg Nessie test | |
| run: ./gradlew :iceberg:iceberg-nessie:test --scan | |
| - name: Nessie Spark 3.3 / 2.12 Extensions test | |
| run: ./gradlew -DscalaVersion=2.12 :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:intTest --scan | |
| - name: Nessie Spark 3.4 / 2.12 Extensions test | |
| run: ./gradlew -DscalaVersion=2.13 :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.13:intTest --scan | |
| - name: Nessie Spark 3.5 / 2.13 Extensions test | |
| run: ./gradlew -DscalaVersion=2.13 :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:intTest --scan | |
| - name: Build before publish | |
| run: ./gradlew jar testClasses javadoc --scan | |
| # TODO re-enable once Iceberg/main references a Nessie release w/ Nessie PR #6197, | |
| # Gradle's Maven publishing fails with the group-ID relocation. | |
| # | |
| #- name: Publish Nessie + Iceberg to local Maven repo | |
| # run: ./gradlew publishLocal --scan | |
| - name: Gather locally published versions | |
| run: | | |
| NESSIE_VERSION="$(cat included-builds/nessie/version.txt)" | |
| ICEBERG_VERSION="$(cat included-builds/iceberg/build/iceberg-build.properties | grep '^git.build.version=' | cut -d= -f2)" | |
| echo "NESSIE_VERSION=${NESSIE_VERSION}" >> ${GITHUB_ENV} | |
| echo "ICEBERG_VERSION=${ICEBERG_VERSION}" >> ${GITHUB_ENV} | |
| cat <<! >> $GITHUB_STEP_SUMMARY | |
| ## Published versions | |
| | Published Nessie version | Published Iceberg version | | |
| | ------------------------ | ------------------------- | | |
| | ${NESSIE_VERSION} | ${ICEBERG_VERSION} | | |
| ! | |
| # Iceberg does not support builds with more than one Scala versions, so we have to explicitly | |
| # Specify the Scala version here - and run `intTest` for each Scala version. | |
| # But note: Flink does not provide any Scala 2.13 artifacts at the moment, so when selecting | |
| # Scala 2.13 with a build using Iceberg from source, all Flink related code will be skipped. | |
| - name: Show Gradle projects for Scala 2.12 | |
| run: ./gradlew projects -DscalaVersion=2.12 | |
| - name: Tools & Integrations tests / Scala 2.12 | |
| run: ./gradlew intTest -DscalaVersion=2.12 --scan | |
| - name: Show Gradle projects for Scala 2.13 | |
| run: ./gradlew projects -DscalaVersion=2.13 | |
| - name: Tools & Integrations tests / Scala 2.13 | |
| run: ./gradlew intTest -DscalaVersion=2.13 --scan | |
| #- name: Checkout Presto repo | |
| # uses: actions/checkout@v3 | |
| # with: | |
| # repository: ${{env.PRESTO_MAIN_REPOSITORY}} | |
| # ref: ${{env.PRESTO_MAIN_BRANCH}} | |
| # path: ${{env.PRESTO_DIR}} | |
| # | |
| #- name: Cache local Maven repository | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: | | |
| # ~/.m2/repository | |
| # !~/.m2/repository/org/projectnessie | |
| # !~/.m2/repository/org/apache/iceberg | |
| # ~/.m2/wrapper | |
| # key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml', '**/maven-wrapper.properties') }} | |
| # restore-keys: | | |
| # ${{ runner.os }}-maven- | |
| # | |
| #- name: Build Presto w/ latest Nessie+Iceberg | |
| # working-directory: included-builds/presto | |
| # run: | | |
| # ./mvnw \ | |
| # --no-transfer-progress \ | |
| # install -DskipTests \ | |
| # -Ddep.iceberg.version=${ICEBERG_VERSION} \ | |
| # -Ddep.nessie.version=${NESSIE_VERSION} | |
| iceberg160_nessie0931: | |
| name: Nessie 0.93.1 / Iceberg 1.6.1 | |
| runs-on: ubuntu-24.04 | |
| env: | |
| SPARK_LOCAL_IP: localhost | |
| steps: | |
| - name: Checkout Integrations Testing repo | |
| uses: actions/checkout@v4 | |
| - name: Free disk space | |
| uses: ./.github/actions/free-disk-space | |
| - name: Setup gradle.properties | |
| run: | | |
| mkdir -p ~/.gradle | |
| echo "org.gradle.jvmargs=-Xmx1024m -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8" >> ~/.gradle/gradle.properties | |
| echo "org.gradle.vfs.watch=false" >> ~/.gradle/gradle.properties | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| # Java 17 required for Nessie build, Java 11 required for Flink & Presto | |
| java-version: | | |
| 11 | |
| 17 | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v4 | |
| with: | |
| cache-read-only: true | |
| validate-wrappers: false | |
| - name: Show Gradle projects | |
| run: ./gradlew projects -Dnessie.versionNessie=0.93.1 -Dnessie.versionIceberg=1.6.1 | |
| - name: Tools & Integrations tests | |
| run: ./gradlew intTest -Dnessie.versionNessie=0.93.1 -Dnessie.versionIceberg=1.6.1 --scan | |
| iceberg160_nessie0790: | |
| name: Nessie 0.79.0 / Iceberg 1.5.2 | |
| runs-on: ubuntu-24.04 | |
| env: | |
| SPARK_LOCAL_IP: localhost | |
| steps: | |
| - name: Checkout Integrations Testing repo | |
| uses: actions/checkout@v4 | |
| - name: Free disk space | |
| uses: ./.github/actions/free-disk-space | |
| - name: Setup gradle.properties | |
| run: | | |
| mkdir -p ~/.gradle | |
| echo "org.gradle.jvmargs=-Xmx1024m -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8" >> ~/.gradle/gradle.properties | |
| echo "org.gradle.vfs.watch=false" >> ~/.gradle/gradle.properties | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| # Java 17 required for Nessie build, Java 11 required for Flink & Presto | |
| java-version: | | |
| 11 | |
| 17 | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v4 | |
| with: | |
| cache-read-only: true | |
| validate-wrappers: false | |
| - name: Show Gradle projects | |
| run: ./gradlew projects -Dnessie.versionNessie=0.79.0 -Dnessie.versionIceberg=1.5.2 | |
| - name: Tools & Integrations tests | |
| run: ./gradlew intTest -Dnessie.versionNessie=0.79.0 -Dnessie.versionIceberg=1.5.2 --scan | |
| iceberg143_nessie0780: | |
| name: Nessie 0.78.0 / Iceberg 1.4.3 | |
| runs-on: ubuntu-24.04 | |
| env: | |
| SPARK_LOCAL_IP: localhost | |
| steps: | |
| - name: Checkout Integrations Testing repo | |
| uses: actions/checkout@v4 | |
| - name: Free disk space | |
| uses: ./.github/actions/free-disk-space | |
| - name: Setup gradle.properties | |
| run: | | |
| mkdir -p ~/.gradle | |
| echo "org.gradle.jvmargs=-Xmx1024m -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8" >> ~/.gradle/gradle.properties | |
| echo "org.gradle.vfs.watch=false" >> ~/.gradle/gradle.properties | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| # Java 17 required for Nessie build, Java 11 required for Flink & Presto | |
| java-version: | | |
| 11 | |
| 17 | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v4 | |
| with: | |
| cache-read-only: true | |
| validate-wrappers: false | |
| - name: Show Gradle projects | |
| run: ./gradlew projects -Dnessie.versionNessie=0.78.0 -Dnessie.versionIceberg=1.4.3 | |
| - name: Tools & Integrations tests | |
| run: ./gradlew intTest -Dnessie.versionNessie=0.78.0 -Dnessie.versionIceberg=1.4.3 --scan |