From 0f32a43c7db2d744136de4eafb28786baa71e47c Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 31 Mar 2023 11:02:22 +0200 Subject: [PATCH] Refactor CI workflow Unifies most main + PR workflows into a single workflow. The unified CI workflow consists of 2 "stages": * Checks - test, intTest, NesQuEIT, etc * Finalize - a "success" dummy job + a "save to github-cache" job Utilizes the Gradle build cache for all stages. The updated build cache of the jobs in the checks stage are saved as artifacts (with the minimum retention period). The updated build cache is pushed back to GigHub's cache when the checks have successfully finished. Java CI runs against Java 11 and Java 17, where it is meaningful. (Spark + Deltalake tests always run against 11, so not run against Java 17.) Some checks also run against the latest Java version. Codecov was not added to the new workflow, it wasn't working for quite a while now or produced wrong results. Build logs and reports are not archived. Test results and relevant logs are available via Gradle build scans. Windows + macOS workflows are not included in the unified workflow. There is also another [Gradle cache action](https://github.com/burrunan/gradle-cache-action), which utilizes the GitHub's cache like a remote Gradle cache. However, that puts too much load (requests) against GitHub's cache, which in turn throttles our CI and responds with HTTP/429 (Too many requests). See [this issue](https://github.com/burrunan/gradle-cache-action/issues/66). Fixes #6365 --- .../ci-incr-build-cache-prepare/action.yml | 74 ++ .../ci-incr-build-cache-save/action.yml | 41 + .github/actions/ci-results/action.yml | 27 - .github/workflows/check-results-upload.yml | 2 +- .github/workflows/ci-mac.yml | 5 +- .github/workflows/ci-win.yml | 5 +- .github/workflows/ci.yml | 803 ++++++++++++++++++ .github/workflows/main.yml | 178 ---- .github/workflows/newer-java.yml | 2 +- .github/workflows/pull-request-docker.yml | 188 ---- .github/workflows/pull-request-helm.yml | 71 -- .github/workflows/pull-request-integ.yml | 151 ---- .github/workflows/pull-request-native.yml | 62 -- .github/workflows/pull-request.yml | 143 ---- .github/workflows/site.yml | 38 - README.md | 1 - build.gradle.kts | 2 + .../src/main/kotlin/ListChilrProjectsTask.kt | 55 ++ 18 files changed, 983 insertions(+), 865 deletions(-) create mode 100644 .github/actions/ci-incr-build-cache-prepare/action.yml create mode 100644 .github/actions/ci-incr-build-cache-save/action.yml delete mode 100644 .github/actions/ci-results/action.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/pull-request-docker.yml delete mode 100644 .github/workflows/pull-request-helm.yml delete mode 100644 .github/workflows/pull-request-integ.yml delete mode 100644 .github/workflows/pull-request-native.yml delete mode 100644 .github/workflows/pull-request.yml delete mode 100644 .github/workflows/site.yml create mode 100644 buildSrc/src/main/kotlin/ListChilrProjectsTask.kt diff --git a/.github/actions/ci-incr-build-cache-prepare/action.yml b/.github/actions/ci-incr-build-cache-prepare/action.yml new file mode 100644 index 00000000000..0e15f156573 --- /dev/null +++ b/.github/actions/ci-incr-build-cache-prepare/action.yml @@ -0,0 +1,74 @@ +name: 'Incremental Gradle build cache prepare' +description: 'Prepare to save incremental Gradle build cache' +inputs: + cache-read-only: + description: 'Gradle cache read only' + default: 'true' + java-version: + description: 'Java version' + default: '11' + job-id: + description: 'Job name to prefer' + default: 'nessie-ci' + job-instance: + description: 'Job instance to prefer' + default: 'ci' +runs: + using: "composite" + steps: + - name: Prep env + shell: bash + run: | + echo "GRADLE_BUILD_ACTION_CACHE_KEY_ENVIRONMENT=java-${{ inputs.java-version }}" >> ${GITHUB_ENV} + echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB=${{ inputs.job-id }}" >> ${GITHUB_ENV} + if [[ -n "${{ inputs.no-daemon }}" ]] ; then + echo "G_DAEMON_FLAG=--no-daemon" >> ${GITHUB_ENV} + fi + if [[ -n "${{ inputs.job-instance }}" ]] ; then + echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB_INSTANCE=${{ inputs.job-instance }}" >> ${GITHUB_ENV} + fi + + - name: Gradle / Init + uses: gradle/gradle-build-action@v2 + with: + cache-read-only: ${{ inputs.cache-read-only }} + arguments: -h + + - name: Download existing workflow artifacts + uses: actions/download-artifact@v3 + # Just in case, don't know the exact inner workings of Gradle's build cache and whether + # the download-action complains about duplicate files. + continue-on-error: true + with: + path: ~/downloaded-artifacts/ + + - name: Extract caches + shell: bash + run: | + echo "::group::Gradle build cache / add incremental updates" + mkdir -p ~/.gradle/caches/build-cache-1/ + echo "Gradle build-cache-1 contains $(ls -1 ~/.gradle/caches/build-cache-1/ | wc -l) files" + + if [[ -d ~/downloaded-artifacts/ ]] ; then + find ~/downloaded-artifacts/ -type f -name "ci-gradle-build-cache-*-${{ inputs.java-version }}.tar" | while read arch ; do + echo "Adding archive content from $arch ..." + (cd ~/.gradle/caches/build-cache-1/ ; tar xf $arch) + done + + else + echo "No previous build cache artifacts downloaded." + fi + echo "::endgroup::" + + - name: Memoize build-cache + shell: bash + run: | + echo "::group::Saving state of Gradle's build-cache-1 ..." + rm -rf ~/saved-build-cache-1/ + mkdir -p ~/saved-build-cache-1/ + + if [[ -d ~/.gradle/caches/build-cache-1/ ]] ; then + echo "Gradle build-cache-1 contains $(ls -1 ~/.gradle/caches/build-cache-1/ | wc -l) files" + (cd ~/.gradle/caches/build-cache-1/ ; cp -r . ~/saved-build-cache-1/) + fi + echo "::endgroup::" diff --git a/.github/actions/ci-incr-build-cache-save/action.yml b/.github/actions/ci-incr-build-cache-save/action.yml new file mode 100644 index 00000000000..149256bb683 --- /dev/null +++ b/.github/actions/ci-incr-build-cache-save/action.yml @@ -0,0 +1,41 @@ +name: 'Save incremental Gradle build cache' +description: 'Save incremental Gradle build cache' +inputs: + job-name: + description: 'job name' + java-version: + description: 'Java version' + default: '11' +runs: + using: "composite" + steps: + - name: Prepare build-cache archive + shell: bash + run: | + if [[ -d ~/.gradle/caches/build-cache-1/ ]] ; then + echo "::group::Gradle build cache / identify updated build cache items" + + cd ~/.gradle/caches/build-cache-1/ + + echo "Gradle build-cache-1 has $(ls -1 . | wc -l) files" + # Identify the added and changed files in build-cache-1. + + echo "Identifying changed/added files..." + # 'diff' returns 1, if files differ :( + (diff --brief --recursive --new-file --no-dereference ~/saved-build-cache-1/ . || true) | \ + cut -d\ -f4 > ~/ci-gradle-build-cache-diff + echo "Identified $(wc -l < ~/ci-gradle-build-cache-diff) changed/added files in build-cache-1" + + # Only call 'tar', if there is some difference + # Note: actions/upload-artifact takes care of compressing the artifact, no need to bug the CPU here + echo "Creating artifact (if necessary)..." + [[ -s ~/ci-gradle-build-cache-diff ]] && tar cf ~/ci-gradle-build-cache-${{ inputs.job-name }}-${{ inputs.java-version }}.tar -T ~/ci-gradle-build-cache-diff + echo "::endgroup::" + fi + - name: Archive code-checks incremental + uses: actions/upload-artifact@v3 + with: + name: ci-gradle-build-cache-${{ inputs.job-name }}-${{ inputs.java-version }} + path: ~/ci-gradle-build-cache-${{ inputs.job-name }}-${{ inputs.java-version }}.tar + if-no-files-found: ignore + retention-days: 1 diff --git a/.github/actions/ci-results/action.yml b/.github/actions/ci-results/action.yml deleted file mode 100644 index d5423e4597e..00000000000 --- a/.github/actions/ci-results/action.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: 'Capture coverage and gatling results' -description: 'Captures code coverage + gatling results' -runs: - using: "composite" - steps: - - name: Capture Gatling simulation logs - uses: actions/upload-artifact@v3 - with: - name: gatling-logs - path: perftest/simulations/build/reports/gatling/*/simulation.log - retention-days: 1 - - uses: codecov/codecov-action@v3 - if: github.repository_owner == 'projectnessie' - continue-on-error: true - with: - verbose: true - fail_ci_if_error: true - flags: java - files: 'build/reports/jacoco/*odeCoverageReport/*odeCoverageReport.xml' - - uses: codecov/codecov-action@v3 - if: github.repository_owner == 'projectnessie' - continue-on-error: true - with: - verbose: true - fail_ci_if_error: true - flags: javascript - files: ui/coverage/clover.xml,ui/coverage/lcov.info diff --git a/.github/workflows/check-results-upload.yml b/.github/workflows/check-results-upload.yml index 2cc539467ef..8677b558a77 100644 --- a/.github/workflows/check-results-upload.yml +++ b/.github/workflows/check-results-upload.yml @@ -12,7 +12,7 @@ name: Check Results Upload on: workflow_run: - workflows: ["PR Build Check", "Main CI"] + workflows: ["CI Build"] types: - completed diff --git a/.github/workflows/ci-mac.yml b/.github/workflows/ci-mac.yml index 66988b8c9ff..0ff3ff77114 100644 --- a/.github/workflows/ci-mac.yml +++ b/.github/workflows/ci-mac.yml @@ -92,6 +92,7 @@ jobs: - name: Gradle / compile uses: gradle/gradle-build-action@v2 with: + cache-read-only: true arguments: assemble --scan - name: Gradle / unit test @@ -123,9 +124,9 @@ jobs: env: working-directory: ./python strategy: - max-parallel: 4 + max-parallel: 1 matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10', '3.7'] steps: - uses: actions/checkout@v3.5.0 diff --git a/.github/workflows/ci-win.yml b/.github/workflows/ci-win.yml index 583ac31e9fb..5e3eceeacae 100644 --- a/.github/workflows/ci-win.yml +++ b/.github/workflows/ci-win.yml @@ -83,6 +83,7 @@ jobs: - name: Gradle / compile uses: gradle/gradle-build-action@v2 with: + cache-read-only: true arguments: assemble --scan - name: Gradle / unit test @@ -114,9 +115,9 @@ jobs: env: working-directory: ./python strategy: - max-parallel: 4 + max-parallel: 1 matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10', '3.7'] steps: - uses: actions/checkout@v3.5.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..ffcc351907f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,803 @@ +# Copyright (C) 2020 Dremio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Unifies main + PR workflow. +# +# The unified CI workflow consists of 4 "stages": +# * Prepare - this is just auto-labeling for PRs (or nothing for main) +# * Checks - test, intTest, NesQuEIT, etc +# * Finalize +# +# Utilizes the Gradle build cache for all stages. The updated build cache +# of the jobs in the checks stage are saved as artifacts (with the minimum +# retention period). The updated build cache is pushed back to GigHub's +# cache when the checks have successfully finished. +# +# Java CI runs against Java 11 and Java 17, where it is meaningful. +# (Spark + Deltalake tests always run against 11, so not run against +# Java 17.) + +name: CI build + +on: + push: + branches: + - main + - release-* + paths-ignore: + - 'LICENSE' + - 'NOTICE' + - '**.md' + - '!site/**' + - '.github/renovate.json5' + - '.github/workflows/release*.yml' + - '.github/workflows/check*.yml' + - '.idea/**' + - '.editorconfig' + pull_request: + types: [labeled, opened, synchronize, reopened] + +concurrency: + # PRs: 1 CI run concurrently + # main branch: 1 CI run per commit (so every commit is checked) + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.head_commit.id }}-${{ github.event.pull_request.head.ref }} + cancel-in-progress: true + +jobs: + code-checks: + name: CI Code Checks et al + runs-on: ubuntu-22.04 + strategy: + max-parallel: 2 + matrix: + java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( + steps: + - uses: actions/checkout@v3.5.0 + - name: Setup runner + uses: ./.github/actions/setup-runner + - name: Setup Java, Gradle + uses: ./.github/actions/dev-tool-java + with: + java-version: ${{ matrix.java-version }} + + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + with: + java-version: ${{ matrix.java-version }} + + - name: Gradle / Code Checks + uses: gradle/gradle-build-action@v2 + with: + arguments: spotlessCheck checkstyle --scan + + - name: Gradle / Assemble + uses: gradle/gradle-build-action@v2 + with: + arguments: assemble publishToMavenLocal --scan + + # This is a rather quick one and uses the output of 'publishToMavenLocal', which uses the + # outputs of 'assemble' + - name: Gradle / build tools integration tests + uses: gradle/gradle-build-action@v2 + with: + arguments: buildToolsIntegrationTest + + - name: Gatling + uses: gradle/gradle-build-action@v2 + with: + cache-read-only: true + arguments: gatlingRun + + - name: Capture Gatling simulation logs + uses: actions/upload-artifact@v3 + with: + name: gatling-logs + path: perftest/simulations/build/reports/gatling/*/simulation.log + retention-days: 1 + + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + with: + job-name: 'code-checks' + java-version: ${{ matrix.java-version }} + + test: + name: CI Test + runs-on: ubuntu-22.04 + strategy: + max-parallel: 2 + matrix: + java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( + steps: + - uses: actions/checkout@v3.5.0 + - name: Setup runner + uses: ./.github/actions/setup-runner + - name: Setup Java, Gradle + uses: ./.github/actions/dev-tool-java + with: + java-version: ${{ matrix.java-version }} + + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + with: + java-version: ${{ matrix.java-version }} + + - name: Gradle / test + uses: gradle/gradle-build-action@v2 + with: + arguments: test -x :nessie-quarkus:test --scan + + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + with: + job-name: 'test' + java-version: ${{ matrix.java-version }} + + test-quarkus: + name: CI Test Quarkus + runs-on: ubuntu-22.04 + strategy: + max-parallel: 2 + matrix: + java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( + steps: + - uses: actions/checkout@v3.5.0 + - name: Setup runner + uses: ./.github/actions/setup-runner + - name: Setup Java, Gradle + uses: ./.github/actions/dev-tool-java + with: + java-version: ${{ matrix.java-version }} + + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + with: + java-version: ${{ matrix.java-version }} + + - name: Gradle / Test Quarkus + uses: gradle/gradle-build-action@v2 + with: + arguments: :nessie-quarkus:test --scan + + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + with: + job-name: 'test-quarkus' + java-version: ${{ matrix.java-version }} + + int-test: + name: CI intTest + runs-on: ubuntu-22.04 + strategy: + max-parallel: 2 + matrix: + java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( + env: + SPARK_LOCAL_IP: localhost + steps: + - uses: actions/checkout@v3.5.0 + - name: Setup runner + uses: ./.github/actions/setup-runner + - name: Setup Java, Gradle + uses: ./.github/actions/dev-tool-java + with: + java-version: ${{ matrix.java-version }} + + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + + - name: Gradle / intTest + run: | + ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-persist- --output ../persist-prjs.txt --exclude + ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-storage- --output ../storage-prjs.txt --exclude + ./gradlew :listProjectsWithPrefix --prefix :nessie-spark-ext --output ../spark-prjs.txt --exclude + ./gradlew intTest \ + -x :nessie-quarkus:intTest \ + -x :nessie-quarkus-cli:intTest \ + $(cat ../persist-prjs.txt) \ + $(cat ../storage-prjs.txt) \ + -x :nessie-deltalake:intTest \ + $(cat ../spark-prjs.txt) \ + --scan + + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + with: + job-name: 'int-test' + java-version: ${{ matrix.java-version }} + + int-test-stores: + name: CI intTest versioned/stores + runs-on: ubuntu-22.04 + strategy: + max-parallel: 2 + matrix: + java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( + steps: + - uses: actions/checkout@v3.5.0 + - name: Setup runner + uses: ./.github/actions/setup-runner + - name: Setup Java, Gradle + uses: ./.github/actions/dev-tool-java + with: + java-version: ${{ matrix.java-version }} + + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + with: + java-version: ${{ matrix.java-version }} + + - name: Gradle / intTest versioned/stores + run: | + ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-storage- --output ../storage-prjs.txt + ./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-persist- --output ../persist-prjs.txt + ./gradlew $(cat ../persist-prjs.txt) $(cat ../storage-prjs.txt) --scan + + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + with: + job-name: 'int-test-stores' + java-version: ${{ matrix.java-version }} + + int-test-integrations: + name: CI intTest integrations + runs-on: ubuntu-22.04 + env: + SPARK_LOCAL_IP: localhost + steps: + - uses: actions/checkout@v3.5.0 + - name: Setup runner + uses: ./.github/actions/setup-runner + - name: Setup Java, Gradle + uses: ./.github/actions/dev-tool-java + + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + + - name: Gradle / intTest integrations + run: | + ./gradlew :listProjectsWithPrefix --prefix :nessie-spark-ext --output ../spark-prjs.txt + ./gradlew :nessie-deltalake:intTest $(cat ../spark-prjs.txt) --scan + + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + with: + job-name: 'int-test-integrations' + + int-test-quarkus: + name: CI intTest Quarkus + runs-on: ubuntu-22.04 + strategy: + max-parallel: 2 + matrix: + java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( + steps: + - uses: actions/checkout@v3.5.0 + - name: Setup runner + uses: ./.github/actions/setup-runner + - name: Setup Java, Gradle + uses: ./.github/actions/dev-tool-java + with: + java-version: ${{ matrix.java-version }} + + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + with: + java-version: ${{ matrix.java-version }} + + - name: Gradle / intTest Quarkus + uses: gradle/gradle-build-action@v2 + with: + arguments: :nessie-quarkus:intTest :nessie-quarkus-cli:intTest --scan + + - name: Save partial Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-save + with: + job-name: 'int-test-quarkus' + java-version: ${{ matrix.java-version }} + + native-quarkus: + name: CI intTest Quarkus native + runs-on: ubuntu-22.04 + strategy: + max-parallel: 2 + matrix: + java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( + env: + WF_EXEC: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'pr-native') }} + steps: + - uses: actions/checkout@v3.5.0 + if: env.WF_EXEC == 'true' + - name: Setup runner + if: env.WF_EXEC == 'true' + uses: ./.github/actions/setup-runner + with: + more-memory: 'true' + - name: Setup Java, Gradle + if: env.WF_EXEC == 'true' + uses: ./.github/actions/dev-tool-java + with: + java-version: ${{ matrix.java-version }} + + - name: Prepare Gradle build cache + if: env.WF_EXEC == 'true' + uses: ./.github/actions/ci-incr-build-cache-prepare + with: + java-version: ${{ matrix.java-version }} + + - name: Gradle / intTest Quarkus native + if: env.WF_EXEC == 'true' + uses: gradle/gradle-build-action@v2 + with: + arguments: :nessie-quarkus:intTest -Pnative --scan + + # A "native" build doesn't produce any meaningful build-cache artifacts that would not + # be produced by other jobs in this workflow. + + docker-testing: + name: CI Docker build and publishing + runs-on: ubuntu-22.04 + if: github.event_name == 'pull_request' + env: + # Only run for PRs with the appropriate label + WF_EXEC: ${{ contains(github.event.pull_request.labels.*.name, 'pr-docker') }} + steps: + - name: Checkout + if: env.WF_EXEC == 'true' + uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + - name: Setup runner + if: env.WF_EXEC == 'true' + uses: ./.github/actions/setup-runner + with: + more-memory: 'true' + - name: Setup Java, Gradle + if: env.WF_EXEC == 'true' + uses: ./.github/actions/dev-tool-java + + - name: Setup docker-registry + if: env.WF_EXEC == 'true' + run: | + sudo apt-get install -y docker-registry apache2-utils + cat < config.yml + version: 0.1 + log: + fields: + service: registry + storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/docker-registry + delete: + enabled: true + http: + addr: 127.0.0.1:5000 + headers: + X-Content-Type-Options: [nosniff] + auth: + htpasswd: + realm: basic-realm + path: /etc/docker/registry/htpasswd + health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 + ! + sudo mv config.yml /etc/docker/registry/config.yml + sudo htpasswd -cBb /etc/docker/registry/htpasswd micky mouse + + sudo service docker-registry restart + + echo mouse | docker login -u micky --password-stdin localhost:5000 + + VERSION="$(cat version.txt)" + DOCKER_VERSION="${VERSION%-SNAPSHOT}" + echo "DOCKER_VERSION=${DOCKER_VERSION}" >> ${GITHUB_ENV} + echo "DOCKER_IMAGE=localhost:5000/nessie-testing" >> ${GITHUB_ENV} + + - name: Prepare Gradle build cache + if: env.WF_EXEC == 'true' + uses: ./.github/actions/ci-incr-build-cache-prepare + + - name: Docker images publishing + if: env.WF_EXEC == 'true' + env: + ARTIFACTS: ../build-artifacts + run: | + rm -rf "${ARTIFACTS}" + mkdir -p "${ARTIFACTS}" + + tools/dockerbuild/build-push-images.sh \ + -a "${ARTIFACTS}" \ + -g ":nessie-quarkus" \ + -p "servers/quarkus-server" \ + -n \ + ${DOCKER_IMAGE} + rm -rf "${ARTIFACTS}" + + - name: Cleanup buildx + if: env.WF_EXEC == 'true' + run: | + docker buildx use default + docker buildx rm nessiebuild + + - name: Docker images exist test + if: env.WF_EXEC == 'true' + run: | + docker pull ${DOCKER_IMAGE}:latest + docker pull ${DOCKER_IMAGE}:latest-java + docker pull ${DOCKER_IMAGE}:latest-native + docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION} + docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}-native + docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}-java + cat <> $GITHUB_STEP_SUMMARY + ## Docker images + + \`\`\` + $(docker images) + \`\`\` + ! + + - name: Check if Docker native image works + if: env.WF_EXEC == 'true' + run: | + docker run --detach --name nessie ${DOCKER_IMAGE}:latest-native + echo "Let native Nessie Docker image run for one minute (to make sure it starts up fine)..." + for i in {1..60}; do + STATUS="$(docker container inspect nessie | jq -r '.[0].State.Status')" + if [[ ${STATUS} != "running" ]] ; then + echo "Nessie native Docker image stopped on its own ... a bug?" > /dev/stderr + docker logs nessie + cat <> $GITHUB_STEP_SUMMARY + ## Nessie native Docker image FAILED + + \`\`\` + $(docker logs nessie) + \`\`\` + ! + exit 1 + fi + sleep 1 + done + echo "## Nessie native Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY + echo "Nessie native Docker image smoke test: PASSED" + docker stop nessie + docker rm nessie + + - name: Check if Docker Java image works + if: env.WF_EXEC == 'true' + run: | + docker run --detach --name nessie ${DOCKER_IMAGE}:latest-java + echo "Let Nessie Java Docker image run for one minute (to make sure it starts up fine)..." + for i in {1..60}; do + STATUS="$(docker container inspect nessie | jq -r '.[0].State.Status')" + if [[ ${STATUS} != "running" ]] ; then + echo "Nessie Java Docker image stopped on its own ... a bug?" > /dev/stderr + docker logs nessie + cat <> $GITHUB_STEP_SUMMARY + ## Nessie Java Docker image FAILED + + \`\`\` + $(docker logs nessie) + \`\`\` + ! + exit 1 + fi + sleep 1 + done + echo "## Nessie Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY + echo "Nessie Java Docker image smoke test: PASSED" + docker stop nessie + docker rm nessie + + nesqueit: + name: CI NesQuEIT + runs-on: ubuntu-22.04 + if: github.event_name == 'pull_request' + env: + # Only run NesQuEIT tests for PRs, if requested. This job can easily run for 30+ minutes. + WF_EXEC: ${{ contains(github.event.pull_request.labels.*.name, 'pr-integrations') }} + NESSIE_DIR: included-builds/nessie + NESSIE_PATCH_REPOSITORY: '' + NESSIE_PATCH_BRANCH: '' + NESQUEIT_REPOSITORY: projectnessie/query-engine-integration-tests + NESQUEIT_BRANCH: main + ICEBERG_DIR: included-builds/iceberg + ICEBERG_MAIN_REPOSITORY: apache/iceberg + ICEBERG_MAIN_BRANCH: master + ICEBERG_PATCH_REPOSITORY: snazy/iceberg + ICEBERG_PATCH_BRANCH: iceberg-nesqueit + SPARK_LOCAL_IP: localhost + steps: + - name: Prepare Git + if: env.WF_EXEC == 'true' + run: | + git config --global user.email "integrations-testing@projectnessie.org" + git config --global user.name "Integrations Testing [Bot]" + + - name: Checkout NeQuEIT repo + if: env.WF_EXEC == 'true' + uses: actions/checkout@v3.5.0 + with: + repository: ${{env.NESQUEIT_REPOSITORY}} + ref: ${{env.NESQUEIT_BRANCH}} + + - name: Setup runner + if: env.WF_EXEC == 'true' + uses: ./.github/actions/setup-runner + with: + more-memory: 'true' + + - name: Checkout and patch Nessie PR + if: env.WF_EXEC == 'true' + uses: ./.github/actions/patch-git + with: + name: Nessie + local-dir: ${{env.NESSIE_DIR}} + main-repository: ${{ env.GITHUB_REPOSITORY }} + patch-repository: ${{env.NESSIE_PATCH_REPOSITORY}} + patch-branch: ${{env.NESSIE_PATCH_BRANCH}} + work-branch: nessie-integration-patched + + - name: Checkout and patch Iceberg + if: env.WF_EXEC == 'true' + uses: ./.github/actions/patch-git + with: + name: Nessie + local-dir: ${{env.ICEBERG_DIR}} + main-repository: ${{env.ICEBERG_MAIN_REPOSITORY}} + main-branch: ${{env.ICEBERG_MAIN_BRANCH}} + patch-repository: ${{env.ICEBERG_PATCH_REPOSITORY}} + patch-branch: ${{env.ICEBERG_PATCH_BRANCH}} + work-branch: iceberg-integration-patched + + # Setup Gradle properties, heap requirements are for the "Integration test w/ Nessie". + - name: Setup gradle.properties + if: env.WF_EXEC == 'true' + run: | + mkdir -p ~/.gradle + echo "org.gradle.jvmargs=-Xms2g -Xmx4g -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8" >> ~/.gradle/gradle.properties + echo "org.gradle.vfs.watch=false" >> ~/.gradle/gradle.properties + + - name: Set up JDK ${{ matrix.java-version }} + if: env.WF_EXEC == 'true' + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 11 + + - name: Iceberg Nessie test + if: env.WF_EXEC == 'true' + uses: gradle/gradle-build-action@v2 + with: + cache-read-only: true + arguments: :iceberg:iceberg-nessie:test --scan + + - name: Nessie Spark 3.1 Extensions test + if: env.WF_EXEC == 'true' + uses: gradle/gradle-build-action@v2 + with: + arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.1_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.1_2.12:intTest --scan + + - name: Nessie Spark 3.2 / 2.12 Extensions test + if: env.WF_EXEC == 'true' + uses: gradle/gradle-build-action@v2 + with: + arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.2_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.2_2.12:intTest --scan + + - name: Nessie Spark 3.3 / 2.12 Extensions test + if: env.WF_EXEC == 'true' + uses: gradle/gradle-build-action@v2 + with: + arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:intTest --scan + + - name: Publish Nessie + Iceberg to local Maven repo + if: env.WF_EXEC == 'true' + uses: gradle/gradle-build-action@v2 + with: + arguments: publishLocal --scan + + - name: Gather locally published versions + if: env.WF_EXEC == 'true' + run: | + NESSIE_VERSION="$(cat included-builds/nessie/version.txt)" + ICEBERG_VERSION="$(cat included-builds/iceberg/build/iceberg-build.properties | grep '^git.build.version=' | cut -d= -f2)" + echo "NESSIE_VERSION=${NESSIE_VERSION}" >> ${GITHUB_ENV} + echo "ICEBERG_VERSION=${ICEBERG_VERSION}" >> ${GITHUB_ENV} + cat <> $GITHUB_STEP_SUMMARY + ## Published versions + | Published Nessie version | Published Iceberg version | + | ------------------------ | ------------------------- | + | ${NESSIE_VERSION} | ${ICEBERG_VERSION} | + ! + + - name: Tools & Integrations tests + if: env.WF_EXEC == 'true' + uses: gradle/gradle-build-action@v2 + with: + arguments: intTest --scan + + helm-testing: + name: CI Lint & Test Helm chart + runs-on: ubuntu-22.04 + env: + WF_EXEC: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'pr-helm') }} + steps: + - name: Checkout + if: env.WF_EXEC == 'true' + uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + - name: Set up Helm + if: env.WF_EXEC == 'true' + uses: azure/setup-helm@v3 + with: + version: v3.8.1 + - uses: actions/setup-python@v4 + if: env.WF_EXEC == 'true' + with: + python-version: '3.8' + - name: Set up chart-testing + if: env.WF_EXEC == 'true' + uses: helm/chart-testing-action@v2.4.0 + + - name: Run chart-testing (list-changed) + if: env.WF_EXEC == 'true' + id: list-changed + run: | + ct list-changed --target-branch ${{ github.event.repository.default_branch }} + + - name: Run chart-testing (lint) + if: env.WF_EXEC == 'true' + run: ct lint --debug --charts ./helm/nessie + + - name: Set up & Start Minikube + if: env.WF_EXEC == 'true' + uses: medyagh/setup-minikube@v0.0.12 + with: + cache: false + + - name: Show pods + if: env.WF_EXEC == 'true' + run: kubectl get pods -A + + - name: Run chart-testing (install) + if: env.WF_EXEC == 'true' + run: ct install --debug --charts ./helm/nessie + + python: + name: CI Python + runs-on: ubuntu-22.04 + strategy: + # Python CI is relatively quick, don't parallelize to reduce number of concurrent jobs, + # respecting GitHub's concurrent jobs limit. + max-parallel: 1 + matrix: + python-version: ['3.8', '3.9', '3.10', '3.7'] # 3.8 first, it has "all the tox envs" + steps: + - uses: actions/checkout@v3.5.0 + - name: Setup Python + uses: ./.github/actions/dev-tool-python + with: + python-version: ${{ matrix.python-version }} + - name: Test with tox + run: tox + working-directory: ./python + + site: + name: CI Website + runs-on: ubuntu-22.04 + if: github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v3.5.0 + - name: Setup Python + uses: ./.github/actions/dev-tool-python + with: + python-version: '3.8' + - name: Upgrade pip + run: | + # install pip=>20.1 to use "pip cache dir" + python3 -m pip install --upgrade pip + - name: Install dependencies + run: python3 -m pip install -r ./site/requirements.txt + - name: Check code block indentation + run: python3 check_code_indent.py + working-directory: ./site + - name: Generate Static Site + run: mkdocs build + working-directory: ./site + - name: Deploy Static Site to GitHub + if: github.event_name == 'push' && github.repository_owner == 'projectnessie' + uses: peaceiris/actions-gh-pages@v3 + with: + external_repository: projectnessie/projectnessie.github.io + publish_branch: main + deploy_key: ${{ secrets.NESSIE_SITE_DEPLOY_KEY }} + publish_dir: ./site/site + cname: projectnessie.org + + store-cache: + # Store the Gradle cache to GH cache as soon as all relevant Nessie/Gradle jobs have finished. + name: CI Store Cache + runs-on: ubuntu-22.04 + if: github.event_name == 'push' + needs: + # Only include jobs that use Nessie's Gradle cache, especially excluding NesQuEIT, which + # is a "very special" citizen and also not run for "main" CI, and Quarkus/native, which does + # not add anything to the Gradle cache that's not already produced by other jobs. + - code-checks + - test + - test-quarkus + - int-test + - int-test-stores + - int-test-integrations + - int-test-quarkus + strategy: + max-parallel: 2 + matrix: + java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( + steps: + - uses: actions/checkout@v3.5.0 + - name: Setup runner + uses: ./.github/actions/setup-runner + - name: Setup Java, Gradle + uses: ./.github/actions/dev-tool-java + with: + java-version: ${{ matrix.java-version }} + + - name: Collect partial Gradle build caches + uses: ./.github/actions/ci-incr-build-cache-prepare + with: + java-version: ${{ matrix.java-version }} + cache-read-only: false + + - name: Trigger Gradle home cleanup + run: ./gradlew --no-daemon :showVersion + + # Note: the "Post Gradle invocation" archives the updated build cache. + + finish: + # 'Ci Success' is the (only) required check for PRs, so we can change the jobs above without + # having to update the GH configuration every time. + name: CI Success + runs-on: ubuntu-22.04 + if: github.event_name == 'pull_request' + needs: + - code-checks + - test + - test-quarkus + - int-test + - int-test-stores + - int-test-integrations + - int-test-quarkus + - native-quarkus + - python + - docker-testing + - nesqueit + # Note: help-testing can run into errors, when the recently released Nessie version is + # not yet available - so: Nessie release commits pushed, but Helm chart not yet built and + # published. + - helm-testing + - site + strategy: + max-parallel: 2 + matrix: + java-version: ['11'] # Ideally also '17', but GH concurrent job limit ... :( + steps: + # Intentionally empty job (for all GH WF) events so that the "required checks" setting + # only needs to contain this job as the only required check for PRs. + - run: echo "Success" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 77c9fa09917..00000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,178 +0,0 @@ -name: Main CI - -on: - push: - branches: - - main - - release-* - paths-ignore: - - 'LICENSE' - - 'NOTICE' - - '**.md' - - '!site/**' - - '.github/renovate.json5' - - '.github/workflows/release*.yml' - - '.github/workflows/check*.yml' - - '.idea/**' - - '.editorconfig' - -jobs: - java: - name: Java/Gradle - runs-on: ubuntu-22.04 - env: - SPARK_LOCAL_IP: localhost - MAVEN_USERNAME: ${{ secrets.OSSRH_ACCESS_ID }} - - steps: - - uses: actions/checkout@v3.5.0 - - name: Setup runner - uses: ./.github/actions/setup-runner - - name: Setup Java, Gradle - uses: ./.github/actions/dev-tool-java - with: - gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} - - - name: Gradle / compile - uses: gradle/gradle-build-action@v2 - with: - gradle-home-cache-cleanup: true - arguments: spotlessCheck checkstyle assemble --scan - - - name: Gradle / unit test - uses: gradle/gradle-build-action@v2 - if: github.repository_owner == 'projectnessie' - with: - arguments: test testCodeCoverageReport --scan - - - name: Gradle / check incl. integ-test - uses: gradle/gradle-build-action@v2 - with: - arguments: check -x test intTestCodeCoverageReport --scan - - - name: Gradle / Gatling simulations - uses: gradle/gradle-build-action@v2 - with: - arguments: gatlingRun - - - name: Gradle / assemble + publish local - uses: gradle/gradle-build-action@v2 - with: - arguments: assemble publishToMavenLocal --scan - - - name: Gradle / build tools integration tests - uses: gradle/gradle-build-action@v2 - with: - arguments: buildToolsIntegrationTest - - - name: Capture Test Reports - uses: actions/upload-artifact@v3 - if: ${{ failure() }} - with: - name: test-results - path: | - **/build/reports/* - **/build/test-results/* - retention-days: 7 - - - name: Capture Results - uses: ./.github/actions/ci-results - - native: - name: Java/Gradle/Native - runs-on: ubuntu-22.04 - if: github.repository_owner == 'projectnessie' - - steps: - - uses: actions/checkout@v3.5.0 - - name: Setup runner - uses: ./.github/actions/setup-runner - with: - more-memory: 'true' - - name: Setup Java, Gradle - uses: ./.github/actions/dev-tool-java - - - name: Gradle / integration test native - uses: gradle/gradle-build-action@v2 - with: - cache-read-only: true - arguments: | - --no-daemon - :nessie-quarkus:quarkusBuild - :nessie-quarkus:intTest - -Pnative - --scan - - - name: Capture Test Reports - uses: actions/upload-artifact@v3 - if: ${{ failure() }} - with: - name: test-results-native - path: | - **/build/reports/* - **/build/test-results/* - retention-days: 7 - - python: - name: Python - runs-on: ubuntu-22.04 - env: - working-directory: ./python - strategy: - max-parallel: 4 - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - - steps: - - uses: actions/checkout@v3.5.0 - - name: Setup Python - uses: ./.github/actions/dev-tool-python - with: - python-version: ${{ matrix.python-version }} - - name: Test with tox - run: tox - working-directory: ${{env.working-directory}} - - uses: codecov/codecov-action@v3 - if: github.repository_owner == 'projectnessie' - continue-on-error: true - with: - verbose: true - fail_ci_if_error: true - flags: python - - helm-testing: - name: Lint & Test Helm chart - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v3.5.0 - with: - fetch-depth: 0 - - name: Set up Helm - uses: azure/setup-helm@v3 - with: - version: v3.8.1 - - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - name: Set up chart-testing - uses: helm/chart-testing-action@v2.4.0 - - - name: Run chart-testing (list-changed) - id: list-changed - run: | - ct list-changed --target-branch ${{ github.event.repository.default_branch }} - - - name: Run chart-testing (lint) - run: ct lint --debug --charts ./helm/nessie - - - name: Set up & Start Minikube - uses: medyagh/setup-minikube@v0.0.12 - with: - cache: false - - - name: Show pods - run: kubectl get pods -A - - - name: Run chart-testing (install) - run: ct install --debug --charts ./helm/nessie diff --git a/.github/workflows/newer-java.yml b/.github/workflows/newer-java.yml index f368cb31c79..c35a6b17036 100644 --- a/.github/workflows/newer-java.yml +++ b/.github/workflows/newer-java.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-22.04 if: github.repository_owner == 'projectnessie' strategy: - max-parallel: 4 + max-parallel: 1 matrix: include: - java-version: 17 diff --git a/.github/workflows/pull-request-docker.yml b/.github/workflows/pull-request-docker.yml deleted file mode 100644 index 2e77dd68771..00000000000 --- a/.github/workflows/pull-request-docker.yml +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (C) 2020 Dremio -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Projectnessie GitHub Pull-Request / Docker build and publishing - -name: Docker images test - -on: - pull_request: - types: [labeled, opened, synchronize, reopened] - schedule: - # Run daily on week days - - cron: '0 4 * * 1-5' - workflow_dispatch: - -jobs: - docker-testing: - name: Docker build and publishing - runs-on: ubuntu-22.04 - env: - WF_EXEC: ${{ (github.repository == 'projectnessie/nessie' && github.event_name != 'pull_request') || contains(github.event.pull_request.labels.*.name, 'pr-docker') }} - steps: - - name: Checkout - if: env.WF_EXEC == 'true' - uses: actions/checkout@v3.5.0 - with: - fetch-depth: 0 - - name: Setup runner - if: env.WF_EXEC == 'true' - uses: ./.github/actions/setup-runner - with: - more-memory: 'true' - - name: Setup Java, Gradle - if: env.WF_EXEC == 'true' - uses: ./.github/actions/dev-tool-java - - - name: Setup docker-registry - if: env.WF_EXEC == 'true' - run: | - sudo apt-get install -y docker-registry apache2-utils - cat < config.yml - version: 0.1 - log: - fields: - service: registry - storage: - cache: - blobdescriptor: inmemory - filesystem: - rootdirectory: /var/lib/docker-registry - delete: - enabled: true - http: - addr: 127.0.0.1:5000 - headers: - X-Content-Type-Options: [nosniff] - auth: - htpasswd: - realm: basic-realm - path: /etc/docker/registry/htpasswd - health: - storagedriver: - enabled: true - interval: 10s - threshold: 3 - ! - sudo mv config.yml /etc/docker/registry/config.yml - sudo htpasswd -cBb /etc/docker/registry/htpasswd micky mouse - - sudo service docker-registry restart - - echo mouse | docker login -u micky --password-stdin localhost:5000 - - VERSION="$(cat version.txt)" - DOCKER_VERSION="${VERSION%-SNAPSHOT}" - echo "DOCKER_VERSION=${DOCKER_VERSION}" >> ${GITHUB_ENV} - echo "DOCKER_IMAGE=localhost:5000/nessie-testing" >> ${GITHUB_ENV} - - - name: Gradle / prepare - if: env.WF_EXEC == 'true' - uses: gradle/gradle-build-action@v2 - with: - cache-read-only: true - # Just fetch cache, defer build to "later" - arguments: projects - - - name: Docker images publishing - if: env.WF_EXEC == 'true' - env: - ARTIFACTS: ../build-artifacts - run: | - rm -rf "${ARTIFACTS}" - mkdir -p "${ARTIFACTS}" - - tools/dockerbuild/build-push-images.sh \ - -a "${ARTIFACTS}" \ - -g ":nessie-quarkus" \ - -p "servers/quarkus-server" \ - -n \ - ${DOCKER_IMAGE} - rm -rf "${ARTIFACTS}" - - - name: Cleanup buildx - if: env.WF_EXEC == 'true' - run: | - docker buildx use default - docker buildx rm nessiebuild - - - name: Docker images exist test - if: env.WF_EXEC == 'true' - run: | - docker pull ${DOCKER_IMAGE}:latest - docker pull ${DOCKER_IMAGE}:latest-java - docker pull ${DOCKER_IMAGE}:latest-native - docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION} - docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}-native - docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}-java - cat <> $GITHUB_STEP_SUMMARY - ## Docker images - - \`\`\` - $(docker images) - \`\`\` - ! - - - name: Check if Docker native image works - if: env.WF_EXEC == 'true' - run: | - docker run --detach --name nessie ${DOCKER_IMAGE}:latest-native - echo "Let native Nessie Docker image run for one minute (to make sure it starts up fine)..." - for i in {1..60}; do - STATUS="$(docker container inspect nessie | jq -r '.[0].State.Status')" - if [[ ${STATUS} != "running" ]] ; then - echo "Nessie native Docker image stopped on its own ... a bug?" > /dev/stderr - docker logs nessie - cat <> $GITHUB_STEP_SUMMARY - ## Nessie native Docker image FAILED - - \`\`\` - $(docker logs nessie) - \`\`\` - ! - exit 1 - fi - sleep 1 - done - echo "## Nessie native Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY - echo "Nessie native Docker image smoke test: PASSED" - docker stop nessie - docker rm nessie - - - name: Check if Docker Java image works - if: env.WF_EXEC == 'true' - run: | - docker run --detach --name nessie ${DOCKER_IMAGE}:latest-java - echo "Let Nessie Java Docker image run for one minute (to make sure it starts up fine)..." - for i in {1..60}; do - STATUS="$(docker container inspect nessie | jq -r '.[0].State.Status')" - if [[ ${STATUS} != "running" ]] ; then - echo "Nessie Java Docker image stopped on its own ... a bug?" > /dev/stderr - docker logs nessie - cat <> $GITHUB_STEP_SUMMARY - ## Nessie Java Docker image FAILED - - \`\`\` - $(docker logs nessie) - \`\`\` - ! - exit 1 - fi - sleep 1 - done - echo "## Nessie Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY - echo "Nessie Java Docker image smoke test: PASSED" - docker stop nessie - docker rm nessie diff --git a/.github/workflows/pull-request-helm.yml b/.github/workflows/pull-request-helm.yml deleted file mode 100644 index cc4aaf0a600..00000000000 --- a/.github/workflows/pull-request-helm.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (C) 2020 Dremio -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Projectnessie GitHub Pull-Request / Helm - -name: PR Helm - -on: - pull_request: - types: [labeled, opened, synchronize, reopened] - -jobs: - helm-testing: - name: Lint & Test Helm chart - runs-on: ubuntu-22.04 - env: - WF_EXEC: ${{ contains(github.event.pull_request.labels.*.name, 'pr-helm') }} - steps: - - name: Checkout - if: env.WF_EXEC == 'true' - uses: actions/checkout@v3.5.0 - with: - fetch-depth: 0 - - name: Set up Helm - if: env.WF_EXEC == 'true' - uses: azure/setup-helm@v3 - with: - version: v3.8.1 - - uses: actions/setup-python@v4 - if: env.WF_EXEC == 'true' - with: - python-version: '3.8' - - name: Set up chart-testing - if: env.WF_EXEC == 'true' - uses: helm/chart-testing-action@v2.4.0 - - - name: Run chart-testing (list-changed) - if: env.WF_EXEC == 'true' - id: list-changed - run: | - ct list-changed --target-branch ${{ github.event.repository.default_branch }} - - - name: Run chart-testing (lint) - if: env.WF_EXEC == 'true' - run: ct lint --debug --charts ./helm/nessie - - - name: Set up & Start Minikube - if: env.WF_EXEC == 'true' - uses: medyagh/setup-minikube@v0.0.12 - with: - cache: false - - - name: Show pods - if: env.WF_EXEC == 'true' - run: kubectl get pods -A - - - name: Run chart-testing (install) - if: env.WF_EXEC == 'true' - run: ct install --debug --charts ./helm/nessie diff --git a/.github/workflows/pull-request-integ.yml b/.github/workflows/pull-request-integ.yml deleted file mode 100644 index 168c3325d75..00000000000 --- a/.github/workflows/pull-request-integ.yml +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright (C) 2020 Dremio -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -name: PR Integrations Tests - -on: - pull_request: - types: [labeled, opened, synchronize, reopened] - -jobs: - java: - name: Integrations Tests - runs-on: ubuntu-22.04 - env: - WF_EXEC: ${{ contains(github.event.pull_request.labels.*.name, 'pr-integrations') }} - NESSIE_DIR: included-builds/nessie - NESSIE_PATCH_REPOSITORY: '' - NESSIE_PATCH_BRANCH: '' - NESQUEIT_REPOSITORY: projectnessie/query-engine-integration-tests - NESQUEIT_BRANCH: main - ICEBERG_DIR: included-builds/iceberg - ICEBERG_MAIN_REPOSITORY: apache/iceberg - ICEBERG_MAIN_BRANCH: master - ICEBERG_PATCH_REPOSITORY: snazy/iceberg - ICEBERG_PATCH_BRANCH: iceberg-nesqueit - SPARK_LOCAL_IP: localhost - - steps: - - - name: Prepare Git - if: env.WF_EXEC == 'true' - run: | - git config --global user.email "integrations-testing@projectnessie.org" - git config --global user.name "Integrations Testing [Bot]" - - - name: Checkout NeQuEIT repo - if: env.WF_EXEC == 'true' - uses: actions/checkout@v3.5.0 - with: - repository: ${{env.NESQUEIT_REPOSITORY}} - ref: ${{env.NESQUEIT_BRANCH}} - - - name: Setup runner - if: env.WF_EXEC == 'true' - uses: ./.github/actions/setup-runner - with: - more-memory: 'true' - - - name: Checkout and patch Nessie PR - if: env.WF_EXEC == 'true' - uses: ./.github/actions/patch-git - with: - name: Nessie - local-dir: ${{env.NESSIE_DIR}} - main-repository: ${{ env.GITHUB_REPOSITORY }} - patch-repository: ${{env.NESSIE_PATCH_REPOSITORY}} - patch-branch: ${{env.NESSIE_PATCH_BRANCH}} - work-branch: nessie-integration-patched - - - name: Checkout and patch Iceberg - if: env.WF_EXEC == 'true' - uses: ./.github/actions/patch-git - with: - name: Nessie - local-dir: ${{env.ICEBERG_DIR}} - main-repository: ${{env.ICEBERG_MAIN_REPOSITORY}} - main-branch: ${{env.ICEBERG_MAIN_BRANCH}} - patch-repository: ${{env.ICEBERG_PATCH_REPOSITORY}} - patch-branch: ${{env.ICEBERG_PATCH_BRANCH}} - work-branch: iceberg-integration-patched - - # Setup Gradle properties, heap requirements are for the "Integration test w/ Nessie". - - name: Setup gradle.properties - if: env.WF_EXEC == 'true' - run: | - mkdir -p ~/.gradle - echo "org.gradle.jvmargs=-Xms2g -Xmx4g -XX:MaxMetaspaceSize=768m -Dfile.encoding=UTF-8" >> ~/.gradle/gradle.properties - echo "org.gradle.vfs.watch=false" >> ~/.gradle/gradle.properties - - - name: Set up JDK ${{ matrix.java-version }} - if: env.WF_EXEC == 'true' - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 11 - - - name: Iceberg Nessie test - if: env.WF_EXEC == 'true' - uses: gradle/gradle-build-action@v2 - with: - cache-read-only: true - arguments: :iceberg:iceberg-nessie:test --scan - - - name: Nessie Spark 3.1 Extensions test - if: env.WF_EXEC == 'true' - uses: gradle/gradle-build-action@v2 - with: - arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.1_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.1_2.12:intTest --scan - - - name: Nessie Spark 3.2 / 2.12 Extensions test - if: env.WF_EXEC == 'true' - uses: gradle/gradle-build-action@v2 - with: - arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.2_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.2_2.12:intTest --scan - - - name: Nessie Spark 3.3 / 2.12 Extensions test - if: env.WF_EXEC == 'true' - uses: gradle/gradle-build-action@v2 - with: - arguments: :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:intTest --scan - - # TODO re-enable once Iceberg/master references a Nessie release w/ Nessie PR #6197, - # Gradle's Maven publishing fails with the group-ID relocation. - # - # - name: Publish Nessie + Iceberg to local Maven repo - # if: env.WF_EXEC == 'true' - # uses: gradle/gradle-build-action@v2 - # with: - # arguments: publishLocal --scan - - - name: Gather locally published versions - if: env.WF_EXEC == 'true' - run: | - NESSIE_VERSION="$(cat included-builds/nessie/version.txt)" - ICEBERG_VERSION="$(cat included-builds/iceberg/build/iceberg-build.properties | grep '^git.build.version=' | cut -d= -f2)" - echo "NESSIE_VERSION=${NESSIE_VERSION}" >> ${GITHUB_ENV} - echo "ICEBERG_VERSION=${ICEBERG_VERSION}" >> ${GITHUB_ENV} - cat <> $GITHUB_STEP_SUMMARY - ## Published versions - | Published Nessie version | Published Iceberg version | - | ------------------------ | ------------------------- | - | ${NESSIE_VERSION} | ${ICEBERG_VERSION} | - ! - - - name: Tools & Integrations tests - if: env.WF_EXEC == 'true' - uses: gradle/gradle-build-action@v2 - with: - arguments: intTest --scan diff --git a/.github/workflows/pull-request-native.yml b/.github/workflows/pull-request-native.yml deleted file mode 100644 index afcf23b1639..00000000000 --- a/.github/workflows/pull-request-native.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (C) 2020 Dremio -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Projectnessie GitHub Pull-Request / Native-image tests - -name: PR Build Check Native - -on: - pull_request: - types: [labeled, opened, synchronize, reopened] - -jobs: - java: - name: Java/Gradle Native - runs-on: ubuntu-22.04 - env: - WF_EXEC: ${{ contains(github.event.pull_request.labels.*.name, 'pr-native') }} - steps: - - uses: actions/checkout@v3.5.0 - if: env.WF_EXEC == 'true' - - name: Setup runner - if: env.WF_EXEC == 'true' - uses: ./.github/actions/setup-runner - with: - more-memory: 'true' - - name: Setup Java, Gradle - if: env.WF_EXEC == 'true' - uses: ./.github/actions/dev-tool-java - - - name: Gradle / integration test native - if: env.WF_EXEC == 'true' - uses: gradle/gradle-build-action@v2 - with: - cache-read-only: true - arguments: | - --no-watch-fs - :nessie-quarkus:quarkusBuild - :nessie-quarkus:intTest - -Pnative - --scan - - - name: Capture Test Reports - uses: actions/upload-artifact@v3 - if: env.WF_EXEC == 'true' && failure() - with: - name: test-results-native - path: | - **/build/reports/* - **/build/test-results/* - retention-days: 3 diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml deleted file mode 100644 index 8ad6c9050ac..00000000000 --- a/.github/workflows/pull-request.yml +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (C) 2020 Dremio -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Projectnessie GitHub Pull-Request / Default CI - -name: PR Build Check - -on: - pull_request: - types: [opened, synchronize, reopened] -# NOTE: using paths-ignore here doesn't play well with "Require status checks to pass before merging" -# due to https://github.community/t/feature-request-conditional-required-checks/16761 -# paths-ignore: -# - 'LICENSE' -# - 'NOTICE' -# - '**.md' -# - '!site/**' -# - '.github/renovate.json5' -# - '.github/workflows/release*.yml' -# - '.github/workflows/check*.yml' -# - '.idea/**' -# - '.editorconfig' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - java: - name: Java/Gradle - runs-on: ubuntu-22.04 - env: - SPARK_LOCAL_IP: localhost - - steps: - - uses: actions/checkout@v3.5.0 - - name: Setup runner - uses: ./.github/actions/setup-runner - - name: Setup Java, Gradle - uses: ./.github/actions/dev-tool-java - - - name: Gradle / compile - uses: gradle/gradle-build-action@v2 - with: - arguments: spotlessCheck checkstyle assemble --scan - - - name: Gradle / unit test - uses: gradle/gradle-build-action@v2 - env: - SPARK_LOCAL_IP: localhost - with: - arguments: test testCodeCoverageReport --scan - - - name: Gradle / check incl. integ-test - uses: gradle/gradle-build-action@v2 - with: - arguments: check -x test intTestCodeCoverageReport --scan - - - name: Gradle / Gatling simulations - uses: gradle/gradle-build-action@v2 - with: - arguments: gatlingRun - - # Exercise these tasks to ensure those do not error out - - name: Gradle / assemble + publish local - uses: gradle/gradle-build-action@v2 - with: - arguments: assemble publishToMavenLocal --scan - - - name: Gradle / build tools integration tests - uses: gradle/gradle-build-action@v2 - with: - arguments: buildToolsIntegrationTest - - - name: Capture Test Reports - uses: actions/upload-artifact@v3 - if: ${{ failure() }} - with: - name: test-results - path: | - **/build/reports/* - **/build/test-results/* - retention-days: 3 - - - name: Capture Results - uses: ./.github/actions/ci-results - - python: - name: Python - runs-on: ubuntu-22.04 - env: - working-directory: ./python - strategy: - max-parallel: 4 - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - - steps: - - uses: actions/checkout@v3.5.0 - - name: Setup Python - uses: ./.github/actions/dev-tool-python - with: - python-version: ${{ matrix.python-version }} - - name: Test with tox - run: tox - working-directory: ${{env.working-directory}} - - uses: codecov/codecov-action@v3 - if: github.repository_owner == 'projectnessie' - continue-on-error: true - with: - verbose: true - fail_ci_if_error: true - flags: python - - site: - name: Build Website - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3.5.0 - - name: Setup Python - uses: ./.github/actions/dev-tool-python - with: - python-version: '3.8' - - name: Install dependencies - run: python3 -m pip install -r ./site/requirements.txt - - name: Check code block indentation - run: python3 check_code_indent.py - working-directory: ./site - - name: Generate Static Site - run: mkdocs build - working-directory: ./site diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml deleted file mode 100644 index 104054caa28..00000000000 --- a/.github/workflows/site.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Site - -on: - push: - branches: [ main ] - paths: - - 'site/**' -jobs: - - site: - name: Build & Deploy Website - runs-on: ubuntu-22.04 - if: github.repository_owner == 'projectnessie' - steps: - - uses: actions/checkout@v3.5.0 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - cache: 'pip' - cache-dependency-path: 'site/requirements.txt' - - name: Upgrade pip - run: | - # install pip=>20.1 to use "pip cache dir" - python3 -m pip install --upgrade pip - - name: Install dependencies - run: python3 -m pip install -r ./site/requirements.txt - - name: Generate Static Site - run: mkdocs build - working-directory: ./site - - name: Deploy Static Site to GitHub - uses: peaceiris/actions-gh-pages@v3 - with: - external_repository: projectnessie/projectnessie.github.io - publish_branch: main - deploy_key: ${{ secrets.NESSIE_SITE_DEPLOY_KEY }} - publish_dir: ./site/site - cname: projectnessie.org diff --git a/README.md b/README.md index cfeb254e12a..43f8d4ee373 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ Project Nessie is a Transactional Catalog for Data Lakes with Git-like semantics [![Java 17+18](https://img.shields.io/github/actions/workflow/status/projectnessie/nessie/newer-java.yml?label=Java%2017%2B&logo=Github&style=flat-square)](https://github.com/projectnessie/nessie/actions/workflows/newer-java.yml) [![Windows Build](https://img.shields.io/github/actions/workflow/status/projectnessie/nessie/ci-win.yml?label=Windows&logo=windows&style=flat-square)](https://github.com/projectnessie/nessie/actions/workflows/ci-win.yml) [![macOS Build](https://img.shields.io/github/actions/workflow/status/projectnessie/nessie/ci-mac.yml?label=macOS&logo=apple&style=flat-square)](https://github.com/projectnessie/nessie/actions/workflows/ci-mac.yml) -[![codecov](https://img.shields.io/codecov/c/github/projectnessie/nessie?style=flat-square)](https://codecov.io/gh/projectnessie/nessie) More information can be found at [projectnessie.org](https://projectnessie.org/). diff --git a/build.gradle.kts b/build.gradle.kts index 266734eee98..0279d88d753 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -108,3 +108,5 @@ spotless { target("nessie-iceberg/*.gradle.kts", "*.gradle.kts", "buildSrc/*.gradle.kts") } } + +val listProjectsWithPrefix by tasks.registering(ListChildProjectsTask::class) diff --git a/buildSrc/src/main/kotlin/ListChilrProjectsTask.kt b/buildSrc/src/main/kotlin/ListChilrProjectsTask.kt new file mode 100644 index 00000000000..47f98c020b1 --- /dev/null +++ b/buildSrc/src/main/kotlin/ListChilrProjectsTask.kt @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.gradle.api.DefaultTask +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.TaskAction +import org.gradle.api.tasks.options.Option +import org.gradle.work.DisableCachingByDefault + +@DisableCachingByDefault(because = "Projects list is worth caching") +abstract class ListChildProjectsTask : DefaultTask() { + @get:Option(option = "prefix", description = "Project path prefix.") + @get:Internal + abstract val prefix: Property + + @get:Option(option = "task", description = "Name of the task to print.") + @get:Internal + abstract val task: Property + + @get:Option(option = "output", description = "Output file name.") + @get:Internal + abstract val output: Property + + @get:Option(option = "exclude", description = "Output file name.") + @get:Internal + abstract val exclude: Property + + @TaskAction + fun exec() { + val prefix = prefix.get() + val task = task.convention("intTest").get() + val outputFile = output.get() + val exclude = if (exclude.convention(false).get()) "-x " else "" + project.file(outputFile).writer().use { + val writer = it + project.childProjects.values + .filter { it.path.startsWith(prefix) } + .forEach { writer.write("$exclude${it.path}:$task\n") } + } + } +}