Refactor CI workflow

Unifies most main + PR workflows into a single workflow.

The unified CI workflow consists of 2 "stages":
* Checks - test, intTest, NesQuEIT, etc
* Finalize - a "success" dummy job + a "save to github-cache" job

Utilizes the Gradle build cache for all stages. The updated build cache
of the jobs in the checks stage are saved as artifacts (with the minimum
retention period). The updated build cache is pushed back to GigHub's
cache when the checks have successfully finished.

Java CI runs against Java 11 and Java 17, where it is meaningful.
(Spark + Deltalake tests always run against 11, so not run against
Java 17.) Some checks also run against the latest Java version.

Codecov was not added to the new workflow, it wasn't working for quite
a while now or produced wrong results.

Build logs and reports are not archived. Test results and relevant logs
are available via Gradle build scans.

Windows + macOS workflows are not included in the unified workflow.

There is also another [Gradle cache
action](https://github.com/burrunan/gradle-cache-action), which utilizes
the GitHub's cache like a remote Gradle cache. However, that puts too
much load (requests) against GitHub's cache, which in turn throttles our
CI and responds with HTTP/429 (Too many requests). See [this
issue](burrunan/gradle-cache-action#66).

Fixes projectnessie#6365

.github/actions/ci-incr-build-cache-prepare/action.yml

@@ -0,0 +1,74 @@
+name: 'Incremental Gradle build cache prepare'
+description: 'Prepare to save incremental Gradle build cache'
+inputs:
+ cache-read-only:
+ description: 'Gradle cache read only'
+ default: 'true'
+ java-version:
+ description: 'Java version'
+ default: '11'
+ job-id:
+ description: 'Job name to prefer'
+ default: 'nessie-ci'
+ job-instance:
+ description: 'Job instance to prefer'
+ default: 'ci'
+runs:
+ using: "composite"
+ steps:
+ - name: Prep env
+ shell: bash
+ run: |
+ echo "GRADLE_BUILD_ACTION_CACHE_KEY_ENVIRONMENT=java-${{ inputs.java-version }}" >> ${GITHUB_ENV}
+ echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB=${{ inputs.job-id }}" >> ${GITHUB_ENV}
+ if [[ -n "${{ inputs.no-daemon }}" ]] ; then
+ echo "G_DAEMON_FLAG=--no-daemon" >> ${GITHUB_ENV}
+ fi
+ if [[ -n "${{ inputs.job-instance }}" ]] ; then
+ echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB_INSTANCE=${{ inputs.job-instance }}" >> ${GITHUB_ENV}
+ fi
+
+ - name: Gradle / Init
+ uses: gradle/gradle-build-action@v2
+ with:
+ cache-read-only: ${{ inputs.cache-read-only }}
+ arguments: -h
+
+ - name: Download existing workflow artifacts
+ uses: actions/download-artifact@v3
+ # Just in case, don't know the exact inner workings of Gradle's build cache and whether
+ # the download-action complains about duplicate files.
+ continue-on-error: true
+ with:
+ path: ~/downloaded-artifacts/
+
+ - name: Extract caches
+ shell: bash
+ run: |
+ echo "::group::Gradle build cache / add incremental updates"
+ mkdir -p ~/.gradle/caches/build-cache-1/
+ echo "Gradle build-cache-1 contains $(ls -1 ~/.gradle/caches/build-cache-1/ | wc -l) files"
+
+ if [[ -d ~/downloaded-artifacts/ ]] ; then
+ find ~/downloaded-artifacts/ -type f -name "ci-gradle-build-cache-*-${{ inputs.java-version }}.tar" | while read arch ; do
+ echo "Adding archive content from $arch ..."
+ (cd ~/.gradle/caches/build-cache-1/ ; tar xf $arch)
+ done
+
+ else
+ echo "No previous build cache artifacts downloaded."
+ fi
+ echo "::endgroup::"
+
+ - name: Memoize build-cache
+ shell: bash
+ run: |
+ echo "::group::Saving state of Gradle's build-cache-1 ..."
+ rm -rf ~/saved-build-cache-1/
+ mkdir -p ~/saved-build-cache-1/
+
+ if [[ -d ~/.gradle/caches/build-cache-1/ ]] ; then
+ echo "Gradle build-cache-1 contains $(ls -1 ~/.gradle/caches/build-cache-1/ | wc -l) files"
+ (cd ~/.gradle/caches/build-cache-1/ ; cp -r . ~/saved-build-cache-1/)
+ fi
+ echo "::endgroup::"

.github/actions/ci-incr-build-cache-save/action.yml

@@ -0,0 +1,41 @@
+name: 'Save incremental Gradle build cache'
+description: 'Save incremental Gradle build cache'
+inputs:
+ job-name:
+ description: 'job name'
+ java-version:
+ description: 'Java version'
+ default: '11'
+runs:
+ using: "composite"
+ steps:
+ - name: Prepare build-cache archive
+ shell: bash
+ run: |
+ if [[ -d ~/.gradle/caches/build-cache-1/ ]] ; then
+ echo "::group::Gradle build cache / identify updated build cache items"
+
+ cd ~/.gradle/caches/build-cache-1/
+
+ echo "Gradle build-cache-1 has $(ls -1 . | wc -l) files"
+ # Identify the added and changed files in build-cache-1.
+
+ echo "Identifying changed/added files..."
+ # 'diff' returns 1, if files differ :(
+ (diff --brief --recursive --new-file --no-dereference ~/saved-build-cache-1/ . || true) | \
+ cut -d\ -f4 > ~/ci-gradle-build-cache-diff
+ echo "Identified $(wc -l < ~/ci-gradle-build-cache-diff) changed/added files in build-cache-1"
+
+ # Only call 'tar', if there is some difference
+ # Note: actions/upload-artifact takes care of compressing the artifact, no need to bug the CPU here
+ echo "Creating artifact (if necessary)..."
+ [[ -s ~/ci-gradle-build-cache-diff ]] && tar cf ~/ci-gradle-build-cache-${{ inputs.job-name }}-${{ inputs.java-version }}.tar -T ~/ci-gradle-build-cache-diff
+ echo "::endgroup::"
+ fi
+ - name: Archive code-checks incremental
+ uses: actions/upload-artifact@v3
+ with:
+ name: ci-gradle-build-cache-${{ inputs.job-name }}-${{ inputs.java-version }}
+ path: ~/ci-gradle-build-cache-${{ inputs.job-name }}-${{ inputs.java-version }}.tar
+ if-no-files-found: ignore
+ retention-days: 1

.github/workflows/check-results-upload.yml

@@ -12,7 +12,7 @@ name: Check Results Upload
 
 on:
  workflow_run:
- workflows: ["PR Build Check", "Main CI"]
+ workflows: ["CI Build"]
  types:
  - completed
 

.github/workflows/ci-mac.yml

@@ -92,6 +92,7 @@ jobs:
  - name: Gradle / compile
  uses: gradle/gradle-build-action@v2
  with:
+ cache-read-only: true
  arguments: assemble --scan
 
  - name: Gradle / unit test
@@ -123,9 +124,9 @@ jobs:
  env:
  working-directory: ./python
  strategy:
- max-parallel: 4
+ max-parallel: 1
  matrix:
- python-version: ['3.7', '3.8', '3.9', '3.10']
+ python-version: ['3.8', '3.9', '3.10', '3.7']
 
  steps:
  - uses: actions/checkout@v3.5.0

.github/workflows/ci-win.yml

@@ -83,6 +83,7 @@ jobs:
  - name: Gradle / compile
  uses: gradle/gradle-build-action@v2
  with:
+ cache-read-only: true
  arguments: assemble --scan
 
  - name: Gradle / unit test
@@ -114,9 +115,9 @@ jobs:
  env:
  working-directory: ./python
  strategy:
- max-parallel: 4
+ max-parallel: 1
  matrix:
- python-version: ['3.7', '3.8', '3.9', '3.10']
+ python-version: ['3.8', '3.9', '3.10', '3.7']
 
  steps:
  - uses: actions/checkout@v3.5.0