Skip to content

Commit ad77bd9

Browse files
adutraadutra
authored
feat(ci): Improve Gradle cache in CI (#1928)
1 parent 6ddd148 commit ad77bd9

File tree

6 files changed

+219
-1
lines changed

6 files changed

+219
-1
lines changed

‎.github/actions/ci-incr-build-cache-prepare/action.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Copyright (C) 2020 Dremio
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# CODE_COPIED_TO_POLARIS Copied from Project Nessie v. 0.104.2
16+
17+
name: 'Incremental Gradle build cache prepare'
18+
description: 'Prepare to save incremental Gradle build cache'
19+
inputs:
20+
cache-read-only:
21+
description: 'Gradle cache read only'
22+
default: 'true'
23+
java-version:
24+
description: 'Java version'
25+
default: '21'
26+
job-id:
27+
description: 'Job ID to prefer'
28+
default: 'polaris-gradle'
29+
job-instance:
30+
description: 'Job instance to prefer'
31+
default: 'gradle'
32+
runs:
33+
using: "composite"
34+
steps:
35+
- name: Prep env
36+
shell: bash
37+
run: |
38+
echo "GRADLE_BUILD_ACTION_CACHE_KEY_ENVIRONMENT=java-${{ inputs.java-version }}" >> ${GITHUB_ENV}
39+
echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB=${{ inputs.job-id }}" >> ${GITHUB_ENV}
40+
if [[ -n "${{ inputs.no-daemon }}" ]] ; then
41+
echo "G_DAEMON_FLAG=--no-daemon" >> ${GITHUB_ENV}
42+
fi
43+
if [[ -n "${{ inputs.job-instance }}" ]] ; then
44+
echo "GRADLE_BUILD_ACTION_CACHE_KEY_JOB_INSTANCE=${{ inputs.job-instance }}" >> ${GITHUB_ENV}
45+
fi
46+
47+
- name: Gradle / Setup
48+
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
49+
with:
50+
cache-read-only: ${{ inputs.cache-read-only }}
51+
validate-wrappers: false
52+
53+
- name: Gradle / Init
54+
shell: bash
55+
run: ./gradlew -h
56+
57+
- name: Download existing workflow artifacts
58+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
59+
# Just in case, don't know the exact inner workings of Gradle's build cache and whether
60+
# the download-action complains about duplicate files.
61+
continue-on-error: true
62+
with:
63+
path: ~/downloaded-artifacts/
64+
65+
- name: Extract caches
66+
shell: bash
67+
run: |
68+
echo "::group::Gradle build cache / add incremental updates"
69+
mkdir -p ~/.gradle/caches/
70+
71+
if [[ -d ~/downloaded-artifacts/ ]] ; then
72+
find ~/downloaded-artifacts/ -type f -name "ci-gradle-caches-*-${{ inputs.java-version }}.tar" | while read arch ; do
73+
echo "Adding archive content from $arch ..."
74+
(cd ~/.gradle/caches/ ; tar xf $arch)
75+
done
76+
else
77+
echo "No previous build cache artifacts downloaded."
78+
fi
79+
80+
date +%s > ~/caches-prepared-at-epoch
81+
82+
echo "::endgroup::"

‎.github/actions/ci-incr-build-cache-save/action.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Copyright (C) 2020 Dremio
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# CODE_COPIED_TO_POLARIS Copied from Project Nessie v. 0.104.2
16+
17+
name: 'Save incremental Gradle caches'
18+
description: 'Save incremental Gradle caches'
19+
inputs:
20+
job-name:
21+
description: 'job name'
22+
java-version:
23+
description: 'Java version'
24+
default: '21'
25+
runs:
26+
using: "composite"
27+
steps:
28+
- name: Prepare Gradle caches archive
29+
shell: bash
30+
run: |
31+
if [[ -d ~/.gradle/caches/ ]] ; then
32+
echo "::group::Gradle caches / identify updated cache items"
33+
34+
cd ~/.gradle/caches/
35+
36+
echo "Gradle caches/ contains $(find . -type f | wc -l) files"
37+
# Identify the added and changed files in caches/.
38+
39+
echo "Identifying changed/added files..."
40+
AGE_SECS=$(($(date +%s) - $(cat ~/caches-prepared-at-epoch)))
41+
echo "Build started ~ $AGE_SECS seconds ago"
42+
AGE_MINS=$(($AGE_SECS / 60 + 1))
43+
echo " ... assuming that is ~ $AGE_MINS minutes"
44+
# This lists all relevant files that have been created or modified during by the Gradle
45+
# runs of the current job.
46+
find . -mmin -$AGE_MINS -type f '(' \
47+
-path './[0-9]*/kotlin-dsl/*' -or \
48+
-path './jars-*/*' -or \
49+
-path './modules-*/files-*/*' -or \
50+
-path './modules-*/files-*/*' -or \
51+
-path './build-cache-*/*' \
52+
')' | grep -v '[.]lock$' > ~/ci-gradle-caches-diff || true
53+
echo "Identified $(wc -l < ~/ci-gradle-caches-diff) changed/added files in caches/"
54+
55+
# Only call 'tar', if there is some difference
56+
# Note: actions/upload-artifact takes care of compressing the artifact, no need to bug the CPU here
57+
echo "Creating artifact (if necessary)..."
58+
if [[ -s ~/ci-gradle-caches-diff ]] ; then
59+
tar --create --ignore-failed-read --file ~/ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}.tar -T ~/ci-gradle-caches-diff
60+
ls -al ~/ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}.tar
61+
fi
62+
echo "::endgroup::"
63+
fi
64+
- name: Archive code-checks incremental
65+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
66+
with:
67+
name: ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}
68+
path: ~/ci-gradle-caches-${{ inputs.job-name }}-${{ inputs.java-version }}.tar
69+
if-no-files-found: ignore
70+
retention-days: 1

‎.github/workflows/gradle.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,19 @@ jobs:
5050
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
5151
with:
5252
validate-wrappers: false
53+
- name: Prepare Gradle build cache
54+
uses: ./.github/actions/ci-incr-build-cache-prepare
5355
- name: Run unit tests
5456
run: |
5557
./gradlew check sourceTarball distTar distZip publishToMavenLocal \
5658
-x :polaris-runtime-service:test \
5759
-x :polaris-admin:test \
5860
-x intTest --continue
61+
- name: Save partial Gradle build cache
62+
uses: ./.github/actions/ci-incr-build-cache-save
63+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
64+
with:
65+
job-name: 'unit-tests'
5966
- name: Archive test results
6067
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
6168
if: always()
@@ -80,12 +87,19 @@ jobs:
8087
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
8188
with:
8289
validate-wrappers: false
90+
- name: Prepare Gradle build cache
91+
uses: ./.github/actions/ci-incr-build-cache-prepare
8392
- name: Run Quarkus tests
8493
run: |
8594
./gradlew \
8695
:polaris-runtime-service:test \
8796
:polaris-admin:test \
8897
--continue
98+
- name: Save partial Gradle build cache
99+
uses: ./.github/actions/ci-incr-build-cache-save
100+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
101+
with:
102+
job-name: 'quarkus-tests'
89103
- name: Archive test results
90104
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
91105
if: always()
@@ -110,12 +124,46 @@ jobs:
110124
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
111125
with:
112126
validate-wrappers: false
127+
- name: Prepare Gradle build cache
128+
uses: ./.github/actions/ci-incr-build-cache-prepare
113129
- name: Run integration tests
114130
run: ./gradlew intTest --continue
131+
- name: Save partial Gradle build cache
132+
uses: ./.github/actions/ci-incr-build-cache-save
133+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
134+
with:
135+
job-name: 'integration-tests'
115136
- name: Archive test results
116137
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
117138
if: always()
118139
with:
119140
name: upload-integration-test-artifacts
120141
path: |
121142
**/build/test-results/**
143+
144+
store-gradle-cache:
145+
name: Store Gradle Cache
146+
runs-on: ubuntu-24.04
147+
timeout-minutes: 30
148+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
149+
needs:
150+
- unit-tests
151+
- quarkus-tests
152+
- integration-tests
153+
steps:
154+
- name: Set up JDK 21
155+
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4
156+
with:
157+
java-version: '21'
158+
distribution: 'temurin'
159+
- name: Setup Gradle
160+
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
161+
with:
162+
validate-wrappers: false
163+
- name: Collect partial Gradle build caches
164+
uses: ./.github/actions/ci-incr-build-cache-prepare
165+
with:
166+
cache-read-only: false
167+
- name: Trigger Gradle home cleanup
168+
run: ./gradlew --no-daemon :showVersion
169+
# Note: the "Post Gradle invocation" archives the updated build cache.

‎.github/workflows/regtest.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ jobs:
4040
java-version: '21'
4141
distribution: 'temurin'
4242

43+
- name: Setup Gradle
44+
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
45+
with:
46+
validate-wrappers: false
47+
48+
- name: Prepare Gradle build cache
49+
uses: ./.github/actions/ci-incr-build-cache-prepare
50+
4351
- name: Fix permissions
4452
run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
4553

@@ -58,4 +66,4 @@ jobs:
5866
AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
5967
AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
6068
run: |
61-
docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest
69+
docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest

‎.github/workflows/spark_client_regtests.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ jobs:
4040
java-version: '21'
4141
distribution: 'temurin'
4242

43+
- name: Setup Gradle
44+
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4
45+
with:
46+
validate-wrappers: false
47+
48+
- name: Prepare Gradle build cache
49+
uses: ./.github/actions/ci-incr-build-cache-prepare
50+
4351
- name: Fix permissions
4452
run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
4553

‎LICENSE‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,8 @@ License: https://www.apache.org/licenses/LICENSE-2.0
264264

265265
This product includes code from Project Nessie.
266266

267+
* .github/actions/ci-incr-build-cache-prepare/action.yml
268+
* .github/actions/ci-incr-build-cache-save/action.yml
267269
* build-logic/src/main/kotlin/LicenseFileValidation.kt
268270
* build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerPlugin.kt
269271
* build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerExtension.kt

0 commit comments

Comments
 (0)