Skip to content

Commit 4ec81f1

Browse files
LKreutzerLKreutzer
authored
chore(ci): Modify the bazel workflow to use remote caching (magma#12743)
Signed-off-by: Lars Kreutzer <lars.kreutzer@tngtech.com>
1 parent 648289f commit 4ec81f1

File tree

5 files changed

+100
-80
lines changed

5 files changed

+100
-80
lines changed

‎.bazelrc

+14
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,20 @@ build:devcontainer --disk_cache=/workspaces/magma/.bazel-cache
3333
common:devcontainer --repository_cache=/workspaces/magma/.bazel-cache-repo
3434
build:devcontainer --define=folly_so=1
3535

36+
# REMOTE CACHING READ AND WRITE CONFIGS
37+
# The file bazel/bazelrcs/remote_caching_rw.bazelrc is templated in CI
38+
# The full config is then written to remote-cache.bazelrc
39+
build:remote_caching_rw --remote_download_toplevel
40+
41+
# REMOTE CACHING READ-ONLY CONFIGS
42+
# The file bazel/bazelrcs/remote_caching_ro.bazelrc is templated in CI
43+
# The full config is then written to remote-cache.bazelrc
44+
build:remote_caching_ro --remote_download_toplevel
45+
build:remote_caching_ro --remote_upload_local_results=false
46+
47+
# Try importing the bazel remote caching config (relevant in CI)
48+
try-import remote-cache.bazelrc
49+
3650
# TEST CONFIGS
3751
# Bazel test runtime default: PATH=/bin:/usr/bin:/usr/local/bin
3852
# Some python tests require access to /usr/sbin binaries (e.g. route, ifconfig)

‎.github/workflows/bazel.yml

+9-80
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
---
22
name: "Bazel Build & Test"
33
on: # yamllint disable-line rule:truthy
4+
workflow_dispatch:
45
pull_request:
56
types:
67
- opened
@@ -16,10 +17,8 @@ on: # yamllint disable-line rule:truthy
1617
- cron: '0 0,6,12,18 * * *'
1718
env:
1819
BAZEL_BASE_IMAGE: "ghcr.io/magma/magma/bazel-base:latestv2"
19-
BAZEL_CACHE: bazel-cache
20-
BAZEL_CACHE_REPO: bazel-cache-repo
21-
CACHE_SUB_KEY_BUILD_ALL: build-all
22-
CACHE_SUB_KEY_TEST_ALL: test-all
20+
CACHE_KEY: bazel-base-image
21+
2322

2423
concurrency:
2524
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
@@ -73,48 +72,6 @@ jobs:
7372
- name: Check Out Repo
7473
# This is necessary for overlays into the Docker container below.
7574
uses: actions/checkout@v2
76-
- name: Bazel Cache
77-
uses: actions/cache@v2
78-
with:
79-
path: ${{ github.workspace }}/.${{ env.BAZEL_CACHE }}
80-
key: ${{ runner.os }}-${{ env.BAZEL_CACHE }}-${{ env.CACHE_SUB_KEY_BUILD_ALL }}-${{ github.sha }}
81-
restore-keys: |
82-
${{ runner.os }}-${{ env.BAZEL_CACHE }}-${{ env.CACHE_SUB_KEY_BUILD_ALL }}-
83-
84-
- name: Bazel Cache Repo
85-
uses: actions/cache@v2
86-
with:
87-
path: ${{ github.workspace }}/.${{ env.BAZEL_CACHE_REPO }}
88-
key: ${{ runner.os }}-${{ env.BAZEL_CACHE_REPO }}-${{ github.sha }}
89-
restore-keys: |
90-
${{ runner.os }}-${{ env.BAZEL_CACHE_REPO }}-
91-
92-
# This check is needed to ensure that Bazel's unbounded cache growth doesn't result in a
93-
# situation where the cache never updates (e.g. due to exceeding GitHub's cache size limit)
94-
# thereby only ever using the last successful cache version. This solution will result in a
95-
# few slower CI actions around the time cache is detected to be too large, but it should
96-
# incrementally improve thereafter.
97-
- name: Ensure cache size BAZEL_CACHE
98-
# Only run on master to avoid slow build on PRs
99-
if: github.event_name == 'schedule' || github.event_name == 'push'
100-
env:
101-
BAZEL_CACHE_DIR: .${{ env.BAZEL_CACHE }}
102-
# Use a 6.5GB threshold since actions/cache compresses the results, and Bazel caches seem
103-
# to only increase by a few hundred megabytes across changes for unrelated branches.
104-
# Uncompressed cache on master is looking to be around 6GB (from observing jobs on master)
105-
BAZEL_CACHE_CUTOFF_MB: 6500
106-
run: |
107-
./.github/workflows/check-bazel-cache-dir-size.sh "$BAZEL_CACHE_DIR" "$BAZEL_CACHE_CUTOFF_MB"
108-
- name: Ensure cache size BAZEL_CACHE_REPO
109-
# Only run on master to avoid slow build on PRs
110-
if: github.event_name == 'schedule' || github.event_name == 'push'
111-
env:
112-
BAZEL_CACHE_REPO_DIR: .${{ env.BAZEL_CACHE_REPO }}
113-
# Use a 600 threshold since actions/cache compresses the results, and the repository cache should not increase unless we add more dependencies
114-
# Uncompressed cache on master is looking to be around 400MB (from observing jobs on master)
115-
BAZEL_CACHE_REPO_CUTOFF_MB: 600
116-
run: |
117-
./.github/workflows/check-bazel-cache-dir-size.sh "$BAZEL_CACHE_REPO_DIR" "$BAZEL_CACHE_REPO_CUTOFF_MB"
11875
- name: Setup Bazel Base Image
11976
uses: addnab/docker-run-action@v3
12077
with:
@@ -131,6 +88,7 @@ jobs:
13188
options: -v ${{ github.workspace }}:/workspaces/magma/
13289
run: |
13390
cd /workspaces/magma
91+
bazel/scripts/remote_cache_bazelrc_setup.sh "${{ env.CACHE_KEY }}" "${{ secrets.BAZEL_REMOTE_PASSWORD }}"
13492
bazel build //...
13593
- name: Build space left after run
13694
shell: bash
@@ -162,39 +120,6 @@ jobs:
162120
- name: Check Out Repo
163121
# This is necessary for overlays into the Docker container below.
164122
uses: actions/checkout@v2
165-
- name: Bazel Cache
166-
uses: actions/cache@v2
167-
with:
168-
path: ${{ github.workspace }}/.${{ env.BAZEL_CACHE }}
169-
key: ${{ runner.os }}-${{ env.BAZEL_CACHE }}-${{ env.CACHE_SUB_KEY_TEST_ALL }}-${{ github.sha }}
170-
restore-keys: |
171-
${{ runner.os }}-${{ env.BAZEL_CACHE }}-${{ env.CACHE_SUB_KEY_TEST_ALL }}-
172-
173-
- name: Bazel Cache Repo
174-
uses: actions/cache@v2
175-
with:
176-
path: ${{ github.workspace }}/.${{ env.BAZEL_CACHE_REPO }}
177-
key: ${{ runner.os }}-${{ env.BAZEL_CACHE_REPO }}-${{ github.sha }}
178-
restore-keys: |
179-
${{ runner.os }}-${{ env.BAZEL_CACHE_REPO }}-
180-
181-
# This check is needed to ensure that Bazel's unbounded cache growth doesn't result in a
182-
# situation where the cache never updates (e.g. due to exceeding GitHub's cache size limit)
183-
# thereby only ever using the last successful cache version. This solution will result in a
184-
# few slower CI actions around the time cache is detected to be too large, but it should
185-
# incrementally improve thereafter.
186-
- name: Ensure cache size BAZEL_CACHE
187-
# Only run on master to avoid slow build on PRs
188-
if: github.event_name == 'schedule' || github.event_name == 'push'
189-
env:
190-
BAZEL_CACHE_DIR: .${{ env.BAZEL_CACHE }}
191-
# Use a 6.5GB threshold since actions/cache compresses the results, and Bazel caches seem
192-
# to only increase by a few hundred megabytes across changes for unrelated branches.
193-
# Uncompressed cache on master is looking to be around 6GB (from observing jobs on master)
194-
BAZEL_CACHE_CUTOFF_MB: 6500
195-
run: |
196-
./.github/workflows/check-bazel-cache-dir-size.sh "$BAZEL_CACHE_DIR" "$BAZEL_CACHE_CUTOFF_MB"
197-
# Letting the Build job above handle cache clean up for bazel-cache-repo
198123
- name: Setup Bazel Base Image
199124
uses: addnab/docker-run-action@v3
200125
with:
@@ -211,7 +136,10 @@ jobs:
211136
options: -v ${{ github.workspace }}:/workspaces/magma/ -v ${{ github.workspace }}/lte/gateway/configs:/etc/magma
212137
run: |
213138
cd /workspaces/magma
214-
bazel test //... --test_output=errors --cache_test_results=no
139+
bazel/scripts/remote_cache_bazelrc_setup.sh "${{ env.CACHE_KEY }}" "${{ secrets.BAZEL_REMOTE_PASSWORD }}"
140+
bazel test //... \
141+
--cache_test_results=no \
142+
--test_output=errors
215143
- name: Run `bazel run //:check_starlark_format`
216144
uses: addnab/docker-run-action@v3
217145
with:
@@ -229,6 +157,7 @@ jobs:
229157
options: -v ${{ github.workspace }}:/workspaces/magma/
230158
run: |
231159
cd /workspaces/magma
160+
bazel/scripts/remote_cache_bazelrc_setup.sh "${{ env.CACHE_KEY }}" "${{ secrets.BAZEL_REMOTE_PASSWORD }}"
232161
bazel/scripts/test_python_service_imports.sh
233162
- name: Build space left after run
234163
shell: bash

‎bazel/bazelrcs/remote_caching_ro.bazelrc

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# This file is templated in CI (see e.g. .github/workflows/bazel.yml)
2+
# The full config is then written to remote-cache.bazelrc which is imported in the .bazelrc
3+
build --config=remote_caching_ro
4+
build:remote_caching_ro --remote_cache="https://bazel-remote-cache.magmacore-ci.org:9090/~~CACHE_KEY~~"

‎bazel/bazelrcs/remote_caching_rw.bazelrc

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# This file is templated in CI (see e.g. .github/workflows/bazel.yml)
2+
# The full config is then written to remote-cache.bazelrc which is imported in the .bazelrc
3+
build --config=remote_caching_rw
4+
build:remote_caching_rw --remote_cache="https://bazelbuild:~~BAZEL_REMOTE_PASSWORD~~@bazel-remote-cache.magmacore-ci.org:9090/~~CACHE_KEY~~"

‎bazel/scripts/remote_cache_bazelrc_setup.sh

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#!/usr/bin/env bash
2+
3+
################################################################################
4+
# Copyright 2022 The Magma Authors.
5+
6+
# This source code is licensed under the BSD-style license found in the
7+
# LICENSE file in the root directory of this source tree.
8+
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
###############################################################################
15+
16+
set -euo pipefail
17+
18+
###############################################################################
19+
# VARIABLES SECTION
20+
###############################################################################
21+
22+
# The CACHE_KEY is the mandatory first argument.
23+
CACHE_KEY=${1:-}
24+
25+
if [[ -z "$CACHE_KEY" ]]
26+
then
27+
echo "Required argument CACHE_KEY not set!" >&2
28+
exit 1
29+
fi
30+
31+
# The BAZEL_REMOTE_PASSWORD is an optional second argument.
32+
BAZEL_REMOTE_PASSWORD=${2:-}
33+
34+
###############################################################################
35+
# FUNCTIONS SECTION
36+
###############################################################################
37+
38+
create_config () {
39+
local cache_key=$1
40+
local bazel_remote_password=$2
41+
if [[ -n "$bazel_remote_password" ]]
42+
then
43+
create_config_for_rw_remote_cache "$cache_key" "$bazel_remote_password"
44+
else
45+
create_config_for_ro_remote_cache "$cache_key"
46+
fi
47+
}
48+
49+
create_config_for_rw_remote_cache () {
50+
local cache_key=$1
51+
local bazel_remote_password=$2
52+
sed \
53+
-e s/~~CACHE_KEY~~/"$cache_key"/ \
54+
-e s/~~BAZEL_REMOTE_PASSWORD~~/"$bazel_remote_password"/ \
55+
bazel/bazelrcs/remote_caching_rw.bazelrc
56+
}
57+
58+
create_config_for_ro_remote_cache () {
59+
local cache_key=$1
60+
sed \
61+
-e s/~~CACHE_KEY~~/"$cache_key"/ \
62+
bazel/bazelrcs/remote_caching_ro.bazelrc
63+
}
64+
65+
###############################################################################
66+
# SCRIPT SECTION
67+
###############################################################################
68+
69+
create_config "$CACHE_KEY" "$BAZEL_REMOTE_PASSWORD" > remote-cache.bazelrc

0 commit comments

Comments
 (0)