From 44ead5e6c55d04f9473a908a8b03f2a6b392c20d Mon Sep 17 00:00:00 2001 From: Shuyang Xin Date: Mon, 29 Jan 2024 11:22:05 +0800 Subject: [PATCH] Update BuildKit Configurations to Enhance Compatibility * Add BuildKit cache cleanup in CI * Declare DOCKER_BUILDKIT when building image for docker compatibility. fixes #5941 Signed-off-by: Shuyang Xin --- ci/jenkins/test-mc.sh | 1 + ci/jenkins/test-rancher.sh | 1 + ci/jenkins/test-vmc.sh | 1 + ci/jenkins/test.sh | 1 + ci/jenkins/utils.sh | 17 +++++++++++++++++ ci/test-conformance-aks.sh | 4 +++- ci/test-conformance-eks.sh | 3 ++- ci/test-conformance-gke.sh | 3 ++- docs/minikube.md | 2 +- hack/build-antrea-linux-all.sh | 1 + 10 files changed, 30 insertions(+), 4 deletions(-) diff --git a/ci/jenkins/test-mc.sh b/ci/jenkins/test-mc.sh index 36f7968dc25..1f037e9f9c4 100755 --- a/ci/jenkins/test-mc.sh +++ b/ci/jenkins/test-mc.sh @@ -137,6 +137,7 @@ function clean_images() { docker images | grep -E 'mc-controller|antrea-ubuntu' | awk '{print $3}' | xargs -r docker rmi -f || true # Clean up dangling images generated in previous builds. docker image prune -f --filter "until=24h" || true > /dev/null + check_and_cleanup_docker_build_cache } function cleanup_multicluster_ns { diff --git a/ci/jenkins/test-rancher.sh b/ci/jenkins/test-rancher.sh index bb2d82eec71..d0148faecec 100644 --- a/ci/jenkins/test-rancher.sh +++ b/ci/jenkins/test-rancher.sh @@ -138,6 +138,7 @@ function deliver_antrea { git show --numstat make clean ${CLEAN_STALE_IMAGES} + check_and_upgrade_golang chmod -R g-w build/images/ovs chmod -R g-w build/images/base DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh index 8376edac8c5..e4077b88915 100755 --- a/ci/jenkins/test-vmc.sh +++ b/ci/jenkins/test-vmc.sh @@ -355,6 +355,7 @@ function deliver_antrea { # because they might be being used in other builds running simultaneously. docker image prune -af --filter "until=1h" > /dev/null docker system df -v + check_and_cleanup_docker_build_cache set -e cd $GIT_CHECKOUT_DIR diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index b8f31dc4d91..05fe3dfbdbb 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -186,6 +186,7 @@ function clean_antrea { done docker images | grep 'antrea' | awk '{print $3}' | xargs -r docker rmi || true docker images | grep '' | awk '{print $3}' | xargs -r docker rmi || true + check_and_cleanup_docker_build_cache } function clean_for_windows_install_cni { diff --git a/ci/jenkins/utils.sh b/ci/jenkins/utils.sh index 988ed891513..50aef3976b5 100644 --- a/ci/jenkins/utils.sh +++ b/ci/jenkins/utils.sh @@ -14,6 +14,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +function check_and_cleanup_docker_build_cache() { + free_space=$(df -h -B 1G / | awk 'NR==2 {print $4}') + free_space_threshold=40 + if [[ $free_space -lt $free_space_threshold ]]; then + # If cleaning up unused dangling images doesn't free up sufficient disk space, + # we will have to reduce the builder cache to 10GB to release enough disk space. + docker builder prune -af --keep-storage=10gb > /dev/null + free_space=$(df -h -B 1G / | awk 'NR==2 {print $4}') + if [[ $free_space -lt $free_space_threshold ]]; then + # If the first round cleanup doesn't free up sufficient disk space, + # we will have to clean up all builder cache to release enough disk space. + docker builder prune -af > /dev/null + fi + fi + docker system df -v +} + function check_and_upgrade_golang() { if [ -z "${GOLANG_RELEASE_DIR}" ]; then GOLANG_RELEASE_DIR="/var/lib/jenkins/golang-releases" diff --git a/ci/test-conformance-aks.sh b/ci/test-conformance-aks.sh index 4d508434d70..42a474905b2 100755 --- a/ci/test-conformance-aks.sh +++ b/ci/test-conformance-aks.sh @@ -193,10 +193,12 @@ function deliver_antrea_to_aks() { # because they might be being used in other builds running simultaneously. docker image prune -f --filter "until=2h" > /dev/null docker system df -v + check_and_cleanup_docker_build_cache + set -e cd ${GIT_CHECKOUT_DIR} - VERSION="$CLUSTER" make + VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull if [[ "$?" -ne "0" ]]; then echo "=== Antrea Image build failed ===" exit 1 diff --git a/ci/test-conformance-eks.sh b/ci/test-conformance-eks.sh index aaf915e1b6b..4ed97a8e9c5 100755 --- a/ci/test-conformance-eks.sh +++ b/ci/test-conformance-eks.sh @@ -270,10 +270,11 @@ function deliver_antrea_to_eks() { # because they might be being used in other builds running simultaneously. docker image prune -f --filter "until=2h" > /dev/null docker system df -v + check_and_cleanup_docker_build_cache set -e cd ${GIT_CHECKOUT_DIR} - VERSION="$CLUSTER" make + VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull if [[ "$?" -ne "0" ]]; then echo "=== Antrea Image build failed ===" exit 1 diff --git a/ci/test-conformance-gke.sh b/ci/test-conformance-gke.sh index 283413d2865..6ea7e6a8154 100755 --- a/ci/test-conformance-gke.sh +++ b/ci/test-conformance-gke.sh @@ -211,10 +211,11 @@ function deliver_antrea_to_gke() { # because they might be being used in other builds running simultaneously. docker image prune -f --filter "until=2h" > /dev/null docker system df -v + check_and_cleanup_docker_build_cache set -e cd ${GIT_CHECKOUT_DIR} - VERSION="$CLUSTER" make -C ${GIT_CHECKOUT_DIR} + VERSION="$CLUSTER" ./hack/build-antrea-linux-all.sh --pull if [[ "$?" -ne "0" ]]; then echo "=== Antrea Image build failed ===" exit 1 diff --git a/docs/minikube.md b/docs/minikube.md index ebd9f4424e1..ea818130dc4 100644 --- a/docs/minikube.md +++ b/docs/minikube.md @@ -26,7 +26,7 @@ minikube start --cni=antrea.yml --network-plugin=cni These instructions assume that you have built the Antrea Docker image locally (e.g. by running `make` from the root of the repository, or in case of arm64 architecture by running -`DOCKER_BUILDKIT=1 ./hack/build-antrea-ubuntu-all.sh --platform linux/arm64`). +`./hack/build-antrea-linux-all.sh --platform linux/arm64`). ```bash # load the Antrea Docker image in the minikube nodes diff --git a/hack/build-antrea-linux-all.sh b/hack/build-antrea-linux-all.sh index 56604ccd08c..b48766ce7b6 100755 --- a/hack/build-antrea-linux-all.sh +++ b/hack/build-antrea-linux-all.sh @@ -165,6 +165,7 @@ cd build/images/base cd - export NO_PULL=1 +export DOCKER_BUILDKIT=1 if [ "$DISTRO" == "ubuntu" ]; then if $COVERAGE; then make build-controller-ubuntu-coverage