diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 6fc2c4e9bf649..1acb191d21bc8 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -206,3 +206,5 @@ jobs: if-no-files-found: 'error' retention-days: 2 if: inputs.upload-mount-cache-artifact == 'true' + - name: "Check disk space after build" + run: df -H diff --git a/scripts/ci/make_mnt_writeable.sh b/scripts/ci/make_mnt_writeable.sh index 54a791eb3a5c9..ddabf84b6b79d 100755 --- a/scripts/ci/make_mnt_writeable.sh +++ b/scripts/ci/make_mnt_writeable.sh @@ -17,9 +17,13 @@ # under the License. function make_mnt_writeable { set -x + echo "Investigating node disks" + lsblk + sudo blkid echo "Checking free space!" df -H echo "Cleaning /mnt just in case it is not empty" + sudo mkdir -p /mnt sudo rm -rf /mnt/* echo "Checking free space!" df -H diff --git a/scripts/ci/move_docker_to_mnt.sh b/scripts/ci/move_docker_to_mnt.sh index 742fe14e2043d..e3d69ef60bea1 100755 --- a/scripts/ci/move_docker_to_mnt.sh +++ b/scripts/ci/move_docker_to_mnt.sh @@ -17,25 +17,37 @@ # under the License. function cleanup_runner { set -x - local target_docker_volume_location="/mnt/var-lib-docker" - echo "Checking free space!" - df -H - # This is faster than docker prune - echo "Stopping docker" - sudo systemctl stop docker - echo "Checking free space!" - df -H - echo "Cleaning docker" - sudo rm -rf /var/lib/docker - echo "Checking free space!" - df -H - echo "Mounting ${target_docker_volume_location} to /var/lib/docker" - sudo mkdir -p "${target_docker_volume_location}" /var/lib/docker - sudo mount --bind "${target_docker_volume_location}" /var/lib/docker - sudo chown -R 0:0 "${target_docker_volume_location}" - sudo systemctl start docker echo "Checking free space!" df -H + # Note: + # Disk layout on x86_64 (2026-01): + # - root (/) on /dev/sdb1 (75G) + # - an additional mount as /mnt with 75G on /dev/sda1 + # It seems that arm64 runners have + # - root (/) on /dev/sda1 (75G) + # - An un-used nvme with 220G + # Hence we only move docker to /mnt on x86_64 where we have a separate /mnt mount + # If we get short on disk space on arm64 as well we need to revisit this logic + # and make the idle nvme being used as well. + if uname -i|grep -q x86_64; then + local target_docker_volume_location="/mnt/var-lib-docker" + # This is faster than docker prune + echo "Stopping docker" + sudo systemctl stop docker + echo "Checking free space!" + df -H + echo "Cleaning docker" + sudo rm -rf /var/lib/docker + echo "Checking free space!" + df -H + echo "Mounting ${target_docker_volume_location} to /var/lib/docker" + sudo mkdir -p "${target_docker_volume_location}" /var/lib/docker + sudo mount --bind "${target_docker_volume_location}" /var/lib/docker + sudo chown -R 0:0 "${target_docker_volume_location}" + sudo systemctl start docker + echo "Checking free space!" + df -H + fi } cleanup_runner