Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci-image-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -206,3 +206,5 @@ jobs:
if-no-files-found: 'error'
retention-days: 2
if: inputs.upload-mount-cache-artifact == 'true'
- name: "Check disk space after build"
run: df -H
7 changes: 1 addition & 6 deletions scripts/ci/make_mnt_writeable.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,10 @@ function make_mnt_writeable {
echo "Investigating node disks"
lsblk
sudo blkid
echo "Check that we have expected /mnt to be a separate mount"
if ! lsblk | grep -q /mnt; then
echo "!!!! /mnt is missing as a separate mount, runner misconfigured!"
echo "Creating /mnt drive hoping that it will be enough space to use in /"
sudo mkdir -p /mnt/
fi
echo "Checking free space!"
df -H
echo "Cleaning /mnt just in case it is not empty"
sudo mkdir -p /mnt
sudo rm -rf /mnt/*
echo "Checking free space!"
df -H
Expand Down
46 changes: 29 additions & 17 deletions scripts/ci/move_docker_to_mnt.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,25 +17,37 @@
# under the License.
function cleanup_runner {
set -x
local target_docker_volume_location="/mnt/var-lib-docker"
echo "Checking free space!"
df -H
# This is faster than docker prune
echo "Stopping docker"
sudo systemctl stop docker
echo "Checking free space!"
df -H
echo "Cleaning docker"
sudo rm -rf /var/lib/docker
echo "Checking free space!"
df -H
echo "Mounting ${target_docker_volume_location} to /var/lib/docker"
sudo mkdir -p "${target_docker_volume_location}" /var/lib/docker
sudo mount --bind "${target_docker_volume_location}" /var/lib/docker
sudo chown -R 0:0 "${target_docker_volume_location}"
sudo systemctl start docker
echo "Checking free space!"
df -H
# Note:
# Disk layout on x86_64 (2026-01):
# - root (/) on /dev/sdb1 (75G)
# - an additional mount as /mnt with 75G on /dev/sda1
# It seems that arm64 runners have
# - root (/) on /dev/sda1 (75G)
# - An un-used nvme with 220G
# Hence we only move docker to /mnt on x86_64 where we have a separate /mnt mount
# If we get short on disk space on arm64 as well we need to revisit this logic
# and make the idle nvme being used as well.
if uname -i|grep -q x86_64; then
local target_docker_volume_location="/mnt/var-lib-docker"
# This is faster than docker prune
echo "Stopping docker"
sudo systemctl stop docker
echo "Checking free space!"
df -H
echo "Cleaning docker"
sudo rm -rf /var/lib/docker
echo "Checking free space!"
df -H
echo "Mounting ${target_docker_volume_location} to /var/lib/docker"
sudo mkdir -p "${target_docker_volume_location}" /var/lib/docker
sudo mount --bind "${target_docker_volume_location}" /var/lib/docker
sudo chown -R 0:0 "${target_docker_volume_location}"
sudo systemctl start docker
echo "Checking free space!"
df -H
fi
}

cleanup_runner
Loading