Skip to content

Commit 6660b29

Browse files
Kalimuthu-VelappanKalimuthu-Velappan
committed
05.Version cache - docker dpkg caching support
This feature caches all the deb files during docker build and stores them into version cache. It loads the cache file if already exists in the version cache and copies the extracted deb file from cache file into Debian cache path( /var/cache/apt/archives). The apt-install always installs the deb file from the cache if exists, this avoid unnecessary package download from the repo and speeds up the overall build. The cache file is selected based on the SHA value of version dependency files.
1 parent 6787457 commit 6660b29

9 files changed

+164
-7
lines changed

‎Makefile.work

-1
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,6 @@ SLAVE_TAG = $(shell \
234234
(cat $(SLAVE_DIR)/Dockerfile.user \
235235
$(SLAVE_DIR)/Dockerfile \
236236
$(SLAVE_DIR)/buildinfo/versions/versions-* \
237-
.git/HEAD 2>/dev/null \
238237
&& echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
239238
| sha1sum \
240239
| awk '{print substr($$1,0,11);}')

‎build_debian.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -443,10 +443,10 @@ if [[ $TARGET_BOOTLOADER == grub ]]; then
443443
GRUB_PKG=grub-efi-arm64-bin
444444
fi
445445

446-
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y download \
446+
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get install -d -o dir::cache=/var/cache/apt \
447447
$GRUB_PKG
448448

449-
sudo mv $FILESYSTEM_ROOT/grub*.deb $FILESYSTEM_ROOT/$PLATFORM_DIR/grub
449+
sudo cp $FILESYSTEM_ROOT/var/cache/apt/archives/grub*.deb $FILESYSTEM_ROOT/$PLATFORM_DIR/grub
450450
fi
451451

452452
## Disable kexec supported reboot which was installed by default

‎platform/broadcom/sai.mk

+5
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,9 @@ $(BRCM_DNX_SAI)_URL = "$(LIBSAIBCM_DNX_URL_PREFIX)/$(BRCM_DNX_SAI)"
1717
SONIC_ONLINE_DEBS += $(BRCM_XGS_SAI)
1818
SONIC_ONLINE_DEBS += $(BRCM_DNX_SAI)
1919
$(BRCM_XGS_SAI_DEV)_DEPENDS += $(BRCM_XGS_SAI)
20+
21+
$(BRCM_XGS_SAI)_SKIP_VERSION=y
22+
$(BRCM_XGS_SAI_DEV)_SKIP_VERSION=y
23+
$(BRCM_DNX_SAI)_SKIP_VERSION=y
24+
2025
$(eval $(call add_conflict_package,$(BRCM_XGS_SAI_DEV),$(LIBSAIVS_DEV)))

‎scripts/collect_docker_version_files.sh

+51
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,55 @@ DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup --no-cache
4848
docker rmi tmp-${DOCKER_IMAGE_TAG}
4949
docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/
5050

51+
52+
# Save the cache contents from docker build
53+
LOCAL_CACHE_FILE=target/vcache/${DOCKER_IMAGE_NAME}/cache.tgz
54+
CACHE_ENCODE_FILE=${DOCKER_PATH}/vcache/cache.base64
55+
sleep 1; sync ${CACHE_ENCODE_FILE}
56+
57+
# Decode the cache content into gz format
58+
SRC_VERSION_PATH=files/build/versions
59+
if [[ -e ${CACHE_ENCODE_FILE} ]]; then
60+
61+
cat ${CACHE_ENCODE_FILE} | base64 -d >${LOCAL_CACHE_FILE}
62+
rm -f ${CACHE_ENCODE_FILE}
63+
fi
64+
65+
# Version package cache
66+
IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
67+
if [[ ${DOCKER_IMAGE_NAME} == sonic-slave-* ]]; then
68+
GLOBAL_CACHE_DIR=${SONIC_VERSION_CACHE_SOURCE}/${DOCKER_IMAGE_NAME}
69+
else
70+
GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
71+
fi
72+
73+
if [[ ! -z ${SONIC_VERSION_CACHE} && -e ${CACHE_ENCODE_FILE} ]]; then
74+
75+
# Select version files for SHA calculation
76+
VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
77+
DEP_FILES="${DOCKER_PATH}/Dockerfile.j2"
78+
if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
79+
DEP_FILES="${DEP_FILES} build_debug_docker_j2.sh"
80+
fi
81+
82+
# Calculate the version SHA
83+
VERSION_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; cat ${DEP_FILES} ${VERSION_FILES}) | sha1sum | awk '{print substr($1,0,23);}')"
84+
GLOBAL_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_SHA}.tgz
85+
86+
GIT_FILE_STATUS=$(git status -s ${DEP_FILES})
87+
88+
# If the cache file is not exists in the global cache for the given SHA,
89+
# store the new cache file into version cache path.
90+
if [ -f ${LOCAL_CACHE_FILE} ]; then
91+
if [[ -z ${GIT_FILE_STATUS} && ! -e ${GLOBAL_CACHE_FILE} ]]; then
92+
mkdir -p ${GLOBAL_CACHE_DIR}
93+
chmod -f 777 ${GLOBAL_CACHE_DIR}
94+
FLOCK ${GLOBAL_CACHE_FILE}
95+
cp ${LOCAL_CACHE_FILE} ${GLOBAL_CACHE_FILE}
96+
chmod -f 777 ${LOCAL_CACHE_FILE} ${GLOBAL_CACHE_FILE}
97+
FUNLOCK ${GLOBAL_CACHE_FILE}
98+
fi
99+
fi
100+
fi
101+
51102
docker container rm $DOCKER_CONTAINER

‎scripts/prepare_docker_buildinfo.sh

+72
Original file line numberDiff line numberDiff line change
@@ -84,3 +84,75 @@ touch $BUILDINFO_VERSION_PATH/versions-deb
8484
LOCAL_CACHE_DIR=target/vcache/${IMAGENAME}
8585
mkdir -p ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
8686
chmod -f 777 ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
87+
88+
if [[ "$SKIP_BUILD_HOOK" == y || ${ENABLE_VERSION_CONTROL_DOCKER} != y ]]; then
89+
exit 0
90+
fi
91+
92+
# Version cache
93+
DOCKER_IMAGE_NAME=${IMAGENAME}
94+
IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
95+
96+
if [[ ${DOCKER_IMAGE_NAME} == sonic-slave-* ]]; then
97+
GLOBAL_CACHE_DIR=${SONIC_VERSION_CACHE_SOURCE}/${DOCKER_IMAGE_NAME}
98+
else
99+
GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
100+
fi
101+
102+
SRC_VERSION_PATH=files/build/versions
103+
if [ ! -z ${SONIC_VERSION_CACHE} ]; then
104+
105+
# Version files for SHA calculation
106+
VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
107+
DEP_FILES="Dockerfile.j2"
108+
if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
109+
DEP_DBG_FILES="build_debug_docker_j2.sh"
110+
fi
111+
112+
#Calculate the version SHA
113+
VERSION_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; \
114+
(cd ${DOCKER_PATH}; cat ${DEP_FILES}); \
115+
cat ${DEP_DBG_FILES} ${VERSION_FILES}) \
116+
| sha1sum | awk '{print substr($1,0,23);}')"
117+
118+
GLOBAL_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_SHA}.tgz
119+
LOCAL_CACHE_FILE=${LOCAL_CACHE_DIR}/cache.tgz
120+
GIT_FILE_STATUS=$(git status -s ${DEP_FILES})
121+
122+
# Create the empty cache tar file as local cache
123+
if [[ ! -f ${LOCAL_CACHE_FILE} ]]; then
124+
tar -zcf ${LOCAL_CACHE_FILE} -T /dev/null
125+
chmod -f 777 ${LOCAL_CACHE_FILE}
126+
fi
127+
128+
# Global cache file exists, load from global cache.
129+
if [[ -e ${GLOBAL_CACHE_FILE} ]]; then
130+
cp ${GLOBAL_CACHE_FILE} ${LOCAL_CACHE_FILE}
131+
touch ${GLOBAL_CACHE_FILE}
132+
else
133+
# When file is modified, Global SHA is calculated with the local change.
134+
# Load from the previous version of build cache if exists
135+
VERSIONS=( "HEAD" "HEAD~1" "HEAD~2" )
136+
for VERSION in ${VERSIONS[@]}; do
137+
VERSION_PREV_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; \
138+
(cd ${DOCKER_PATH}; git --no-pager show $(ls -f ${DEP_FILES}|sed 's|.*|'${VERSION}':./&|g')); \
139+
(git --no-pager show $(ls -f ${DEP_DBG_FILES} ${VERSION_FILES}|sed 's|.*|'${VERSION}':&|g'))) \
140+
| sha1sum | awk '{print substr($1,0,23);}')"
141+
GLOBAL_PREV_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_PREV_SHA}.tgz
142+
if [[ -e ${GLOBAL_PREV_CACHE_FILE} ]]; then
143+
cp ${GLOBAL_PREV_CACHE_FILE} ${LOCAL_CACHE_FILE}
144+
touch ${GLOBAL_PREV_CACHE_FILE}
145+
break
146+
fi
147+
done
148+
fi
149+
150+
rm -f ${DOCKER_PATH}/vcache/cache.tgz
151+
ln -f ${LOCAL_CACHE_FILE} ${DOCKER_PATH}/vcache/cache.tgz
152+
153+
154+
else
155+
# Delete the cache file if version cache is disabled.
156+
rm -f ${DOCKER_PATH}/vcache/cache.tgz
157+
fi
158+

‎src/sonic-build-hooks/scripts/collect_version_files

+6-3
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,12 @@ DIST=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)
1212
mkdir -p $TARGET_PATH
1313
chmod a+rw $TARGET_PATH
1414

15-
dpkg-query -W -f '${Package}==${Version}\n' >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
16-
([ -x "/usr/local/bin/pip2" ] || [ -x "/usr/bin/pip2" ]) && pip2 freeze >> "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}"
17-
([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze >> "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}"
15+
# Skip the package that does have a static build version.
16+
# SAI package versions are changed too frequently.
17+
SKIP_VERSION_PACKAGE="libsaibcm|libpaibcm|linuxptp|@ file://"
18+
dpkg-query -W -f '${Package}==${Version}\n' | grep -Ev "${SKIP_VERSION_PACKAGE}" > "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
19+
([ -x "/usr/local/bin/pip2" ] || [ -x "/usr/bin/pip2" ]) && pip2 freeze --all| grep -Ev "${SKIP_VERSION_PACKAGE}" > "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}"
20+
([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze --all| grep -Ev "${SKIP_VERSION_PACKAGE}" > "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}"
1821

1922
[ -f "${BUILD_WEB_VERSION_FILE}" ] && cp ${BUILD_WEB_VERSION_FILE} ${TARGET_PATH}
2023
[ -f "${BUILD_GIT_VERSION_FILE}" ] && cp ${BUILD_GIT_VERSION_FILE} ${TARGET_PATH}

‎src/sonic-build-hooks/scripts/post_run_buildinfo

+11
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,14 @@ collect_version_files $POST_VERSION_PATH
1313
#Save the cache file for exporting it to host.
1414
tar -C ${PKG_CACHE_PATH} --exclude=cache.tgz -zcvf /cache.tgz .
1515

16+
[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $POST_VERSION_PATH
17+
rm -rf $BUILD_VERSION_PATH/*
18+
if [ ! -z "$(get_version_cache_option)" ]; then
19+
# Restore he deletion of cache files
20+
cat <<-EOF >/etc/apt/apt.conf.d/docker-clean
21+
DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
22+
APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
23+
EOF
24+
25+
fi
26+

‎src/sonic-build-hooks/scripts/pre_run_buildinfo

+16
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,22 @@ symlink_build_hooks
1818
set_reproducible_mirrors
1919

2020
mkdir -p /var/cache/apt/archives/
21+
mkdir -p ${PKG_CACHE_PATH}/deb/
22+
23+
if [ ! -z "$(get_version_cache_option)" ]; then
24+
# Skip the deletion of cache files
25+
cat <<-EOF >/etc/apt/apt.conf.d/docker-clean
26+
DPkg::Post-Invoke { "test -f /usr/bin/rsync && rsync -avzh --ignore-errors /var/cache/apt/archives/ ${PKG_CACHE_PATH}/deb/; rm -f /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
27+
APT::Update::Post-Invoke { "test -f /usr/bin/rsync && rsync -avzh --ignore-errors /var/cache/apt/archives/ ${PKG_CACHE_PATH}/deb/; rm -f /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
28+
APT::Keep-Downloaded-Packages "true";
29+
EOF
30+
fi
31+
32+
# Extract the cache inside the docker build.
33+
if [ -f ${PKG_CACHE_FILE_NAME} ]; then
34+
tar -C ${PKG_CACHE_PATH} -xvf ${PKG_CACHE_FILE_NAME}
35+
test -e ${PKG_CACHE_PATH}/deb && cp ${PKG_CACHE_PATH}/deb/* /var/cache/apt/archives/
36+
fi
2137

2238
chmod -R a+rw $BUILDINFO_PATH
2339

‎src/sonic-build-hooks/scripts/utils.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ function FLOCK()
1717
eval $(echo exec {${lock_fd}}\<\>"${filename}.flock")
1818
#echo ${!lock_fd}
1919
if ! flock -x -w ${timeout} "${!lock_fd}" ; then
20-
echo "ERROR: Lock timeout trying to access ${filename}.flock";
20+
echo "ERROR: Lock timeout trying to access ${filename}.flock" 1>&2;
2121
exit 1;
2222
fi
2323
#echo "Lock acquired .."

0 commit comments

Comments
 (0)