From d5edc7b8c34044b9a50aa6a2d3e5c557c3c785f0 Mon Sep 17 00:00:00 2001
From: Kalimuthu Velappan <kalimuthu.velappan@broadcom.com>
Date: Fri, 2 Sep 2022 20:37:43 -0700
Subject: [PATCH] 02.Version cache - docker cache build framework

During docker build, host files can be passed to the docker build through
docker context files. But there is no straightforward way to transfer
the files from docker build to host.

This feature provides a tricky way to pass the cache contents from docker
build to host. It uses the multi-stage docker file to copy the cache
content, cleanup the temporary files and creates the final sonic docker image.
---
 Makefile.work                                   | 13 +++++++++----
 files/build_templates/build_docker_cache.j2     | 11 +++++++++++
 scripts/collect_build_version_files.sh          |  2 +-
 scripts/collect_docker_version_files.sh         | 17 +++++++++++++++--
 scripts/collect_host_image_version_files.sh     |  4 +++-
 scripts/prepare_docker_buildinfo.sh             | 13 +++++++++++--
 slave.mk                                        | 15 ++++++++++++---
 sonic-slave-buster/Dockerfile.j2                |  2 +-
 src/sonic-build-hooks/scripts/buildinfo_base.sh |  9 +++++++++
 .../scripts/post_run_buildinfo                  |  6 ++++--
 src/sonic-build-hooks/scripts/pre_run_buildinfo |  6 ++++++
 11 files changed, 82 insertions(+), 16 deletions(-)
 create mode 100644 files/build_templates/build_docker_cache.j2

diff --git a/Makefile.work b/Makefile.work
index adc40f381613..1aeb87568640 100644
--- a/Makefile.work
+++ b/Makefile.work
@@ -228,8 +228,12 @@ SLAVE_TAG = $(shell \
 
 COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
 	scripts/collect_docker_version_files.sh \
-	$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
-		target 
+		$(SLAVE_BASE_IMAGE) \
+		target \
+		$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
+		$(SLAVE_DIR) \
+		$(SLAVE_DIR)/Dockerfile
+
 OVERLAY_MODULE_CHECK := \
     lsmod | grep -q "^overlay " &>/dev/null || \
     zgrep -q 'CONFIG_OVERLAY_FS=y' /proc/config.gz &>/dev/null || \
@@ -376,7 +380,8 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \
 		    --build-arg http_proxy=$(http_proxy) \
 		    --build-arg https_proxy=$(https_proxy) \
 		    --build-arg no_proxy=$(no_proxy) \
-		    $(SLAVE_DIR) $(SPLIT_LOG) $(DOCKER_BASE_LOG)
+		    $(SLAVE_DIR) \
+		    $(SPLIT_LOG) $(DOCKER_BASE_LOG)
 
 DOCKER_BASE_PULL = docker pull \
 			$(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)
@@ -507,7 +512,7 @@ SONIC_BUILD_INSTRUCTION :=  $(MAKE) \
 
 
 ifeq ($(filter clean,$(MAKECMDGOALS)),)
-COLLECT_BUILD_VERSION = { DBGOPT='$(DBGOPT)' scripts/collect_build_version_files.sh \$$?; }
+COLLECT_BUILD_VERSION = { scripts/collect_build_version_files.sh \$$?; }
 endif
 
 ifdef SOURCE_FOLDER
diff --git a/files/build_templates/build_docker_cache.j2 b/files/build_templates/build_docker_cache.j2
new file mode 100644
index 000000000000..9f1d0eebf77f
--- /dev/null
+++ b/files/build_templates/build_docker_cache.j2
@@ -0,0 +1,11 @@
+# Base docker build
+FROM {{IMAGENAME}}
+
+# Copy the cache data to host
+From scratch as output
+COPY --from={{IMAGENAME}} /cache.tgz cache.tgz
+
+# Clean up the cache data
+FROM {{IMAGENAME}} as final
+RUN rm /cache.tgz
+
diff --git a/scripts/collect_build_version_files.sh b/scripts/collect_build_version_files.sh
index b650e421996b..e35ca0590334 100755
--- a/scripts/collect_build_version_files.sh
+++ b/scripts/collect_build_version_files.sh
@@ -23,6 +23,6 @@ mkdir -p $VERSION_SLAVE_PATH
 
 scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH
 
-rm -rf $BUILD_VERSION_PATH/*
+[ -d $BUILD_VERSION_PATH ] && rm -rf $BUILD_VERSION_PATH/*
 
 exit $RET
diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh
index 546d8458f259..1f5cc544972c 100755
--- a/scripts/collect_docker_version_files.sh
+++ b/scripts/collect_docker_version_files.sh
@@ -1,12 +1,17 @@
 #!/bin/bash
   
+set -x 
 DOCKER_IMAGE=$1
 TARGET_PATH=$2
+DOCKER_IMAGE_TAG=$3
+DOCKER_PATH=$4
+DOCKER_FILE=$5
 
 [ -z "$TARGET_PATH" ] && TARGET_PATH=./target
 
 DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1 | sed "s/-$DOCKER_USERNAME\$//")
-DOCKER_CONTAINER=$DOCKER_IMAGE_NAME
+#Create the container specific to the user tag and slave tag
+DOCKER_CONTAINER=${DOCKER_IMAGE_TAG/:/-}
 TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME
 
 [ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH
@@ -18,8 +23,16 @@ export DOCKER_CLI_EXPERIMENTAL=enabled
 if docker container inspect $DOCKER_IMAGE > /dev/null 2>&1; then
     docker container rm $DOCKER_IMAGE > /dev/null
 fi
-docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE
+docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE_TAG
 docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/
 docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/
 docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/
+
+# Save the cache contents from docker build
+IMAGENAME=${DOCKER_IMAGE_TAG} j2 files/build_templates/build_docker_cache.j2 > ${DOCKER_FILE}.cleanup
+docker tag ${DOCKER_IMAGE_TAG} tmp-${DOCKER_IMAGE_TAG}
+DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --target output -o target/vcache/${DOCKER_IMAGE_NAME} ${DOCKER_PATH}
+DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --no-cache --target final --tag ${DOCKER_IMAGE_TAG} ${DOCKER_PATH}
+docker rmi tmp-${DOCKER_IMAGE_TAG}
+
 docker container rm $DOCKER_CONTAINER
diff --git a/scripts/collect_host_image_version_files.sh b/scripts/collect_host_image_version_files.sh
index 2cabc049d9c7..d0d7aabcf9d2 100755
--- a/scripts/collect_host_image_version_files.sh
+++ b/scripts/collect_host_image_version_files.sh
@@ -3,11 +3,13 @@
 TARGET=$1
 FILESYSTEM_ROOT=$2
 VERSIONS_PATH=$TARGET/versions/host-image
+IMAGENAME="host-image"
 
 [ -d $VERSIONS_PATH ] && sudo rm -rf $VERSIONS_PATH
 mkdir -p $VERSIONS_PATH
 
-sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo
+mkdir -p target/vcache/${IMAGENAME}
+sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo ${IMAGENAME}
 
 cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/
 cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/
diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh
index 3c1104eddc53..92fc78c9e9d3 100755
--- a/scripts/prepare_docker_buildinfo.sh
+++ b/scripts/prepare_docker_buildinfo.sh
@@ -11,6 +11,7 @@ DISTRO=$5
 DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE")
 BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo"
 BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions"
+DOCKER_PATH=$(dirname $DOCKERFILE)
 
 [ -d $BUILDINFO_PATH ] && rm -rf $BUILDINFO_PATH
 mkdir -p $BUILDINFO_VERSION_PATH
@@ -31,8 +32,11 @@ scripts/docker_version_control.sh $@
 
 DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo
 COPY ["buildinfo", "/usr/local/share/buildinfo"]
+COPY vcache/ /sonic/target/vcache/'${IMAGENAME}'
 RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb
-RUN pre_run_buildinfo'
+ENV IMAGENAME='${IMAGENAME}'
+RUN pre_run_buildinfo '${IMAGENAME}'
+'
 
 # Add the auto-generate code if it is not added in the target Dockerfile
 if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then
@@ -42,7 +46,7 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE
     awk -v text="${DOCKERFILE_PRE_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE
 
     # Append the docker build script at the end of the docker file
-    echo -e "\nRUN post_run_buildinfo" >> $TEMP_FILE
+    echo -e "\nRUN post_run_buildinfo ${IMAGENAME} " >> $TEMP_FILE
 
     cat $TEMP_FILE > $DOCKERFILE_TARGE
     rm -f $TEMP_FILE
@@ -55,3 +59,8 @@ cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH
 scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH"
 
 touch $BUILDINFO_VERSION_PATH/versions-deb
+
+# Create the cache directories
+LOCAL_CACHE_DIR=target/vcache/${IMAGENAME}
+mkdir -p ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
+chmod -f 777 ${LOCAL_CACHE_DIR} ${DOCKER_PATH}/vcache/
diff --git a/slave.mk b/slave.mk
index ae3c6b4dcc3d..8f6ec448f710 100644
--- a/slave.mk
+++ b/slave.mk
@@ -111,6 +111,7 @@ configure :
 	$(Q)mkdir -p $(PYTHON_DEBS_PATH)
 	$(Q)mkdir -p $(PYTHON_WHEELS_PATH)
 	$(Q)mkdir -p $(DPKG_ADMINDIR_PATH)
+	$(Q)mkdir -p $(TARGET_PATH)/vcache
 	$(Q)echo $(PLATFORM) > .platform
 	$(Q)echo $(PLATFORM_ARCH) > .arch
 
@@ -914,8 +915,10 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
 		--label Tag=$(SONIC_IMAGE_VERSION) \
 		-f $(TARGET_DOCKERFILE)/Dockerfile.buildinfo \
 		-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
+
 	if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
-	scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH)
+	scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
+
 	$(call docker-image-save,$*,$@)
 	# Clean up
 	if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
@@ -993,6 +996,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
 		mkdir -p $($*.gz_PATH)/files $(LOG)
 		mkdir -p $($*.gz_PATH)/python-debs $(LOG)
 		mkdir -p $($*.gz_PATH)/python-wheels $(LOG)
+		mkdir -p $(TARGET_PATH)/vcache/$* $($*.gz_PATH)/vcache $(LOG)
 		sudo mount --bind $($*.gz_DEBS_PATH) $($*.gz_PATH)/debs $(LOG)
 		sudo mount --bind $($*.gz_FILES_PATH) $($*.gz_PATH)/files $(LOG)
 		sudo mount --bind $(PYTHON_DEBS_PATH) $($*.gz_PATH)/python-debs $(LOG)
@@ -1034,9 +1038,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
 			--label Tag=$(SONIC_IMAGE_VERSION) \
 		        $($(subst -,_,$(notdir $($*.gz_PATH)))_labels) \
 			-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
+
 		if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
-		scripts/collect_docker_version_files.sh $(DOCKER_IMAGE_REF) $(TARGET_PATH)
+		scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG)
 		if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi
+
 		$(call docker-image-save,$*,$@)
 		# Clean up
 		if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi
@@ -1065,6 +1071,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
 
 		mkdir -p $($*.gz_PATH)/debs $(LOG)
 		sudo mount --bind $($*.gz_DEBS_PATH) $($*.gz_PATH)/debs $(LOG)
+		mkdir -p $(TARGET_PATH)/vcache/$*-dbg $($*.gz_PATH)/vcache $(LOG)
 		# Export variables for j2. Use path for unique variable names, e.g. docker_orchagent_debs
 		$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_debs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_DEPENDS),RDEPENDS))\n" | awk '!a[$$0]++'))
 		$(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_IMAGE_PACKAGES)))\n" | awk '!a[$$0]++'))
@@ -1088,9 +1095,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
 			--label Tag=$(SONIC_IMAGE_VERSION) \
 			--file $($*.gz_PATH)/Dockerfile-dbg \
 			-t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG)
+
 		if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
-		scripts/collect_docker_version_files.sh $(DOCKER_DBG_IMAGE_REF) $(TARGET_PATH)
+		scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH)  $($*.gz_PATH)/Dockerfile-dbg $(LOG)
 		if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi
+
 		$(call docker-image-save,$*-$(DBG_IMAGE_MARK),$@)
 		# Clean up
 		docker rmi -f $(DOCKER_IMAGE_REF) &> /dev/null || true
diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
index 73e9febda14b..0c2ce405f2ad 100644
--- a/sonic-slave-buster/Dockerfile.j2
+++ b/sonic-slave-buster/Dockerfile.j2
@@ -599,7 +599,7 @@ RUN add-apt-repository \
            $(lsb_release -cs) \
            stable"
 RUN apt-get update
-RUN apt-get install -y docker-ce=5:18.09.5~3-0~debian-buster docker-ce-cli=5:18.09.5~3-0~debian-buster
+RUN apt-get install -y docker-ce=5:20.10.21~3-0~debian-buster docker-ce-cli=5:20.10.21~3-0~debian-buster
 RUN echo "DOCKER_OPTS=\"--experimental --storage-driver=vfs {{ DOCKER_EXTRA_OPTS }}\"" >> /etc/default/docker
 RUN update-alternatives --set iptables /usr/sbin/iptables-legacy
 
diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh
index ff249dfdfb91..1494993448b8 100755
--- a/src/sonic-build-hooks/scripts/buildinfo_base.sh
+++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh
@@ -15,6 +15,15 @@ REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/debian'
 DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock
 
 . $BUILDINFO_PATH/config/buildinfo.config
+if [ -e /vcache ]; then
+	PKG_CACHE_PATH=/vcache/${IMAGENAME}
+else
+	PKG_CACHE_PATH=/sonic/target/vcache/${IMAGENAME}
+fi
+PKG_CACHE_FILE_NAME=${PKG_CACHE_PATH}/cache.tgz
+mkdir -p ${PKG_CACHE_PATH}
+
+
 
 URL_PREFIX=$(echo "${PACKAGE_URL_PREFIX}" | sed -E "s#(//[^/]*/).*#\1#")
 
diff --git a/src/sonic-build-hooks/scripts/post_run_buildinfo b/src/sonic-build-hooks/scripts/post_run_buildinfo
index 97f47f7efcf1..00f2b0722dab 100755
--- a/src/sonic-build-hooks/scripts/post_run_buildinfo
+++ b/src/sonic-build-hooks/scripts/post_run_buildinfo
@@ -1,12 +1,14 @@
 #!/bin/bash
 
+IMAGENAME=$1
+
 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh
 
 # Collect the version files
 collect_version_files $POST_VERSION_PATH
 
-[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $POST_VERSION_PATH
-rm -rf $BUILD_VERSION_PATH/*
+#Save the cache file for exporting it to host.
+tar -C ${PKG_CACHE_PATH} --exclude=cache.tgz -zcvf /cache.tgz .
 
 # Disable the build hooks
 symlink_build_hooks -d
diff --git a/src/sonic-build-hooks/scripts/pre_run_buildinfo b/src/sonic-build-hooks/scripts/pre_run_buildinfo
index 5a8f00b55ecb..eb4d04225ec5 100755
--- a/src/sonic-build-hooks/scripts/pre_run_buildinfo
+++ b/src/sonic-build-hooks/scripts/pre_run_buildinfo
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+IMAGENAME=$1
+
 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh
 
 [ -d $DIFF_VERSION_PATH ] && rm -rf $DIFF_VERSION_PATH
@@ -15,10 +17,14 @@ update_version_files
 symlink_build_hooks
 set_reproducible_mirrors
 
+mkdir -p /var/cache/apt/archives/
+
 chmod -R a+rw $BUILDINFO_PATH
 
 if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]; then
     cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/
 fi
 
+DISTRO=${DISTRO} apt-get update && apt-get install -y rsync
+
 exit 0