diff --git a/Makefile.work b/Makefile.work index 1aeb87568640..9feb7985ee44 100644 --- a/Makefile.work +++ b/Makefile.work @@ -175,11 +175,23 @@ ifeq ($(ENABLE_FIPS), y) endif endif +SONIC_VERSION_CACHE := $(filter-out none,$(SONIC_VERSION_CACHE_METHOD)) +SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) +SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) +SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) +export SONIC_VERSION_CACHE SONIC_VERSION_CACHE_SOURCE +$(shell test -d $(SONIC_VERSION_CACHE_SOURCE) || \ + mkdir -p $(SONIC_VERSION_CACHE_SOURCE) && chmod -f 777 $(SONIC_VERSION_CACHE_SOURCE) 2>/dev/null ) + # Generate the version control build info $(shell \ SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \ TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \ scripts/generate_buildinfo_config.sh) # Generate the slave Dockerfile, and prepare build info for it @@ -198,6 +210,9 @@ $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \ PREPARE_DOCKER=BUILD_SLAVE=y \ DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \ + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ scripts/prepare_docker_buildinfo.sh \ $(SLAVE_BASE_IMAGE) \ $(SLAVE_DIR)/Dockerfile \ @@ -227,6 +242,10 @@ SLAVE_TAG = $(shell \ | awk '{print substr($$1,0,11);}') COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \ + SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \ + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ scripts/collect_docker_version_files.sh \ $(SLAVE_BASE_IMAGE) \ target \ @@ -287,6 +306,10 @@ ifneq ($(SONIC_DPKG_CACHE_SOURCE),) DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache:rw" endif +ifneq ($(SONIC_VERSION_CACHE_SOURCE),) + DOCKER_RUN += -v "$(SONIC_VERSION_CACHE_SOURCE):/vcache:rw" +endif + ifeq ($(SONIC_ENABLE_SECUREBOOT_SIGNATURE), y) ifneq ($(SIGNING_KEY),) DOCKER_SIGNING_SOURCE := $(shell dirname $(SIGNING_KEY)) @@ -380,6 +403,8 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \ --build-arg http_proxy=$(http_proxy) \ --build-arg https_proxy=$(https_proxy) \ --build-arg no_proxy=$(no_proxy) \ + --build-arg SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + --build-arg SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \ $(SLAVE_DIR) \ $(SPLIT_LOG) $(DOCKER_BASE_LOG) diff --git a/build_debian.sh b/build_debian.sh index 1486be65c0fd..846904efd4fd 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -81,7 +81,9 @@ echo '[INFO] Build host debian base system...' TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT # Prepare buildinfo -sudo scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy +sudo SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE} \ + scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy + sudo chown root:root $FILESYSTEM_ROOT @@ -635,7 +637,8 @@ if [[ $TARGET_BOOTLOADER == uboot ]]; then fi # Collect host image version files before cleanup -scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT +SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE} \ + scripts/collect_host_image_version_files.sh $CONFIGURED_ARCH $IMAGE_DISTRO $TARGET_PATH $FILESYSTEM_ROOT # Remove GCC sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove gcc diff --git a/rules/config b/rules/config index 9bb3326faa28..60a236901126 100644 --- a/rules/config +++ b/rules/config @@ -229,6 +229,15 @@ TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,htt # docker: docker base images SONIC_VERSION_CONTROL_COMPONENTS ?= none +# SONIC_VERSION_CACHE allows the .deb,.py, wget, git, docker and go files to be stored in the cache path. This allows the submodules to +# cache standard installation package and restored back to avoid the package download every time. +# SONIC_VERSION_CACHE - Method of deb package caching +# none : no caching +# cache : Use cache if exists else build the source and update the cache +# SONIC_VERSION_CACHE_SOURCE - Defines the version cache location details +SONIC_VERSION_CACHE_METHOD ?= none +SONIC_VERSION_CACHE_SOURCE ?= $(SONIC_DPKG_CACHE_SOURCE)/vcache + # SONiC docker registry # # Set the env variable ENABLE_DOCKER_BASE_PULL = y to enable pulling sonic-slave docker from registry diff --git a/rules/functions b/rules/functions index 62b3c8a98375..4c5ae8b5bfb4 100644 --- a/rules/functions +++ b/rules/functions @@ -47,8 +47,23 @@ log_green = echo -e "$(GREEN)$(1)$(GRAY)" FLUSH_LOG = rm -f $@.log -LOG_SIMPLE = &>> $(PROJECT_ROOT)/$@.log || { [ $$? -eq 0 ] || pushd $(PROJECT_ROOT) > /dev/null ; ./update_screen.sh -e $@ ; popd > /dev/null ; false ; } -LOG = < /dev/null |& $(PROJECT_ROOT)/scripts/process_log.sh $(PROCESS_LOG_OPTION) &>> $(PROJECT_ROOT)/$@.log ; test $${PIPESTATUS[-2]} -eq 0 || { [ $$? -eq 0 ] || pushd $(PROJECT_ROOT) > /dev/null ; ./update_screen.sh -e $@ ; popd > /dev/null ; false ; } +LOG_SIMPLE = &>> $(PROJECT_ROOT)/$@.log || \ + { \ + [ $$? -eq 0 ] \ + || pushd $(PROJECT_ROOT) > /dev/null ; \ + ./update_screen.sh -e $@ ; \ + popd > /dev/null ; \ + false ; \ + } +LOG = < /dev/null |& $(PROJECT_ROOT)/scripts/process_log.sh $(PROCESS_LOG_OPTION) &>> $(PROJECT_ROOT)/$@.log ; \ + test $${PIPESTATUS[-2]} -eq 0 || \ + { \ + [ $$? -eq 0 ] \ + || pushd $(PROJECT_ROOT) > /dev/null ; \ + ./update_screen.sh -e $@ ; \ + popd > /dev/null ; \ + false ; \ + } ############################################################################### ## Header and footer for each target diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh index 1f5cc544972c..2addf2800a11 100755 --- a/scripts/collect_docker_version_files.sh +++ b/scripts/collect_docker_version_files.sh @@ -1,6 +1,14 @@ #!/bin/bash -set -x +BUILDINFO_BASE=/usr/local/share/buildinfo + +SCRIPT_SRC_PATH=src/sonic-build-hooks +if [ -e ${SCRIPT_SRC_PATH} ]; then + . ${SCRIPT_SRC_PATH}/scripts/utils.sh +else + . ${BUILDINFO_BASE}/scripts/utils.sh +fi + DOCKER_IMAGE=$1 TARGET_PATH=$2 DOCKER_IMAGE_TAG=$3 diff --git a/scripts/collect_host_image_version_files.sh b/scripts/collect_host_image_version_files.sh index d0d7aabcf9d2..1adae8b648d7 100755 --- a/scripts/collect_host_image_version_files.sh +++ b/scripts/collect_host_image_version_files.sh @@ -1,7 +1,14 @@ #!/bin/bash -TARGET=$1 -FILESYSTEM_ROOT=$2 +SCRIPT_SRC_PATH=src/sonic-build-hooks +if [ -e ${SCRIPT_SRC_PATH} ]; then + . ${SCRIPT_SRC_PATH}/scripts/utils.sh +fi + +ARCH=$1 +DISTRO=$2 +TARGET=$3 +FILESYSTEM_ROOT=$4 VERSIONS_PATH=$TARGET/versions/host-image IMAGENAME="host-image" @@ -13,3 +20,5 @@ sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo ${IMAGENAME} cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/ cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/ + +sudo LANG=C chroot $FILESYSTEM_ROOT post_run_cleanup ${IMAGENAME} diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh index fe7657a6b6c9..b0ec54924246 100755 --- a/scripts/generate_buildinfo_config.sh +++ b/scripts/generate_buildinfo_config.sh @@ -6,5 +6,8 @@ BUILDINFO_CONFIG=$BUILDINFO_PATH/buildinfo/config/buildinfo.config mkdir -p $BUILDINFO_PATH/buildinfo/config -echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG -echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG +echo "export PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG +echo "export SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG +echo "export SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}" >> $BUILDINFO_CONFIG +echo "export SONIC_VERSION_CACHE_SOURCE=${SONIC_VERSION_CACHE_SOURCE}" >> $BUILDINFO_CONFIG +echo "export DISTRO=${DISTRO}" >> $BUILDINFO_CONFIG diff --git a/scripts/prepare_debian_image_buildinfo.sh b/scripts/prepare_debian_image_buildinfo.sh index 912e0de0b25f..2e5e1fedab62 100755 --- a/scripts/prepare_debian_image_buildinfo.sh +++ b/scripts/prepare_debian_image_buildinfo.sh @@ -4,6 +4,8 @@ ARCH=$1 DISTRO=$2 FILESYSTEM_ROOT=$3 +HOST_IMAGE_NAME=host-image +IMAGENAME=${HOST_IMAGE_NAME} . /usr/local/share/buildinfo/scripts/buildinfo_base.sh VERSION_DEB_PREFERENCE="01-versions-deb" @@ -26,4 +28,8 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then fi sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb" -sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo" +#sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo" +sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo export DISTRO=${DISTRO} >> /usr/local/share/buildinfo/config/buildinfo.config" +sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo export IMAGENAME=${IMAGENAME} >> /usr/local/share/buildinfo/config/buildinfo.config" + +sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo ${HOST_IMAGE_NAME}" diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 84222d1c0a3b..a07deff107ff 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -1,14 +1,24 @@ #!/bin/bash +BUILDINFO_BASE=/usr/local/share/buildinfo + +SCRIPT_SRC_PATH=src/sonic-build-hooks +if [ -e ${SCRIPT_SRC_PATH} ]; then + . ${SCRIPT_SRC_PATH}/scripts/utils.sh +else + . ${BUILDINFO_BASE}/scripts/utils.sh +fi + IMAGENAME=$1 DOCKERFILE=$2 ARCH=$3 -DOCKERFILE_TARGE=$4 +DOCKERFILE_TARGET=$4 DISTRO=$5 + [ -z "$BUILD_SLAVE" ] && BUILD_SLAVE=n -[ -z "$DOCKERFILE_TARGE" ] && DOCKERFILE_TARGE=$DOCKERFILE -DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE") +[ -z "$DOCKERFILE_TARGET" ] && DOCKERFILE_TARGET=$DOCKERFILE +DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGET") BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo" BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions" DOCKER_PATH=$(dirname $DOCKERFILE) @@ -34,15 +44,18 @@ fi scripts/docker_version_control.sh $@ DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo +ARG SONIC_VERSION_CACHE +ARG SONIC_VERSION_CONTROL_COMPONENTS COPY ["buildinfo", "/usr/local/share/buildinfo"] COPY vcache/ /sonic/target/vcache/'${IMAGENAME}' RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb ENV IMAGENAME='${IMAGENAME}' +ENV DISTRO='${DISTRO}' RUN pre_run_buildinfo '${IMAGENAME}' ' # Add the auto-generate code if it is not added in the target Dockerfile -if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then +if [ ! -f $DOCKERFILE_TARGET ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGET; then # Insert the docker build script before the RUN command LINE_NUMBER=$(grep -Fn -m 1 'RUN' $DOCKERFILE | cut -d: -f1) TEMP_FILE=$(mktemp) @@ -50,12 +63,14 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE # Append the docker build script at the end of the docker file echo -e "\nRUN post_run_buildinfo ${IMAGENAME} " >> $TEMP_FILE + echo -e "\nRUN post_run_cleanup ${IMAGENAME} " >> $TEMP_FILE - cat $TEMP_FILE > $DOCKERFILE_TARGE + cat $TEMP_FILE > $DOCKERFILE_TARGET rm -f $TEMP_FILE fi # Copy the build info config +mkdir -p ${BUILDINFO_PATH} cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH # Generate the version lock files diff --git a/scripts/prepare_slave_container_buildinfo.sh b/scripts/prepare_slave_container_buildinfo.sh index 1fb2f006640b..8840496593f0 100755 --- a/scripts/prepare_slave_container_buildinfo.sh +++ b/scripts/prepare_slave_container_buildinfo.sh @@ -5,10 +5,13 @@ ARCH=$2 DISTRO=$3 # Install the latest debian package sonic-build-hooks in the slave container -sudo dpkg -i --force-overwrite $SLAVE_DIR/buildinfo/sonic-build-hooks_*.deb > /dev/null +sudo dpkg -i --force-overwrite $SLAVE_DIR/buildinfo/sonic-build-hooks_*.deb &> /dev/null # Enable the build hooks -symlink_build_hooks +sudo symlink_build_hooks + +# set the global permissions +sudo chmod -f 777 /usr/local/share/buildinfo/ -R # Build the slave running config cp -rf $SLAVE_DIR/buildinfo/* /usr/local/share/buildinfo/ @@ -21,8 +24,8 @@ apt-get update > /dev/null 2>&1 # Build the slave version config [ -d /usr/local/share/buildinfo/versions ] && rm -rf /usr/local/share/buildinfo/versions scripts/versions_manager.py generate -t "/usr/local/share/buildinfo/versions" -n "build-${SLAVE_DIR}" -d "$DISTRO" -a "$ARCH" -touch ${BUILDINFO_PATH}/versions/versions-deb +touch ${BUILDINFO_PATH}/versions/versions-deb ${BUILDINFO_PATH}/versions/versions-web -rm -f /etc/apt/preferences.d/01-versions-deb -([ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]) && cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/ +sudo rm -f /etc/apt/preferences.d/01-versions-deb +([ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]) && sudo cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/ exit 0 diff --git a/slave.mk b/slave.mk index e8be0fcde8ea..962ea5c60281 100644 --- a/slave.mk +++ b/slave.mk @@ -383,6 +383,11 @@ $(info "INCLUDE_PDE" : "$(INCLUDE_PDE)") $(info "SONIC_DEBUGGING_ON" : "$(SONIC_DEBUGGING_ON)") $(info "SONIC_PROFILING_ON" : "$(SONIC_PROFILING_ON)") $(info "KERNEL_PROCURE_METHOD" : "$(KERNEL_PROCURE_METHOD)") +$(info "SONIC_VERSION_CONTROL_COMPONENTS": "$(SONIC_VERSION_CONTROL_COMPONENTS)") +$(info "SONIC_VERSION_CACHE_METHOD" : "$(SONIC_VERSION_CACHE)") +ifneq ($(SONIC_VERSION_CACHE),) +$(info "SONIC_VERSION_CACHE_SOURCE" : "$(SONIC_VERSION_CACHE_SOURCE)") +endif $(info "BUILD_TIMESTAMP" : "$(BUILD_TIMESTAMP)") $(info "BUILD_LOG_TIMESTAMP" : "$(BUILD_LOG_TIMESTAMP)") $(info "SONIC_IMAGE_VERSION" : "$(SONIC_IMAGE_VERSION)") @@ -410,7 +415,6 @@ $(info "ENABLE_NATIVE_WRITE" : "$(ENABLE_NATIVE_WRITE)") $(info "ENABLE_AUTO_TECH_SUPPORT" : "$(ENABLE_AUTO_TECH_SUPPORT)") $(info "PDDF_SUPPORT" : "$(PDDF_SUPPORT)") $(info "MULTIARCH_QEMU_ENVIRON" : "$(MULTIARCH_QEMU_ENVIRON)") -$(info "SONIC_VERSION_CONTROL_COMPONENTS": "$(SONIC_VERSION_CONTROL_COMPONENTS)") $(info "ENABLE_ASAN" : "$(ENABLE_ASAN)") $(info "DEFAULT_CONTAINER_REGISTRY" : "$(SONIC_DEFAULT_CONTAINER_REGISTRY)") ifeq ($(CONFIGURED_PLATFORM),vs) @@ -423,8 +427,9 @@ $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH)) endif # Overwrite the buildinfo in slave container -$(shell sudo scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV)) - +ifeq ($(filter clean,$(MAKECMDGOALS)),) +$(shell scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV)) +endif include Makefile.cache ifeq ($(SONIC_USE_DOCKER_BUILDKIT),y) @@ -558,7 +563,7 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_ONLINE_DEBS)) : $(DEBS_PATH)/% : .platform \ if [ -z '$($*_CACHE_LOADED)' ] ; then $(foreach deb,$* $($*_DERIVED_DEBS), \ - { curl -L -f -o $(DEBS_PATH)/$(deb) $($(deb)_CURL_OPTIONS) $($(deb)_URL) $(LOG) || { exit 1 ; } } ; ) + { SKIP_BUILD_HOOK=$($*_SKIP_VERSION) curl -L -f -o $(DEBS_PATH)/$(deb) $($(deb)_CURL_OPTIONS) $($(deb)_URL) $(LOG) || { exit 1 ; } } ; ) # Save the target deb into DPKG cache $(call SAVE_CACHE,$*,$@) @@ -575,7 +580,7 @@ SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_ONLINE_DEBS)) # SONIC_ONLINE_FILES += $(SOME_NEW_FILE) $(addprefix $(FILES_PATH)/, $(SONIC_ONLINE_FILES)) : $(FILES_PATH)/% : .platform $(HEADER) - curl -L -f -o $@ $($*_CURL_OPTIONS) $($*_URL) $(LOG) + SKIP_BUILD_HOOK=$($*_SKIP_VERSION) curl -L -f -o $@ $($*_CURL_OPTIONS) $($*_URL) $(LOG) $(FOOTER) SONIC_TARGET_LIST += $(addprefix $(FILES_PATH)/, $(SONIC_ONLINE_FILES)) @@ -883,7 +888,7 @@ $(SONIC_INSTALL_WHEELS) : $(PYTHON_WHEELS_PATH)/%-install : .platform $$(addsuff while true; do if mkdir $(PYTHON_WHEELS_PATH)/pip_lock &> /dev/null; then ifneq ($(CROSS_BUILD_ENVIRON),y) - { sudo -E pip$($*_PYTHON_VERSION) install $(PYTHON_WHEELS_PATH)/$* $(LOG) && rm -d $(PYTHON_WHEELS_PATH)/pip_lock && break; } || { rm -d $(PYTHON_WHEELS_PATH)/pip_lock && exit 1 ; } + { sudo -E SKIP_BUILD_HOOK=Y pip$($*_PYTHON_VERSION) install $(PYTHON_WHEELS_PATH)/$* $(LOG) && rm -d $(PYTHON_WHEELS_PATH)/pip_lock && break; } || { rm -d $(PYTHON_WHEELS_PATH)/pip_lock && exit 1 ; } else # Link python script and data expected location to the cross python virtual env istallation locations { PATH=$(VIRTENV_BIN_CROSS_PYTHON$($*_PYTHON_VERSION)):${PATH} sudo -E $(VIRTENV_BIN_CROSS_PYTHON$($*_PYTHON_VERSION))/pip$($*_PYTHON_VERSION) install $(PYTHON_WHEELS_PATH)/$* $(LOG) && $(if $(findstring $(SONIC_CONFIG_ENGINE_PY3),$*),(sudo ln -s $(VIRTENV_BIN_CROSS_PYTHON$($*_PYTHON_VERSION))/sonic-cfggen /usr/local/bin/sonic-cfggen 2>/dev/null || true), true ) && $(if $(findstring $(SONIC_YANG_MODELS_PY3),$*),(sudo ln -s $(VIRTENV_BASE_CROSS_PYTHON3)/yang-models /usr/local/yang-models 2>/dev/null || true), true ) && rm -d $(PYTHON_WHEELS_PATH)/pip_lock && break; } || { rm -d $(PYTHON_WHEELS_PATH)/pip_lock && exit 1 ; } @@ -911,7 +916,12 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g # Apply series of patches if exist if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && QUILT_PATCHES=../$(notdir $($*.gz_PATH)).patch quilt push -a; popd; fi # Prepare docker build info - scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(TARGET_DOCKERFILE)/Dockerfile.buildinfo + SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \ + TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ + scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(TARGET_DOCKERFILE)/Dockerfile.buildinfo $(LOG) docker info $(LOG) docker build --squash --no-cache \ --build-arg http_proxy=$(HTTP_PROXY) \ @@ -926,9 +936,11 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g -t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG) if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi - scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG) + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH} \ + scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG) $(call docker-image-save,$*,$@) + # Clean up if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi $(FOOTER) @@ -1030,7 +1042,10 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \ TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ - scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ + scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(LOG) docker info $(LOG) docker build --squash --no-cache \ --build-arg http_proxy=$(HTTP_PROXY) \ @@ -1042,6 +1057,8 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform --build-arg docker_container_name=$($*.gz_CONTAINER_NAME) \ --build-arg frr_user_uid=$(FRR_USER_UID) \ --build-arg frr_user_gid=$(FRR_USER_GID) \ + --build-arg SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + --build-arg SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \ --build-arg image_version=$(SONIC_IMAGE_VERSION) \ --label com.azure.sonic.manifest="$$(cat $($*.gz_PATH)/manifest.json)" \ --label Tag=$(SONIC_IMAGE_VERSION) \ @@ -1049,10 +1066,13 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform -t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG) if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi - scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG) + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + DISTRO=$(BLDENV) ARCH=${CONFIGURED_ARCH}\ + scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG) if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi $(call docker-image-save,$*,$@) + # Clean up if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi @@ -1092,7 +1112,10 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \ TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ - scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile-dbg $(CONFIGURED_ARCH) + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ + scripts/prepare_docker_buildinfo.sh $*-dbg $($*.gz_PATH)/Dockerfile-dbg $(CONFIGURED_ARCH) $(LOG) docker info $(LOG) docker build \ $(if $($*.gz_DBG_DEPENDS), --squash --no-cache, --no-cache) \ @@ -1100,16 +1123,21 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG --build-arg https_proxy=$(HTTPS_PROXY) \ --build-arg no_proxy=$(NO_PROXY) \ --build-arg docker_container_name=$($*.gz_CONTAINER_NAME) \ + --build-arg SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + --build-arg SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \ --label com.azure.sonic.manifest="$$(cat $($*.gz_PATH)/manifest.json)" \ --label Tag=$(SONIC_IMAGE_VERSION) \ --file $($*.gz_PATH)/Dockerfile-dbg \ -t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG) if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi - scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile-dbg $(LOG) + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) + DISTRO=$(BLDENV) ARCH=$(CONFIGURED_ARCH) \ + scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile-dbg $(LOG) if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi $(call docker-image-save,$*-$(DBG_IMAGE_MARK),$@) + # Clean up docker rmi -f $(DOCKER_IMAGE_REF) &> /dev/null || true if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi @@ -1398,6 +1426,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ SIGNING_KEY="$(SIGNING_KEY)" \ SIGNING_CERT="$(SIGNING_CERT)" \ PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \ CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \ MASTER_KUBERNETES_VERSION=$(MASTER_KUBERNETES_VERSION) \ @@ -1466,7 +1495,7 @@ SONIC_CLEAN_TARGETS += $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \ $(SONIC_SIMPLE_DOCKER_IMAGES) \ $(SONIC_INSTALLERS))) $(SONIC_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform - $(Q)rm -f $(TARGET_PATH)/$* + $(Q)rm -f $(TARGET_PATH)/$* target/versions/dockers/$(subst .gz,,$*) SONIC_CLEAN_STDEB_DEBS = $(addsuffix -clean,$(addprefix $(PYTHON_DEBS_PATH)/, \ $(SONIC_PYTHON_STDEB_DEBS))) @@ -1480,8 +1509,13 @@ $(SONIC_CLEAN_WHEELS) :: $(PYTHON_WHEELS_PATH)/%-clean : .platform clean-logs :: .platform $(Q)rm -f $(TARGET_PATH)/*.log $(DEBS_PATH)/*.log $(FILES_PATH)/*.log $(PYTHON_DEBS_PATH)/*.log $(PYTHON_WHEELS_PATH)/*.log +clean-versions :: .platform + @rm -rf target/versions/* + +vclean:: .platform + @sudo rm -rf target/vcache/* target/baseimage* -clean :: .platform clean-logs $$(SONIC_CLEAN_DEBS) $$(SONIC_CLEAN_FILES) $$(SONIC_CLEAN_TARGETS) $$(SONIC_CLEAN_STDEB_DEBS) $$(SONIC_CLEAN_WHEELS) +clean :: .platform clean-logs clean-versions $$(SONIC_CLEAN_DEBS) $$(SONIC_CLEAN_FILES) $$(SONIC_CLEAN_TARGETS) $$(SONIC_CLEAN_STDEB_DEBS) $$(SONIC_CLEAN_WHEELS) ############################################################################### ## all diff --git a/src/sonic-build-hooks/Makefile b/src/sonic-build-hooks/Makefile index b45be9dd07f6..f20acac0b143 100644 --- a/src/sonic-build-hooks/Makefile +++ b/src/sonic-build-hooks/Makefile @@ -5,7 +5,7 @@ SONIC_BUILD_HOOKS_PACKAGE = $(SONIC_BUILD_HOOKS)_$(SONIC_BUILD_HOOKS_VERSION)_al BUILDINFO_DIR = buildinfo TMP_DIR = tmp SYMBOL_LINKS_SRC_DIR = ../../usr/local/share/buildinfo/scripts -SYMBOL_LINKS = symlink_build_hooks post_run_buildinfo pre_run_buildinfo collect_version_files +SYMBOL_LINKS = symlink_build_hooks post_run_buildinfo pre_run_buildinfo collect_version_files post_run_cleanup SONIC_BUILD_HOOKS_TARGET = $(BUILDINFO_DIR)/$(SONIC_BUILD_HOOKS_PACKAGE) BUILD_ROOT_DIR = $(TMP_DIR)/$(SONIC_BUILD_HOOKS) DEBIAN_DIR = $(BUILD_ROOT_DIR)/DEBIAN diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh index 1494993448b8..08cc63cd07ad 100755 --- a/src/sonic-build-hooks/scripts/buildinfo_base.sh +++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh @@ -23,6 +23,7 @@ fi PKG_CACHE_FILE_NAME=${PKG_CACHE_PATH}/cache.tgz mkdir -p ${PKG_CACHE_PATH} +. ${BUILDINFO_PATH}/scripts/utils.sh URL_PREFIX=$(echo "${PACKAGE_URL_PREFIX}" | sed -E "s#(//[^/]*/).*#\1#") @@ -76,6 +77,28 @@ check_if_url_exist() fi } +get_version_cache_option() +{ + #SONIC_VERSION_CACHE="cache" + if [ ! -z ${SONIC_VERSION_CACHE} ]; then + if [ ${SONIC_VERSION_CACHE} == "rcache" ]; then + echo -n "rcache" + elif [ ${SONIC_VERSION_CACHE} == "wcache" ]; then + echo -n "wcache" + elif [ ${SONIC_VERSION_CACHE} == "cache" ]; then + echo -n "wcache" + else + echo -n "" + return 1 + fi + echo -n "" + return 0 + fi + echo -n "" + return 1 +} + + # Enable or disable the reproducible mirrors set_reproducible_mirrors() { @@ -331,4 +354,8 @@ ENABLE_VERSION_CONTROL_PY2=$(check_version_control "py2") ENABLE_VERSION_CONTROL_PY3=$(check_version_control "py3") ENABLE_VERSION_CONTROL_WEB=$(check_version_control "web") ENABLE_VERSION_CONTROL_GIT=$(check_version_control "git") +ENABLE_VERSION_CONTROL_PIP=$(check_version_control "pip") +ENABLE_VERSION_CONTROL_PYTHON=$(check_version_control "python") +ENABLE_VERSION_CONTROL_EASY_INSTALL=$(check_version_control "easy_install") +ENABLE_VERSION_CONTROL_GO=$(check_version_control "go") ENABLE_VERSION_CONTROL_DOCKER=$(check_version_control "docker") diff --git a/src/sonic-build-hooks/scripts/collect_version_files b/src/sonic-build-hooks/scripts/collect_version_files index a4b33eeaa897..ae772c167020 100755 --- a/src/sonic-build-hooks/scripts/collect_version_files +++ b/src/sonic-build-hooks/scripts/collect_version_files @@ -1,8 +1,9 @@ #!/bin/bash +TARGET_PATH=$1 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh -TARGET_PATH=$1 +[ -d ${TARGET_PATH} ] && rm -rf ${TARGET_PATH} [ -z "$TARGET_PATH" ] && TARGET_PATH=$POST_VERSION_PATH ARCH=$(dpkg --print-architecture) DIST=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) @@ -15,6 +16,9 @@ dpkg-query -W -f '${Package}==${Version}\n' >> "${TARGET_PATH}/versions-deb-${DI ([ -x "/usr/local/bin/pip2" ] || [ -x "/usr/bin/pip2" ]) && pip2 freeze >> "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" ([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze >> "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" +[ -f "${BUILD_WEB_VERSION_FILE}" ] && cp ${BUILD_WEB_VERSION_FILE} ${TARGET_PATH} +[ -f "${BUILD_GIT_VERSION_FILE}" ] && cp ${BUILD_GIT_VERSION_FILE} ${TARGET_PATH} + ## Add the the packages purged [ -f $POST_VERSION_PATH/purge-versions-deb ] && cat $POST_VERSION_PATH/purge-versions-deb >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" diff --git a/src/sonic-build-hooks/scripts/post_run_buildinfo b/src/sonic-build-hooks/scripts/post_run_buildinfo index 00f2b0722dab..878805f51729 100755 --- a/src/sonic-build-hooks/scripts/post_run_buildinfo +++ b/src/sonic-build-hooks/scripts/post_run_buildinfo @@ -4,6 +4,9 @@ IMAGENAME=$1 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +[ -d $POST_VERSION_PATH ] && rm -rf $POST_VERSION_PATH + # Collect the version files collect_version_files $POST_VERSION_PATH @@ -11,8 +14,5 @@ collect_version_files $POST_VERSION_PATH tar -C ${PKG_CACHE_PATH} --exclude=cache.tgz -zcvf /cache.tgz . # Disable the build hooks -symlink_build_hooks -d -set_reproducible_mirrors -d - -# Remove the version deb preference -rm -f $VERSION_DEB_PREFERENCE +# symlink_build_hooks -d +# set_reproducible_mirrors -d diff --git a/src/sonic-build-hooks/scripts/post_run_cleanup b/src/sonic-build-hooks/scripts/post_run_cleanup new file mode 100755 index 000000000000..e8e428e263e6 --- /dev/null +++ b/src/sonic-build-hooks/scripts/post_run_cleanup @@ -0,0 +1,40 @@ +#!/bin/bash + +IMAGENAME=$1 + +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +#set -x + + +if [ ! -z "$(get_version_cache_option)" ]; then + + #Delete the rsync package files + if [[ ! ${IMAGENAME} =~ -slave- ]]; then + /usr/bin/apt-get purge -y --auto-remove rsync + fi +fi + +apt-get -s clean -y +apt-get -s autoclean -y +apt-get -s autoremove -y +#apt-get -s autoremove -y --purge +rm -f /var/cache/apt/archives/*.deb /var/cache/apt/*.bin + +if [[ ! ${IMAGENAME} =~ -slave- ]]; then + rm -f /var/lib/apt/lists/* +fi + +rm -rf /sonic/target /ssh +rm -f /tmp/* +rm -rf /debs /python-wheels ~/.cache +find / | grep -E "__pycache__" | xargs rm -rf + +rm -rf $BUILD_VERSION_PATH/* + +# Disable the build hooks +symlink_build_hooks -d +set_reproducible_mirrors -d + +# Remove the version deb preference +rm -f $VERSION_DEB_PREFERENCE diff --git a/src/sonic-build-hooks/scripts/utils.sh b/src/sonic-build-hooks/scripts/utils.sh new file mode 100644 index 000000000000..3f7a6c6d1a3b --- /dev/null +++ b/src/sonic-build-hooks/scripts/utils.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Lock macro for shared file access +# Parameters: +# $(1) - Lock file name +# $(2) - Timeout value +function FLOCK() +{ + local filename=$(dirname $1) + local timeout=${2:-360} + if [[ ! -f ${filename}.flock ]]; then + touch ${filename}.flock + chmod -f 777 ${filename}.flock; + fi + local lockname=$(basename ${filename}) + local lock_fd=lock_${lockname//[%.\/\-+~]/_}_fd + eval $(echo exec {${lock_fd}}\<\>"${filename}.flock") + #echo ${!lock_fd} + if ! flock -x -w ${timeout} "${!lock_fd}" ; then + echo "ERROR: Lock timeout trying to access ${filename}.flock" 2>&1; + exit 1; + fi + #echo "Lock acquired .." +} + +# UnLock macro for shared file access +# Parameters: +# $(1) - Lock file name +function FUNLOCK() +{ + local filename=$(dirname $1) + local lockname=$(basename ${filename}) + local lock_fd=lock_${lockname//[%.\/\-+~]/_}_fd + eval $(echo exec "${!lock_fd}<&-") + #rm -f ${filename}.flock +} +