Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

01.Version-cache - restructuring of Makefile.work #12000

Merged
merged 1 commit into from
Oct 4, 2022
Merged

01.Version-cache - restructuring of Makefile.work #12000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 21 additions & 13 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,14 @@ NOSTRETCH ?= 0
NOBUSTER ?= 0
NOBULLSEYE ?= 0

override Q := @
ifeq ($(QUIET),n)
override Q :=
endif
override SONIC_OVERRIDE_BUILD_VARS += $(SONIC_BUILD_VARS)
override SONIC_OVERRIDE_BUILD_VARS += Q=$(Q)
export Q SONIC_OVERRIDE_BUILD_VARS

ifeq ($(NOJESSIE),0)
BUILD_JESSIE=1
endif
Expand All @@ -29,50 +37,50 @@ PLATFORM_CHECKOUT_CMD := $(shell if [ -f $(PLATFORM_CHECKOUT_FILE) ]; then PLATF
%::
@echo "+++ --- Making $@ --- +++"
ifeq ($(NOJESSIE), 0)
EXTRA_DOCKER_TARGETS=$(notdir $@) make -f Makefile.work jessie
EXTRA_DOCKER_TARGETS=$(notdir $@) $(MAKE) -f Makefile.work jessie
endif
ifeq ($(NOSTRETCH), 0)
EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch make -f Makefile.work stretch
EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch $(MAKE) -f Makefile.work stretch
endif
ifeq ($(NOBUSTER), 0)
EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=buster make -f Makefile.work buster
EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=buster $(MAKE) -f Makefile.work buster
endif
ifeq ($(NOBULLSEYE), 0)
BLDENV=bullseye make -f Makefile.work $@
BLDENV=bullseye $(MAKE) -f Makefile.work $@
endif
BLDENV=bullseye make -f Makefile.work docker-cleanup
BLDENV=bullseye $(MAKE) -f Makefile.work docker-cleanup

jessie:
@echo "+++ Making $@ +++"
ifeq ($(NOJESSIE), 0)
make -f Makefile.work jessie
$(MAKE) -f Makefile.work jessie
endif

stretch:
@echo "+++ Making $@ +++"
ifeq ($(NOSTRETCH), 0)
make -f Makefile.work stretch
$(MAKE) -f Makefile.work stretch
endif

buster:
@echo "+++ Making $@ +++"
ifeq ($(NOBUSTER), 0)
make -f Makefile.work buster
$(MAKE) -f Makefile.work buster
endif

init:
@echo "+++ Making $@ +++"
make -f Makefile.work $@
$(MAKE) -f Makefile.work $@

#
# Function to invoke target $@ in Makefile.work with proper BLDENV
#
define make_work
@echo "+++ Making $@ +++"
$(if $(BUILD_JESSIE),make -f Makefile.work $@,)
$(if $(BUILD_STRETCH),BLDENV=stretch make -f Makefile.work $@,)
$(if $(BUILD_BUSTER),BLDENV=buster make -f Makefile.work $@,)
$(if $(BUILD_BULLSEYE),BLDENV=bullseye make -f Makefile.work $@,)
$(if $(BUILD_JESSIE),$(MAKE) -f Makefile.work $@,)
$(if $(BUILD_STRETCH),BLDENV=stretch $(MAKE) -f Makefile.work $@,)
$(if $(BUILD_BUSTER),BLDENV=buster $(MAKE) -f Makefile.work $@,)
$(if $(BUILD_BULLSEYE),BLDENV=bullseye $(MAKE) -f Makefile.work $@,)
endef

.PHONY: $(PLATFORM_PATH)
Expand Down
213 changes: 142 additions & 71 deletions Makefile.work
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ endif
# Define a do-nothing target for rules/config.user so that when
# the file is missing, make won't try to rebuld everything.
rules/config.user:
@echo -n ""
$(Q)echo -n ""

include rules/config
-include rules/config.user
Expand Down Expand Up @@ -173,21 +173,59 @@ endif
endif

# Generate the version control build info
$(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
scripts/generate_buildinfo_config.sh)
$(shell \
SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
scripts/generate_buildinfo_config.sh)

# Generate the slave Dockerfile, and prepare build info for it
$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) ENABLE_FIPS_FEATURE=$(ENABLE_FIPS_FEATURE) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) j2 $(SLAVE_DIR)/Dockerfile.user.j2 > $(SLAVE_DIR)/Dockerfile.user)
$(shell BUILD_SLAVE=y DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \
MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \
CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \
ENABLE_FIPS_FEATURE=$(ENABLE_FIPS_FEATURE) \
DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) \
DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)

$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \
MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \
CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \
j2 $(SLAVE_DIR)/Dockerfile.user.j2 > $(SLAVE_DIR)/Dockerfile.user)

PREPARE_DOCKER=BUILD_SLAVE=y \
DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
scripts/prepare_docker_buildinfo.sh \
$(SLAVE_BASE_IMAGE) \
$(SLAVE_DIR)/Dockerfile \
$(CONFIGURED_ARCH) \
"" \
$(BLDENV)

$(shell $(PREPARE_DOCKER) )

# Add the versions in the tag, if the version change, need to rebuild the slave
SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* src/sonic-build-hooks/hooks/* | sha1sum | awk '{print substr($$1,0,11);}')
# Calculate the slave TAG based on $(USER)/$(PWD)/$(CONFIGURED_PLATFORM) to get unique SHA ID
SLAVE_TAG = $(shell (cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* .git/HEAD && echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
| sha1sum | awk '{print substr($$1,0,11);}')
SLAVE_BASE_TAG = $(shell \
cat $(SLAVE_DIR)/Dockerfile \
$(SLAVE_DIR)/buildinfo/versions/versions-* \
src/sonic-build-hooks/hooks/* 2>/dev/null \
| sha1sum \
| awk '{print substr($$1,0,11);}')

# Calculate the slave TAG based on $(USER)/$(PWD)/$(CONFIGURED_PLATFORM) to get unique SHA ID
SLAVE_TAG = $(shell \
(cat $(SLAVE_DIR)/Dockerfile.user \
$(SLAVE_DIR)/Dockerfile \
$(SLAVE_DIR)/buildinfo/versions/versions-* \
.git/HEAD \
&& echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
| sha1sum \
| awk '{print substr($$1,0,11);}')

COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
scripts/collect_docker_version_files.sh \
$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
target
OVERLAY_MODULE_CHECK := \
lsmod | grep -q "^overlay " &>/dev/null || \
zgrep -q 'CONFIG_OVERLAY_FS=y' /proc/config.gz &>/dev/null || \
Expand Down Expand Up @@ -329,7 +367,7 @@ DOCKER_BASE_LOG = $(SLAVE_DIR)/$(SLAVE_BASE_IMAGE)_$(SLAVE_BASE_TAG).log
DOCKER_LOG = $(SLAVE_DIR)/$(SLAVE_IMAGE)_$(SLAVE_TAG).log


DOCKER_BASE_BUILD = docker build --no-cache \
DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \
-t $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
--build-arg http_proxy=$(http_proxy) \
--build-arg https_proxy=$(https_proxy) \
Expand All @@ -339,7 +377,7 @@ DOCKER_BASE_BUILD = docker build --no-cache \
DOCKER_BASE_PULL = docker pull \
$(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)

DOCKER_BUILD = docker build --no-cache \
DOCKER_USER_BUILD = docker build --no-cache \
--build-arg user=$(USER) \
--build-arg uid=$(shell id -u) \
--build-arg guid=$(shell id -g) \
Expand All @@ -349,7 +387,52 @@ DOCKER_BUILD = docker build --no-cache \
-f $(SLAVE_DIR)/Dockerfile.user \
$(SLAVE_DIR) $(SPLIT_LOG) $(DOCKER_LOG)

SONIC_BUILD_INSTRUCTION := make \

DOCKER_SLAVE_BASE_INSPECT = \
{ \
echo Checking sonic-slave-base image: $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG); \
docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null; \
}

DOCKER_SLAVE_BASE_PULL_REGISTRY = \
[ $(ENABLE_DOCKER_BASE_PULL) == y ] && \
{ \
echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Pulling...; \
$(DOCKER_BASE_PULL); \
} && \
{ \
docker tag $(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) && \
$(COLLECT_DOCKER); \
}\

SONIC_SLAVE_BASE_BUILD = \
{ \
$(DOCKER_SLAVE_BASE_INSPECT); \
} || \
{ \
$(DOCKER_SLAVE_BASE_PULL_REGISTRY); \
} || \
{ \
echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \
$(PREPARE_DOCKER) ; \
$(DOCKER_SLAVE_BASE_BUILD) ; \
$(COLLECT_DOCKER) ; \
}

DOCKER_SLAVE_USER_INSPECT = \
{ \
echo Checking sonic-slave-user image: $(SLAVE_IMAGE):$(SLAVE_TAG); \
docker inspect --type image $(SLAVE_IMAGE):$(SLAVE_TAG) &> /dev/null; \
}

SONIC_SLAVE_USER_BUILD = \
{ $(DOCKER_SLAVE_USER_INSPECT) } || \
{ \
echo Image $(SLAVE_IMAGE):$(SLAVE_TAG) not found. Building... ; \
$(DOCKER_USER_BUILD) ; \
}

SONIC_BUILD_INSTRUCTION := $(MAKE) \
-f slave.mk \
PLATFORM=$(PLATFORM) \
PLATFORM_ARCH=$(PLATFORM_ARCH) \
Expand Down Expand Up @@ -415,87 +498,75 @@ SONIC_BUILD_INSTRUCTION := make \

.PHONY: sonic-slave-build sonic-slave-bash init reset


ifeq ($(filter clean,$(MAKECMDGOALS)),)
COLLECT_BUILD_VERSION = { DBGOPT='$(DBGOPT)' scripts/collect_build_version_files.sh $$?; }
endif

ifdef SOURCE_FOLDER
DOCKER_RUN += -v $(SOURCE_FOLDER):/var/$(USER)/src
endif

ifeq "$(KEEP_SLAVE_ON)" "yes"
SLAVE_SHELL={ /bin/bash; }
endif

.DEFAULT_GOAL := all

%::
%:: | sonic-build-hooks
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is no dependency check for the sonic-build-hooks target, so we are checking this target always.

ifneq ($(filter y, $(MULTIARCH_QEMU_ENVIRON) $(CROSS_BUILD_ENVIRON)),)
@$(DOCKER_MULTIARCH_CHECK)
$(Q)$(DOCKER_MULTIARCH_CHECK)
ifneq ($(BLDENV), )
@$(DOCKER_SERVICE_MULTIARCH_CHECK)
@$(DOCKER_SERVICE_DOCKERFS_CHECK)
endif
endif
@$(OVERLAY_MODULE_CHECK)

@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
@docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \
{ [ $(ENABLE_DOCKER_BASE_PULL) == y ] && { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Pulling...; } && \
$(DOCKER_BASE_PULL) && \
{ docker tag $(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) && \
scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target ; } } || \
{ echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \
$(DOCKER_BASE_BUILD) ; \
scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target ; }
@docker inspect --type image $(SLAVE_IMAGE):$(SLAVE_TAG) &> /dev/null || \
{ echo Image $(SLAVE_IMAGE):$(SLAVE_TAG) not found. Building... ; \
$(DOCKER_BUILD) ; }
ifeq "$(KEEP_SLAVE_ON)" "yes"
ifdef SOURCE_FOLDER
@$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?; /bin/bash"
else
@$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?; /bin/bash"
endif
else
@$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?"
$(Q)$(DOCKER_SERVICE_MULTIARCH_CHECK)
$(Q)$(DOCKER_SERVICE_DOCKERFS_CHECK)
endif
endif
$(Q)$(OVERLAY_MODULE_CHECK)
$(Q)$(SONIC_SLAVE_BASE_BUILD)
$(Q)$(SONIC_SLAVE_USER_BUILD)

$(Q)$(DOCKER_RUN) \
$(SLAVE_IMAGE):$(SLAVE_TAG) \
bash -c "$(SONIC_BUILD_INSTRUCTION) $@;$(COLLECT_BUILD_VERSION); $(SLAVE_SHELL)"
$(Q)$(docker-image-cleanup)

docker-cleanup:
$(docker-image-cleanup)
$(Q)$(docker-image-cleanup)

.PHONY: sonic-build-hooks
sonic-build-hooks:
@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
$(Q)pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) $(MAKE) all; popd
$(Q)mkdir -p $(SLAVE_DIR)/buildinfo
$(Q)cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo

sonic-slave-base-build : sonic-build-hooks
sonic-slave-base-build : | sonic-build-hooks
ifeq ($(MULTIARCH_QEMU_ENVIRON), y)
@$(DOCKER_MULTIARCH_CHECK)
endif
@$(OVERLAY_MODULE_CHECK)
@echo Checking sonic-slave-base image: $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)
@docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \
{ [ $(ENABLE_DOCKER_BASE_PULL) == y ] && { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Pulling...; } && \
$(DOCKER_BASE_PULL) && \
{ docker tag $(REGISTRY_SERVER):$(REGISTRY_PORT)/$(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) && \
scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target ; } } || \
{ echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \
$(DOCKER_BASE_BUILD) ; \
scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target ; }
$(Q)$(DOCKER_MULTIARCH_CHECK)
endif
$(Q)$(OVERLAY_MODULE_CHECK)
$(Q)$(SONIC_SLAVE_BASE_BUILD)

sonic-slave-build : sonic-slave-base-build
@echo Checking sonic-slave image: $(SLAVE_IMAGE):$(SLAVE_TAG)
@docker inspect --type image $(SLAVE_IMAGE):$(SLAVE_TAG) &> /dev/null || \
{ echo Image $(SLAVE_IMAGE):$(SLAVE_TAG) not found. Building... ; \
$(DOCKER_BUILD) ; }
$(Q)$(SONIC_SLAVE_USER_BUILD)

sonic-slave-bash : sonic-slave-build
@$(DOCKER_RUN) -t $(SLAVE_IMAGE):$(SLAVE_TAG) bash
$(Q)$(DOCKER_RUN) -t $(SLAVE_IMAGE):$(SLAVE_TAG) bash

sonic-slave-run : sonic-slave-build
@$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_RUN_CMDS)"
$(Q)$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_RUN_CMDS)"

showtag:
@echo $(SLAVE_IMAGE):$(SLAVE_TAG)
@echo $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)
$(Q)echo $(SLAVE_IMAGE):$(SLAVE_TAG)
$(Q)echo $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)

init :
@git submodule update --init --recursive
@git submodule foreach --recursive '[ -f .git ] && echo "gitdir: $$(realpath --relative-to=. $$(cut -d" " -f2 .git))" > .git'
$(Q)git submodule update --init --recursive
$(Q)git submodule foreach --recursive '[ -f .git ] && echo "gitdir: $$(realpath --relative-to=. $$(cut -d" " -f2 .git))" > .git'

.ONESHELL : reset
reset :
@echo && echo -n "Warning! All local changes will be lost. Proceed? [y/N]: "
@read ans && (
$(Q)echo && echo -n "Warning! All local changes will be lost. Proceed? [y/N]: "
$(Q)read ans && (
if [ $$ans == y ]; then
echo "Resetting local repository. Please wait...";
sudo rm -rf fsroot*;
Expand Down
Loading