Build improvements

.gitmodules

@@ -14,16 +14,16 @@
 	path = src/sonic-swss
 	url = https://github.com/Azure/sonic-swss
 [submodule "src/p4-switch/switch"]
-	path = src/p4-switch/switch
+	path = platform/p4/p4-switch/switch
 	url = https://github.com/lguohan/switch
 [submodule "src/p4-bmv2/behavioral-model"]
-	path = src/p4-bmv2/behavioral-model
+	path = platform/p4/p4-bmv/behavioral-model
 	url = https://github.com/lguohan/behavioral-model
 [submodule "src/p4c-bm/p4c-bm"]
-	path = src/p4c-bm/p4c-bm
+	path = platform/p4/p4c-bm/p4c-bm
 	url = https://github.com/krambn/p4c-bm
 [submodule "src/p4-hlir/p4-hlir"]
-	path = src/p4-hlir/p4-hlir
+	path = platform/p4/p4-hlir/p4-hlir
 	url = https://github.com/p4lang/p4-hlir
 [submodule "quagga"]
 	path = src/sonic-quagga

Makefile

@@ -1,197 +1,39 @@
-## TODO: if install dev package really happens, rebuild the depending project
+###############################################################################
+## Wrapper for starting make inside sonic-slave container
+###############################################################################
 
-## Arguments from make command line
-USERNAME=
-PASSWORD_ENCRYPTED=
+SHELL = /bin/bash
 
-## Redis server/tools version
-REDIS_VERSION=3.2.4-1~bpo8+1_amd64
+USER := $(shell id -un)
+PWD := $(shell pwd)
 
-## Select bash for commands
-SHELL := /bin/bash
+MAKEFLAGS += -B
 
-## Capture all the files in SDK directories
-MLNX-SDK-DEBS=$(notdir $(wildcard src/mlnx-sdk/*.deb))
-BRCM-SDK-DEBS=$(notdir $(wildcard src/brcm-sdk/*.deb))
-CAVM-SDK-DEBS=$(notdir $(wildcard src/cavm-sdk/*.deb))
+DOCKER_RUN := docker run --rm=true --privileged \
+    -v $(PWD):/sonic \
+    -it sonic-slave-$(USER)
 
-LIBNL-DEBS=libnl-3-200_3.2.27-1_amd64.deb libnl-3-dev_3.2.27-1_amd64.deb libnl-genl-3-200_3.2.27-1_amd64.deb libnl-genl-3-dev_3.2.27-1_amd64.deb libnl-route-3-200_3.2.27-1_amd64.deb libnl-route-3-dev_3.2.27-1_amd64.deb  libnl-nf-3-200_3.2.27-1_amd64.deb libnl-nf-3-dev_3.2.27-1_amd64.deb libnl-cli-3-200_3.2.27-1_amd64.deb libnl-cli-3-dev_3.2.27-1_amd64.deb
+DOCKER_BUILD = docker build --no-cache \
+	       --build-arg user=$(USER) \
+	       --build-arg uid=$(shell id -u) \
+	       --build-arg guid=$(shell id -g) \
+	       -t sonic-slave-$(USER) \
+	       sonic-slave
 
-LIBTEAM-DEBS=libteam5_1.26-1_amd64.deb libteamdctl0_1.26-1_amd64.deb libteam-dev_1.26-1_amd64.deb libteam-utils_1.26-1_amd64.deb
+.PHONY: sonic-slave-build sonic-slave-bash
 
-## Function: build_docker, image_name save_file
-## build a docker image and save to a file
-define build_docker
-	docker build --no-cache -t $(1) dockers/$(1)
-	mkdir -p `dirname $(2)`
-	docker save $(1) | gzip -c > $(2)
-endef
+.DEFAULT_GOAL :=  all
 
-## Rules: phony targets
-.phony : brcm-all mlnx-all cavm-all p4-all
+%::
+	@docker inspect --type image sonic-slave-$(USER) &> /dev/null || $(DOCKER_BUILD)
+	@$(DOCKER_RUN) make \
+	    -C sonic \
+	    -f slave.mk \
+	    PLATFORM=$(PLATFORM) \
+	    $@
 
-## Rules: redirect to sub directory
-src/%:
-	$(MAKE) 											\
-	REDIS_VERSION=$(REDIS_VERSION)						\
-	LIBNL-DEBS="$(LIBNL-DEBS)"							\
-	LIBTEAM-DEBS="$(LIBTEAM-DEBS)"						\
-	-C src $(subst src/,,$@)
+sonic-slave-build :
+	@$(DOCKER_BUILD)
 
-## Rules: docker-snmp-sv2
-dockers/docker-snmp-sv2/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-snmp-sv2/deps/python3/%.whl: src/%.whl
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-lldp-sv2
-dockers/docker-lldp-sv2/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-lldp-sv2/deps/%.whl: src/%.whl
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-fpm
-dockers/docker-fpm/deps/fpmsyncd: src/fpmsyncd
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-fpm/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-team
-dockers/docker-team/deps/teamsyncd: src/teamsyncd
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-team/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-orchagent-mlnx
-$(addprefix dockers/docker-orchagent-mlnx/deps/,libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb swss_1.0.0_amd64.deb) : dockers/docker-orchagent-mlnx/deps/%.deb : src/mlnx/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-orchagent-mlnx/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-orchagent-cavm
-$(addprefix dockers/docker-orchagent-cavm/deps/,libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb swss_1.0.0_amd64.deb) : dockers/docker-orchagent-cavm/deps/%.deb : src/cavm/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-orchagent-cavm/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-orchagent (brcm)
-$(addprefix dockers/docker-orchagent/deps/,libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb swss_1.0.0_amd64.deb) : dockers/docker-orchagent/deps/%.deb : src/brcm/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-orchagent/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-syncd-mlnx
-$(addprefix dockers/docker-syncd-mlnx/deps/,$(MLNX-SDK-DEBS)) : dockers/docker-syncd-mlnx/deps/%.deb : src/mlnx-sdk/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-$(addprefix dockers/docker-syncd-mlnx/deps/,syncd_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb) : dockers/docker-syncd-mlnx/deps/%.deb : src/mlnx/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-syncd-mlnx/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-syncd-mlnx/deps/fw-SPC.mfa: src/mlnx-sdk/fw-SPC.mfa
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-syncd-cavm
-$(addprefix dockers/docker-syncd-cavm/deps/,$(CAVM-SDK-DEBS)) : dockers/docker-syncd-cavm/deps/%.deb : src/cavm-sdk/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-$(addprefix dockers/docker-syncd-cavm/deps/,syncd_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb) : dockers/docker-syncd-cavm/deps/%.deb : src/cavm/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-syncd-cavm/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-syncd (brcm)
-$(addprefix dockers/docker-syncd/deps/,$(BRCM-SDK-DEBS)) : dockers/docker-syncd/deps/%.deb : src/brcm-sdk/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-$(addprefix dockers/docker-syncd/deps/,syncd_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb): dockers/docker-syncd/deps/%.deb : src/brcm/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-syncd/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-database
-dockers/docker-database/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker-sonic (p4)
-$(addprefix dockers/docker-sonic-p4/deps/,swss_1.0.0_amd64.deb syncd_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb) : dockers/docker-sonic-p4/deps/%.deb : src/p4/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-dockers/docker-sonic-p4/deps/%.deb: src/%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-## Rules: docker images
-target/docker-base.gz:
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-snmp-sv2.gz: target/docker-base.gz $(addprefix dockers/docker-snmp-sv2/deps/,python3/sswsdk-2.0.1-py3-none-any.whl python3/asyncsnmp-2.1.0-py3-none-any.whl libsnmp-base_5.7.3+dfsg-1.5_all.deb libsnmp30_5.7.3+dfsg-1.5_amd64.deb snmp_5.7.3+dfsg-1.5_amd64.deb snmpd_5.7.3+dfsg-1.5_amd64.deb)
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-lldp-sv2.gz: target/docker-base.gz $(addprefix dockers/docker-lldp-sv2/deps/,sswsdk-2.0.1-py2-none-any.whl sonic_d-2.0.0-py2-none-any.whl lldpd_0.9.5-0_amd64.deb)
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-syncd.gz: target/docker-base.gz $(addprefix dockers/docker-syncd/deps/,$(BRCM-SDK-DEBS) libhiredis0.13_0.13.3-2_amd64.deb libswsscommon_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb syncd_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb $(LIBNL-DEBS))
-	## TODO: remove placeholders for the dependencies
-	touch dockers/docker-syncd/deps/{dsserve,bcmcmd}
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-syncd-mlnx.gz: target/docker-base.gz $(addprefix dockers/docker-syncd-mlnx/deps/,$(MLNX-SDK-DEBS) libhiredis0.13_0.13.3-2_amd64.deb libswsscommon_1.0.0_amd64.deb syncd_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb $(LIBNL-DEBS)) dockers/docker-syncd-mlnx/deps/fw-SPC.mfa
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-syncd-cavm.gz: target/docker-base.gz $(addprefix dockers/docker-syncd-cavm/deps/,$(CAVM-SDK-DEBS) libhiredis0.13_0.13.3-2_amd64.deb libswsscommon_1.0.0_amd64.deb syncd_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb $(LIBNL-DEBS))
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-orchagent.gz: target/docker-base.gz $(addprefix dockers/docker-orchagent/deps/,libhiredis0.13_0.13.3-2_amd64.deb libswsscommon_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb swss_1.0.0_amd64.deb $(LIBNL-DEBS) $(LIBTEAM-DEBS))
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-orchagent-mlnx.gz: target/docker-base.gz $(addprefix dockers/docker-orchagent-mlnx/deps/,libhiredis0.13_0.13.3-2_amd64.deb libswsscommon_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb swss_1.0.0_amd64.deb $(LIBNL-DEBS) $(LIBTEAM-DEBS))
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-orchagent-cavm.gz: target/docker-base.gz $(addprefix dockers/docker-orchagent-cavm/deps/,libhiredis0.13_0.13.3-2_amd64.deb libswsscommon_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb swss_1.0.0_amd64.deb $(LIBNL-DEBS) $(LIBTEAM-DEBS))
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-fpm.gz: target/docker-base.gz $(addprefix dockers/docker-fpm/deps/,libswsscommon_1.0.0_amd64.deb libhiredis0.13_0.13.3-2_amd64.deb quagga_0.99.24.1-2.1_amd64.deb fpmsyncd $(LIBNL-DEBS))
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-team.gz: target/docker-base.gz $(addprefix dockers/docker-team/deps/,libswsscommon_1.0.0_amd64.deb libhiredis0.13_0.13.3-2_amd64.deb $(LIBNL-DEBS) $(LIBTEAM-DEBS) teamsyncd)
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-database.gz: target/docker-base.gz $(addprefix dockers/docker-database/deps/,redis-server_$(REDIS_VERSION).deb redis-tools_$(REDIS_VERSION).deb)
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-sonic-p4.gz: target/docker-base.gz $(addprefix dockers/docker-sonic-p4/deps/,libswsscommon_1.0.0_amd64.deb libhiredis0.13_0.13.3-2_amd64.deb quagga_0.99.24.1-2.1_amd64.deb syncd_1.0.0_amd64.deb swss_1.0.0_amd64.deb libsairedis_1.0.0_amd64.deb libsaimetadata_1.0.0_amd64.deb libthrift-0.9.3_0.9.3-2_amd64.deb redis-server_$(REDIS_VERSION).deb redis-tools_$(REDIS_VERSION).deb p4-bmv2_1.0.0_amd64.deb p4-switch_1.0.0_amd64.deb)
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-target/docker-platform-monitor.gz: target/docker-base.gz
-	docker load < $<
-	$(call build_docker,$(patsubst target/%.gz,%,$@),$@)
-
-## Rules: linux image content
-deps/linux-image-3.16.0-4-amd64_%.deb: src/sonic-linux-kernel/linux-image-3.16.0-4-amd64_%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-deps/initramfs-tools_%.deb: src/initramfs-tools/initramfs-tools_%.deb
-	mkdir -p `dirname $@` && cp $< $(dir $@)
-
-target/sonic-generic.bin: deps/linux-image-3.16.0-4-amd64_3.16.7-ckt11-2+acs8u2_amd64.deb deps/initramfs-tools_0.120_all.deb
-	./build_debian.sh "$(USERNAME)" "$(PASSWORD_ENCRYPTED)" && TARGET_MACHINE=generic ./build_image.sh
-target/sonic-aboot.bin: deps/linux-image-3.16.0-4-amd64_3.16.7-ckt11-2+acs8u2_amd64.deb deps/initramfs-tools_0.120_all.deb
-	./build_debian.sh "$(USERNAME)" "$(PASSWORD_ENCRYPTED)" && TARGET_MACHINE=aboot ./build_image.sh
-
-## Note: docker-fpm.gz must be the last to build the implicit dependency fpmsyncd
-brcm-all: target/sonic-generic.bin $(addprefix target/,docker-syncd.gz docker-orchagent.gz docker-fpm.gz docker-team.gz docker-database.gz docker-snmp-sv2.gz docker-lldp-sv2.gz docker-platform-monitor.gz)
-
-## Note: docker-fpm.gz must be the last to build the implicit dependency fpmsyncd
-mlnx-all: target/sonic-generic.bin $(addprefix target/,docker-syncd-mlnx.gz docker-orchagent-mlnx.gz docker-fpm.gz docker-team.gz docker-database.gz docker-snmp-sv2.gz docker-lldp-sv2.gz docker-platform-monitor.gz)
-
-## Note: docker-fpm.gz must be the last to build the implicit dependency fpmsyncd
-cavm-all: target/sonic-generic.bin $(addprefix target/,docker-syncd-cavm.gz docker-orchagent-cavm.gz docker-fpm.gz docker-team.gz docker-database.gz docker-snmp-sv2.gz docker-lldp-sv2.gz docker-platform-monitor.gz)
-
-p4-all: $(addprefix target/,docker-sonic-p4.gz)
+sonic-slave-bash :
+	@$(DOCKER_RUN) bash

build_debian.sh

@@ -112,9 +112,9 @@ sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c 'cd /dev && MAKEDEV generic'
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install busybox
 echo '[INFO] Install SONiC linux kernel image'
 ## Note: duplicate apt-get command to ensure every line return zero
-sudo dpkg --root=$FILESYSTEM_ROOT -i deps/initramfs-tools_*.deb || \
+sudo dpkg --root=$FILESYSTEM_ROOT -i target/debs/initramfs-tools_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
-sudo dpkg --root=$FILESYSTEM_ROOT -i deps/linux-image-3.16.0-4-amd64_*.deb || \
+sudo dpkg --root=$FILESYSTEM_ROOT -i target/debs/linux-image-3.16.0-4-amd64_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 
 ## Update initramfs for booting with squashfs+aufs

build_image.sh

@@ -13,7 +13,7 @@
 }
 
 ## Retrieval short version of Git revision hash for partition metadata
-[ -z "$(git status --untracked-files=no -s)" ] || {
+[ -z "$(git status --untracked-files=no -s --ignore-submodules)" ] || {
     echo "Error: There is local changes not committed to git repo. Cannot get a revision hash for partition metadata."
     exit 1
 }

dockers/docker-database/Dockerfile.template

@@ -0,0 +1,19 @@
+FROM docker-base
+
+RUN apt-get update
+
+COPY deps/ deps
+
+SED_DPKG
+
+## Clean up
+RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
+RUN rm -rf /deps
+
+RUN sed -ri 's/^daemonize yes$/daemonize no/;                                           \
+             s/^logfile .*$/logfile ""/;                                                \
+             s/^# syslog-enabled no$/syslog-enabled no/;                                \
+             s/^# unixsocket/unixsocket/                                                \
+            ' /etc/redis/redis.conf
+
+ENTRYPOINT service redis-server start

dockers/docker-fpm/Dockerfile.template

@@ -0,0 +1,16 @@
+FROM docker-base
+
+RUN apt-get update
+
+RUN apt-get install -y libdbus-1-3 libdaemon0 libjansson4
+
+COPY deps/ deps
+
+SED_DPKG
+
+## Clean up
+RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
+RUN rm -rf /deps
+
+ENTRYPOINT /usr/bin/start.sh    \
+        && /bin/bash