Fix per-arch builds & add multi-arch & single arch parallel builds

mk/spksrc.common.mk

@@ -57,19 +57,6 @@ SUPPORTED_ARCHS = $(sort $(filter-out $(ARCHS_DUPES), $(AVAILABLE_TOOLCHAINS)))
 #         all archs except generic archs
 LEGACY_ARCHS = $(sort $(filter-out $(addsuffix %,$(GENERIC_ARCHS)), $(AVAILABLE_TOOLCHAINS)))
 
-
-# Relocate to set conditionally according to existing parallel options in caller
-ifneq ($(PARALLEL_MAKE),)
-ifeq ($(PARALLEL_MAKE),max)
-NCPUS = $(shell grep -c ^processor /proc/cpuinfo)
-else
-NCPUS = $(PARALLEL_MAKE)
-endif
-ifeq ($(filter $(NCPUS),0 1),)
-COMPILE_MAKE_OPTIONS += -j$(NCPUS)
-endif
-endif
-
 # Terminal colors
 RED=`tput setaf 1`
 GREEN=`tput setaf 2`
@@ -80,3 +67,26 @@ version_le = $(shell if printf '%s\n' "$(1)" "$(2)" | sort -VC ; then echo 1; fi
 version_ge = $(shell if printf '%s\n' "$(1)" "$(2)" | sort -VCr ; then echo 1; fi)
 version_lt = $(shell if [ "$(1)" != "$(2)" ] && printf "%s\n" "$(1)" "$(2)" | sort -VC ; then echo 1; fi)
 version_gt = $(shell if [ "$(1)" != "$(2)" ] && printf "%s\n" "$(1)" "$(2)" | sort -VCr ; then echo 1; fi)
+
+# Set parallel build mode
+ifeq ($(PMAKE),)
+# If -j or -l argument passed user must
+# manually specify the value of PMAKE
+# as otherwise this will create too high load
+ifneq ($(strip $(filter -j% -l%, $(shell ps T $$PPID))),)
+PMAKE = nop
+ENV += PMAKE=nop
+else
+# If not set, force max parallel build mode
+PMAKE = max
+ENV += PMAKE=max
+endif
+endif
+
+# Enable stats over parallel build mode
+ifneq ($(filter 1 on ON,$(PSTAT)),)
+PSTAT_TIME = time -o $(PSTAT_LOG) --append
+else
+PSTAT_TIME =
+endif
+PSTAT_LOG = build.stats

mk/spksrc.compile.mk

@@ -6,6 +6,8 @@
 #  compile_target       (override with COMPILE_TARGET)
 #  post_compile_target  (override with POST_COMPILE_TARGET)
 
+.PARALLEL:
+
 COMPILE_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)compile_done
 
 ifeq ($(strip $(PRE_COMPILE_TARGET)),)
@@ -29,6 +31,9 @@ endif
 
 compile_msg:
 	@$(MSG) "Compiling for $(NAME)"
+ifneq ($(filter 1 on ON,$(PSTAT)),)
+	@$(MSG) MAKELEVEL: $(MAKELEVEL), PMAKE: $(PMAKE), ARCH: $(ARCH)-$(TCVERSION) >> $(PSTAT_LOG)
+endif
 
 pre_compile_target: compile_msg
 
@@ -46,4 +51,3 @@ $(COMPILE_COOKIE): $(POST_COMPILE_TARGET)
 else
 compile: ;
 endif
-

mk/spksrc.cross-cc.mk

@@ -1,3 +1,8 @@
+# Default make programs
+#
+
+.NOTPARALLEL:
+
 # Common makefiles
 include ../../mk/spksrc.common.mk
 include ../../mk/spksrc.directories.mk
@@ -19,6 +24,14 @@ ARCH_SUFFIX = -$(ARCH)-$(TCVERSION)
 TC = syno$(ARCH_SUFFIX)
 endif
 
+# Set parallel options in caller
+ifeq ($(PMAKE),max)
+MAKEFLAGS += -j$(shell nproc)
+else ifneq ($(PMAKE),)
+ifneq ($(PMAKE),nop)
+MAKEFLAGS += -j$(PMAKE)
+endif
+endif
 
 #####
 
@@ -62,6 +75,9 @@ clean:
 
 
 all: install plist
+ifneq ($(filter 1 on ON,$(PSTAT)),)
+	@$(MSG) MAKELEVEL: $(MAKELEVEL), PMAKE: $(PMAKE), ARCH: $(ARCH)-$(TCVERSION) >> $(PSTAT_LOG)
+endif
 
 ### For make kernel-required (used by spksrc.spk.mk)
 include ../../mk/spksrc.kernel-required.mk
@@ -77,8 +93,17 @@ all-archs: $(addprefix arch-,$(AVAILABLE_TOOLCHAINS))
 
 ####
 
+cross-cc_msg:
+ifneq ($(filter 1 on ON,$(PSTAT)),)
+	@$(MSG) MAKELEVEL: $(MAKELEVEL), PMAKE: $(PMAKE), ARCH: $(subst build-arch-,,$(MAKECMDGOALS)) >> $(PSTAT_LOG)
+endif
+
 arch-%:
+	@$(MSG) Building package for arch $(or $(filter $(addprefix %, $(DEFAULT_TC)), $(filter %$(word 2,$(subst -, ,$*)), $(filter $(firstword $(subst -, ,$*))%, $(AVAILABLE_TOOLCHAINS)))), $*)
+	$(MAKE) $(addprefix build-arch-, $(or $(filter $(addprefix %, $(DEFAULT_TC)), $(filter %$(word 2,$(subst -, ,$*)), $(filter $(firstword $(subst -, ,$*))%, $(AVAILABLE_TOOLCHAINS)))),$*))
+
+build-arch-%: cross-cc_msg
 	@$(MSG) Building package for arch $*
-	-@MAKEFLAGS= $(MAKE) ARCH=$(basename $(subst -,.,$(basename $(subst .,,$*)))) TCVERSION=$(if $(findstring $*,$(basename $(subst -,.,$(basename $(subst .,,$*))))),$(DEFAULT_TC),$(notdir $(subst -,/,$*)))
+	-@MAKEFLAGS= $(PSTAT_TIME) $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) 2>&1 | tee build-$*.log
 
 ####

mk/spksrc.cross-cmake-env.mk

@@ -13,6 +13,15 @@ CMAKE_ARGS += -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE
 CMAKE_ARGS += -DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE
 CMAKE_ARGS += -DBUILD_SHARED_LIBS=ON
 
+# Set parallel options in caller
+ifeq ($(PMAKE),max)
+MAKEFLAGS += -j$(shell nproc)
+else ifneq ($(PMAKE),)
+ifneq ($(PMAKE),nop)
+MAKEFLAGS += -j$(PMAKE)
+endif
+endif
+
 # Use native cmake
 ifeq ($(strip $(USE_NATIVE_CMAKE)),1)
   BUILD_DEPENDS += native/cmake

mk/spksrc.cross-cmake.mk

@@ -4,6 +4,8 @@
 # - cross/module depends on cmake
 #
 
+.NOTPARALLEL:
+
 # Common makefiles
 include ../../mk/spksrc.common.mk
 include ../../mk/spksrc.directories.mk
@@ -38,7 +40,11 @@ cmake_configure_target:
 	@$(MSG)    - Path BUILD_DIR = $(CMAKE_BUILD_DIR)
 	$(RUN) rm -rf CMakeCache.txt CMakeFiles
 	$(RUN) mkdir --parents $(CMAKE_BUILD_DIR)
+ifeq ($(PMAKE),max)
+	cd $(CMAKE_BUILD_DIR) && env $(ENV) cmake $(CMAKE_ARGS) --parallel $(shell nproc) $(WORK_DIR)/$(PKG_DIR)
+else
 	cd $(CMAKE_BUILD_DIR) && env $(ENV) cmake $(CMAKE_ARGS) $(WORK_DIR)/$(PKG_DIR)
+endif
 
 .PHONY: cmake_compile_target
 

mk/spksrc.cross-env.mk

@@ -36,7 +36,3 @@ $(TC_VARS_MK):
 ENV += TC=$(TC)
 ENV += $(TC_ENV)
 endif
-
-#ifneq ($(COMPILE_MAKE_OPTIONS),)
-#ENV += MAKEFLAGS="$(COMPILE_MAKE_OPTIONS)"
-#endif

mk/spksrc.spk.mk

@@ -398,6 +398,9 @@ spkclean:
 	       work-*/WIZARD_UIFILES
 
 all: package
+ifneq ($(filter 1 on ON,$(PSTAT)),)
+	@$(MSG) MAKELEVEL: $(MAKELEVEL), PMAKE: $(PMAKE), ARCH: $(ARCH)-$(TCVERSION) >> $(PSTAT_LOG)
+endif
 
 ### For make dependency-tree
 include ../../mk/spksrc.dependency-tree.mk
@@ -454,48 +457,53 @@ pre-build-native:
 
 $(PUBLISH)all-$(ACTION): | pre-build-native
 
+spk_msg:
+ifneq ($(filter 1 on ON,$(PSTAT)),)
+	@$(MSG) MAKELEVEL: $(MAKELEVEL), PMAKE: $(PMAKE), ARCH: $(MAKECMDGOALS) >> $(PSTAT_LOG)
+endif
+
 supported-arch-error:
 	@$(MSG) ########################################################
 	@$(MSG) ERROR - Please run make setup from spksrc root directory
 	@$(MSG) ########################################################
 
-supported-arch-%:
+supported-arch-%: spk_msg
 	@$(MSG) BUILDING package for arch $* with SynoCommunity toolchain
-	-@MAKEFLAGS= $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) 2>&1 | tee build-$*.log
+	-@MAKEFLAGS= $(PSTAT_TIME) $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) 2>&1 | tee build-$*.log
 
-publish-supported-arch-%:
+publish-supported-arch-%: spk_msg
 	@$(MSG) BUILDING and PUBLISHING package for arch $* with SynoCommunity toolchain
-	-@MAKEFLAGS= $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) publish 2>&1 | tee build-$*.log
+	-@MAKEFLAGS= $(PSTAT_TIME) $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) publish 2>&1 | tee build-$*.log
 
-latest-arch-%:
+latest-arch-%: spk_msg
 	@$(MSG) BUILDING package for arch $* with SynoCommunity toolchain
-	-@MAKEFLAGS= $(MAKE) ARCH=$(basename $(subst -,.,$*)) TCVERSION=$(notdir $(subst -,/,$(sort $(filter %$(lastword $(notdir $(subst -,/,$(sort $(filter $*%, $(AVAILABLE_TOOLCHAINS)))))),$(sort $(filter $*%, $(AVAILABLE_TOOLCHAINS))))))) 2>&1 | tee build-$*.log
+	-@MAKEFLAGS= $(PSTAT_TIME) $(MAKE) ARCH=$(basename $(subst -,.,$*)) TCVERSION=$(notdir $(subst -,/,$(sort $(filter %$(lastword $(notdir $(subst -,/,$(sort $(filter $*%, $(AVAILABLE_TOOLCHAINS)))))),$(sort $(filter $*%, $(AVAILABLE_TOOLCHAINS))))))) 2>&1 | tee build-$*.log
 
-publish-latest-arch-%:
+publish-latest-arch-%: spk_msg
 	@$(MSG) BUILDING and PUBLISHING package for arch $* with SynoCommunity toolchain
-	-@MAKEFLAGS= $(MAKE) ARCH=$(basename $(subst -,.,$*)) TCVERSION=$(notdir $(subst -,/,$(sort $(filter %$(lastword $(notdir $(subst -,/,$(sort $(filter $*%, $(AVAILABLE_TOOLCHAINS)))))),$(sort $(filter $*%, $(AVAILABLE_TOOLCHAINS))))))) publish 2>&1 | tee build-$*.log
+	-@MAKEFLAGS= $(PSTAT_TIME) $(MAKE) ARCH=$(basename $(subst -,.,$*)) TCVERSION=$(notdir $(subst -,/,$(sort $(filter %$(lastword $(notdir $(subst -,/,$(sort $(filter $*%, $(AVAILABLE_TOOLCHAINS)))))),$(sort $(filter $*%, $(AVAILABLE_TOOLCHAINS))))))) publish 2>&1 | tee build-$*.log
 
 ####
 
-all-legacy:
+all-legacy: spk_msg
+	@$(MSG) BUILDING package for legacy DSM and SRM archs
 	$(MAKE) legacy-toolchain-5.2 legacy-toolchain-1.2
-	@$(MSG) Built legacy DSM and SRM archs
 
-publish-all-legacy:
+publish-all-legacy: spk_msg
+	@$(MSG) BUILDING and PUBLISHING package for legacy DSM archs
 	$(MAKE) publish-legacy-toolchain-5.2
-	@$(MSG) Published legacy DSM archs
 
 ####
 
-legacy-toolchain-%:
-	@$(MSG) Built packages for toolchain $*
+legacy-toolchain-%: spk_msg
+	@$(MSG) BUILDING packages for toolchain $*
 	@for arch in $(sort $(basename $(subst -,.,$(basename $(subst .,,$(filter %$*, $(LEGACY_ARCHS))))))) ; \
 	do \
 	  $(MAKE) arch-$$arch-$* ; \
 	done \
 
-publish-legacy-toolchain-%:
-	@$(MSG) Built packages for toolchain $*
+publish-legacy-toolchain-%: spk_msg
+	@$(MSG) BUILDING and PUBLISHING packages for toolchain $*
 	@for arch in $(sort $(basename $(subst -,.,$(basename $(subst .,,$(filter %$*, $(LEGACY_ARCHS))))))) ; \
 	do \
 	  $(MAKE) publish-arch-$$arch-$* ; \
@@ -504,12 +512,17 @@ publish-legacy-toolchain-%:
 ####
 
 arch-%:
-	@$(MSG) Building package for arch $*
-	-@MAKEFLAGS= $(MAKE) ARCH=$(basename $(subst -,.,$(basename $(subst .,,$*)))) TCVERSION=$(if $(findstring $*,$(basename $(subst -,.,$(basename $(subst .,,$*))))),$(DEFAULT_TC),$(notdir $(subst -,/,$*)))
+	# handle and allow parallel build for:  arch-<arch> | make arch-<arch>-X.Y
+	@$(MSG) BUILDING package for arch $(or $(filter $(addprefix %, $(DEFAULT_TC)), $(filter %$(word 2,$(subst -, ,$*)), $(filter $(firstword $(subst -, ,$*))%, $(AVAILABLE_TOOLCHAINS)))), $*)
+	$(MAKE) $(addprefix build-arch-, $(or $(filter $(addprefix %, $(DEFAULT_TC)), $(filter %$(word 2,$(subst -, ,$*)), $(filter $(firstword $(subst -, ,$*))%, $(AVAILABLE_TOOLCHAINS)))),$*))
+
+build-arch-%: spk_msg
+	@$(MSG) BUILDING package for arch $*
+	-@MAKEFLAGS= $(PSTAT_TIME) $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) 2>&1 | tee build-$*.log
 
-publish-arch-%:
-	@$(MSG) Building and publishing package for arch $*
-	-@MAKEFLAGS= $(MAKE) ARCH=$(basename $(subst -,.,$(basename $(subst .,,$*)))) TCVERSION=$(if $(findstring $*,$(basename $(subst -,.,$(basename $(subst .,,$*))))),$(DEFAULT_TC),$(notdir $(subst -,/,$*))) publish
+publish-arch-%: spk_msg
+	@$(MSG) BUILDING and PUBLISHING package for arch $*
+	-@MAKEFLAGS= $(PSTAT_TIME) $(MAKE) ARCH=$(basename $(subst -,.,$(basename $(subst .,,$*)))) TCVERSION=$(if $(findstring $*,$(basename $(subst -,.,$(basename $(subst .,,$*))))),$(DEFAULT_TC),$(notdir $(subst -,/,$*))) publish 2>&1 | tee build-$*.log
 
 ####