Merge pull request kata-containers#170 from marcov/parallel-build

tests: parallel images build support

.travis.yml

@@ -13,10 +13,6 @@ os:
 
 language: bash
 
-env:
-  - AGENT_INIT=no
-  - AGENT_INIT=yes
-
 services:
   - docker
 

Makefile

@@ -3,59 +3,87 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-
-DISTRO ?= centos
-DISTRO_ROOTFS := $(PWD)/$(DISTRO)_rootfs
-DISTRO_ROOTFS_MARKER := .$(shell basename $(DISTRO_ROOTFS)).done
-IMAGE := kata-containers.img
-INITRD_IMAGE := kata-containers-initrd.img
-IMG_SIZE=500
-AGENT_INIT ?= no
-
-VERSION_FILE := ./VERSION
-VERSION := $(shell grep -v ^\# $(VERSION_FILE))
-COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true)
-COMMIT := $(if $(shell git status --porcelain --untracked-files=no),${COMMIT_NO}-dirty,${COMMIT_NO})
+MK_DIR         := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+TEST_RUNNER    := $(MK_DIR)/tests/test_images.sh
+ROOTFS_BUILDER := $(MK_DIR)/rootfs-builder/rootfs.sh
+INITRD_BUILDER := $(MK_DIR)/initrd-builder/initrd_builder.sh
+IMAGE_BUILDER  := $(MK_DIR)/image-builder/image_builder.sh
+
+IMG_SIZE               = 500
+AGENT_INIT            ?= no
+DISTRO                ?= centos
+ROOTFS_BUILD_DEST     := $(PWD)
+IMAGES_BUILD_DEST     := $(PWD)
+DISTRO_ROOTFS         := $(ROOTFS_BUILD_DEST)/$(DISTRO)_rootfs
+DISTRO_ROOTFS_MARKER  := $(ROOTFS_BUILD_DEST)/.$(DISTRO)_rootfs.done
+DISTRO_IMAGE          := $(IMAGES_BUILD_DEST)/kata-containers.img
+DISTRO_INITRD         := $(IMAGES_BUILD_DEST)/kata-containers-initrd.img
+
+VERSION_FILE   := ./VERSION
+VERSION        := $(shell grep -v ^\# $(VERSION_FILE))
+COMMIT_NO      := $(shell git rev-parse HEAD 2> /dev/null || true)
+COMMIT         := $(if $(shell git status --porcelain --untracked-files=no),${COMMIT_NO}-dirty,${COMMIT_NO})
 VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
 
+################################################################################
+
+rootfs-%: $(ROOTFS_BUILD_DEST)/.%_rootfs.done
+	@ # DONT remove. This is not cancellation rule.
+
+.PRECIOUS: $(ROOTFS_BUILD_DEST)/.%_rootfs.done
+$(ROOTFS_BUILD_DEST)/.%_rootfs.done:: rootfs-builder/%
+	@echo Creating rootfs for "$*"
+	$(ROOTFS_BUILDER) -o $(VERSION_COMMIT) -r $(ROOTFS_BUILD_DEST)/$*_rootfs $*
+	touch $@
+
+image-%: $(IMAGES_BUILD_DEST)/kata-containers-image-%.img
+	@ # DONT remove. This is not cancellation rule.
+
+.PRECIOUS: $(IMAGES_BUILD_DEST)/kata-containers-image-%.img
+$(IMAGES_BUILD_DEST)/kata-containers-image-%.img: rootfs-%
+	@echo Creating image based on $^
+	$(IMAGE_BUILDER) -s $(IMG_SIZE) -o $@ $(ROOTFS_BUILD_DEST)/$*_rootfs
+
+initrd-%: $(IMAGES_BUILD_DEST)/kata-containers-initrd-%.img
+	@ # DONT remove. This is not cancellation rule.
+
+.PRECIOUS: $(IMAGES_BUILD_DEST)/kata-containers-initrd-%.img
+$(IMAGES_BUILD_DEST)/kata-containers-initrd-%.img: rootfs-%
+	@echo Creating initrd image for $*
+	$(INITRD_BUILDER) -o $@ $(ROOTFS_BUILD_DEST)/$*_rootfs
+
 .PHONY: all
 all: image initrd
 
 .PHONY: rootfs
 rootfs: $(DISTRO_ROOTFS_MARKER)
 
-$(DISTRO_ROOTFS_MARKER):
-	@echo Creating rootfs based on "$(DISTRO)"
-	"$(MK_DIR)/rootfs-builder/rootfs.sh" -o $(VERSION_COMMIT) -r $(DISTRO_ROOTFS) $(DISTRO)
-	touch $@
-
 .PHONY: image
-image: $(IMAGE)
+image: $(DISTRO_IMAGE)
 
-$(IMAGE): rootfs
+$(DISTRO_IMAGE): $(DISTRO_ROOTFS_MARKER)
 	@echo Creating image based on "$(DISTRO_ROOTFS)"
-	"$(MK_DIR)/image-builder/image_builder.sh" -s "$(IMG_SIZE)" "$(DISTRO_ROOTFS)"
+	$(IMAGE_BUILDER) -s "$(IMG_SIZE)" "$(DISTRO_ROOTFS)"
 
 .PHONY: initrd
-initrd: $(INITRD_IMAGE)
+initrd: $(DISTRO_INITRD)
 
-$(INITRD_IMAGE): rootfs
+$(DISTRO_INITRD): $(DISTRO_ROOTFS_MARKER)
 	@echo Creating initrd image based on "$(DISTRO_ROOTFS)"
-	"$(MK_DIR)/initrd-builder/initrd_builder.sh" "$(DISTRO_ROOTFS)"
+	$(INITRD_BUILDER) "$(DISTRO_ROOTFS)"
 
 .PHONY: test
 test:
-	"$(MK_DIR)/tests/test_images.sh" "$(DISTRO)"
+	$(TEST_RUNNER) "$(DISTRO)"
 
 .PHONY: test-image-only
 test-image-only:
-	"$(MK_DIR)/tests/test_images.sh" --test-images-only "$(DISTRO)"
+	$(TEST_RUNNER) --test-images-only "$(DISTRO)"
 
 .PHONY: test-initrd-only
 test-initrd-only:
-	"$(MK_DIR)/tests/test_images.sh" --test-initrds-only "$(DISTRO)"
+	$(TEST_RUNNER) --test-initrds-only "$(DISTRO)"
 
 .PHONY: clean
 clean:
-	rm -rf $(DISTRO_ROOTFS_MARKER) $(DISTRO_ROOTFS) $(IMAGE) $(INITRD_IMAGE)
+	rm -rf $(DISTRO_ROOTFS_MARKER) $(DISTRO_ROOTFS) $(DISTRO_IMAGE) $(DISTRO_INITRD)

rootfs-builder/rootfs.sh

@@ -52,8 +52,8 @@ $(get_distros)
 Refer the Platform-OS Compatibility Matrix: https://github.com/kata-containers/osbuilder#platform-distro-compatibility-matrix
 
 Options:
--a  : agent version DEFAULT: ${AGENT_VERSION} ENV: AGENT_VERSION 
--h  : Show this help message
+-a  : agent version DEFAULT: ${AGENT_VERSION} ENV: AGENT_VERSION
+-h  : show this help message
 -o  : specify version of osbuilder
 -r  : rootfs directory DEFAULT: ${ROOTFS_DIR} ENV: ROOTFS_DIR
 
@@ -90,7 +90,7 @@ distro_needs_admin_caps()
 		then
 			echo "true"
 		elif [ "$1" = "debian" ]
-		then 
+		then
 			echo "true"
 		else
 			echo "false"

tests/test_config.sh

@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2018 SUSE LLC
+#
+# SPDX-License-Identifier: Apache-2.0
+
+distrosSystemd=(fedora centos ubuntu debian)
+distrosAgent=(alpine)
+
+if [ $MACHINE_TYPE != "ppc64le" ]; then
+	distrosSystemd+=(clearlinux)
+fi
+
+# "Not testing eurleros on Travis: (timeout, see: https://github.com/kata-containers/osbuilder/issues/46)"
+if [ -z "${TRAVIS:-}" ]; then
+	distrosSystemd+=(euleros)
+fi
+