test: refact travis-ci

refact travis-ci, make it run faster.

1. remove CRI v1alpha1 test
2. split the integration into multiple parts, let them run together,
because the limit of travis ci jobs, we only can split four parts.
3. update travis-ci os version to ubuntu-16.04 xenial,
because in this version, we can use apt-get to install criu, instead of
building it by ourself, it can reduce the setup time.
4. I have built `nsenter` and upload to aliyun oss storage server,
so only download it instead of building it every time.

Signed-off-by: Rudy Zhang <rudyflyzhang@gmail.com>

.gitignore

@@ -13,4 +13,3 @@ coverage.txt
 .DS_Store
 bin/
 coverage/
-install/

.travis.yml

@@ -1,4 +1,4 @@
-dist: trusty
+dist: xenial
 sudo: required
 
 language: go
@@ -22,53 +22,29 @@ notifications:
     on_failure: always
     on_error: always
 
-env:
-  - TEST_SUITE=unittest
-  - TEST_SUITE=integrationtest
-  - TEST_SUITE=criv1alpha1test
-  - TEST_SUITE=criv1alpha2test
-  - TEST_SUITE=nodee2etest
-
-script: |
-  set -e # fast fail
-  if [[ "${TEST_SUITE}" = "unittest" ]]; then
-    sudo env "PATH=$PATH" hack/install/install_ci_related.sh
-    make unit-test
-    make coverage
-  elif [[ "${TEST_SUITE}" = "integrationtest" ]]; then
-    make build
-    make build-daemon-integration
-    TEST_FLAGS= make build-integration-test
-    sudo env "PATH=$PATH" make install
-
-    sudo env "PATH=$PATH" make download-dependencies
-    sudo env "PATH=$PATH" make integration-test
-    make coverage
-  elif [[ "${TEST_SUITE}" = "criv1alpha1test" ]]; then
-    make build
-    TEST_FLAGS= BUILDTAGS="selinux seccomp apparmor" make build-daemon-integration
-    sudo env "PATH=$PATH" make install
-
-    sudo env "PATH=$PATH" make download-dependencies
-    sudo env "PATH=$PATH" make cri-v1alpha1-test
-    make coverage
-  elif [[ "${TEST_SUITE}" = "criv1alpha2test" ]]; then
-    make build
-    TEST_FLAGS= BUILDTAGS="selinux seccomp apparmor" make build-daemon-integration
-    sudo env "PATH=$PATH" make install
-
-    sudo env "PATH=$PATH" make download-dependencies
-    sudo env "PATH=$PATH" make cri-v1alpha2-test
-    make coverage
-  else
-    make build
-    TEST_FLAGS= make build-daemon-integration
-    sudo env "PATH=$PATH" make install
-
-    sudo env "PATH=$PATH" make download-dependencies
-    sudo env "PATH=$PATH" make cri-e2e-test
-    make coverage
-  fi
-  
-after_success:
-  - bash <(curl -s https://codecov.io/bash) -cF ${TEST_SUITE} -y .codecov.yml
+jobs:
+  include:
+    - stage: test
+      name: "Unit test"
+      script: ./travis_ci.sh unittest
+
+    - stage: test
+      name: "CRIv1alpha2 test"
+      script: ./travis_ci.sh criv1alpha2test
+
+    - stage: test
+      name: "Node e2e test"
+      script: ./travis_ci.sh nodee2etest
+
+    - stage: test
+      name: "Integration test 0"
+      script: ./travis_ci.sh integrationtest 4 0
+    - stage: test
+      name: "Integration test 1"
+      script: ./travis_ci.sh integrationtest 4 1
+    - stage: test
+      name: "Integration test 2"
+      script: ./travis_ci.sh integrationtest 4 2
+    - stage: test
+      name: "Integration test 3"
+      script: ./travis_ci.sh integrationtest 4 3

Makefile

@@ -15,6 +15,9 @@
 # TEST_FLAGS used as flags of go test.
 TEST_FLAGS ?= -v --race
 
+# INTEGRATION_FLAGS used as flags of run integration test
+INTEGRATION_FLAGS ?= ""
+
 # DAEMON_BINARY_NAME is the name of binary of daemon.
 DAEMON_BINARY_NAME=pouchd
 
@@ -83,6 +86,7 @@ COVERAGE_PACKAGES=$(shell go list ./... | \
 				  grep -v github.com/alibaba/pouch/cli$$ | \
 				  grep -v github.com/alibaba/pouch/cli/ | \
 				  grep -v github.com/alibaba/pouch/cri/apis | \
+				  grep -v github.com/alibaba/pouch/cri/v1alpha1 | \
 				  grep -v github.com/alibaba/pouch/apis/types )
 
 COVERAGE_PACKAGES_LIST=$(shell echo $(COVERAGE_PACKAGES) | tr " " ",")
@@ -208,12 +212,6 @@ build-daemon-integration: modules plugin ## build PouchContainer daemon integrat
 		-cover -covermode=atomic -coverpkg ${COVERAGE_PACKAGES_LIST} \
 		-o bin/${DAEMON_INTEGRATION_BINARY_NAME}
 
-build-integration-test: modules plugin ## build PouchContainer integration test-case binary
-	@echo $@
-	@mkdir -p bin
-	go test -c \
-		-o bin/${INTEGRATION_TESTCASE_BINARY_NAME} github.com/alibaba/pouch/test
-
 modules: ## run modules to generate volume related code
 	@echo "build volume $@"
 	@./hack/module --clean
@@ -283,10 +281,10 @@ unit-test: modules plugin ## run go unit-test
 	done )
 
 .PHONY: integration-test
-integration-test: build-daemon-integration build-integration-test ## run daemon integration-test
+integration-test: build-daemon-integration ## run daemon integration-test
 	@echo $@
 	@mkdir -p coverage
-	./hack/testing/run_daemon_integration.sh
+	./hack/testing/run_daemon_integration.sh ${INTEGRATION_FLAGS}
 
 .PHONY: cri-v1alpha1-test
 cri-v1alpha1-test: ## run v1 alpha1 cri-v1alpha1-test

hack/install/config.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export OSS_ENDPOINT="oss-cn-beijing.aliyuncs.com"
+export OSS_BUCKET="pouch-oss"

hack/install/install_critest.sh

@@ -61,8 +61,14 @@ critest::install_ginkgo() {
   hack/install/install_ginkgo.sh
 }
 
+# critest::install_socat installs socat if missing.
+critest::install_socat() {
+  sudo apt-get install -y socat
+}
+
 main() {
   critest::install_ginkgo
+  critest::install_socat
 
   local cri_runtime has_installed
   cri_runtime=$1

hack/install/install_criu.sh

@@ -5,33 +5,9 @@ set -euo pipefail
 cd "$(dirname "${BASH_SOURCE[0]}")"
 source "./check.sh"
 
-# criu::ubuntu::install_dependencies will install dependencies packages.
-criu::ubuntu::install_dependencies() {
-  apt-get install -y -q \
-    build-essential \
-    libnet1-dev \
-    libprotobuf-dev \
-    libprotobuf-c0-dev \
-    protobuf-c-compiler \
-    protobuf-compiler \
-    python-protobuf \
-    libnl-3-dev \
-    libcap-dev \
-    asciidoc
-}
-
 # criu::ubuntu::install will install criu from source.
 criu::ubuntu::install() {
-  local tmpdir tag
-
-  tag="v3.10"
-  tmpdir="$(mktemp -d /tmp/criu-build-XXXXXX)"
-
-  git clone https://github.com/checkpoint-restore/criu.git "${tmpdir}/criu"
-  cd "${tmpdir}/criu"
-  git checkout "${tag}" -b "${tag}"
-  make
-  make install
+  apt-get install -y -q criu
 }
 
 main() {
@@ -47,8 +23,6 @@ main() {
 
   os_dist="$(detect_os)"
   if [[ "${os_dist}" = "Ubuntu" ]]; then
-    criu::ubuntu::install_dependencies > /dev/null
-    echo ">>>> start to download criu from github repository <<<<"
     criu::ubuntu::install > /dev/null
   else
     echo "will support redhat soon"

hack/install/install_nsenter.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 
 cd "$(dirname "${BASH_SOURCE[0]}")"
 source "./check.sh"
+source "./config.sh"
 
 readonly NSENTER_VERSION="2.24.1"
 readonly cmd="nsenter"
@@ -29,43 +30,15 @@ nsenter::check_version() {
 
 # nsenter::ubuntu::install_dependencies will use apt-get to install dependencies.
 nsenter::ubuntu::install_dependencies() {
-  apt-get install -y -q \
-    libncurses5-dev \
-    libslang2-dev \
-    gettext \
-    zlib1g-dev \
-    libselinux1-dev \
-    debhelper \
-    lsb-release \
-    pkg-config \
-    po-debconf \
-    autoconf \
-    automake \
-    autopoint \
-    libtool
+  apt-get install -y -q wget
 }
 
 # nsenter::ubuntu::install will install nsenter.
+# TODO: change to get binary from aliyun oss storage.
 nsenter::ubuntu::install() {
-  local url target tmpdir
-
-  target="util-linux-${NSENTER_VERSION}.tar.gz"
-  url="https://www.kernel.org/pub/linux/utils/util-linux/v2.24"
-  url="${url}/${target}"
-
-  tmpdir="$(mktemp -d /tmp/nsenter-install-XXXXXX)"
-  trap 'rm -rf /tmp/nsenter-install-*' EXIT
-
-  wget --quiet "${url}" -P "${tmpdir}"
-  tar xf "${tmpdir}/${target}" -C "${tmpdir}"
-
-  cd "${tmpdir}/util-linux-${NSENTER_VERSION}"
-  ./autogen.sh
-  autoreconf -vfi
-  ./configure
-  make
-
-  cp "${cmd}" /usr/local/bin
+  wget "https://${OSS_BUCKET}.${OSS_ENDPOINT}/pouch-test/ubuntu/nsenter-2.24.1" \
+    -O /usr/local/bin/nsenter
+  chmod +x /usr/local/bin/nsenter
 }
 
 # nsenter::centos::install will install nsenter.

hack/install/install_runc.sh

@@ -11,13 +11,14 @@ source "./check.sh"
 # runc::ubuntu::install_dependencies() install dependencies
 # on ubuntu machine for make runc
 runc::ubuntu::install_dependencies() {
-  sudo apt-get install -y libseccomp-dev/trusty-backports
+  sudo apt-get update
+  sudo apt-get install -y libseccomp-dev
 }
 
 # runc::centos::install_dependencies() install dependencies 
 # on centos machine for make runc
 runc::centos::install_dependencies() {
-  sudo yum install libseccomp-dev
+  sudo yum install -y libseccomp-dev
 }
 
 # runc::check_install checks the command and the version.

hack/testing/run_daemon_cri_integration.sh

@@ -58,7 +58,7 @@ integration::run_daemon_cri_test_cases() {
       --focus="${CRI_FOCUS}" --ginkgo-flags="--skip=\"${CRI_SKIP}\"" validation
   else
     critest --runtime-endpoint=${POUCH_SOCK} \
-      --ginkgo.focus="${CRI_FOCUS}" --ginkgo.skip="${CRI_SKIP}"
+      --ginkgo.focus="${CRI_FOCUS}" --ginkgo.skip="${CRI_SKIP}" --parallel=8
   fi
   code=$?
 

hack/testing/run_daemon_integration.sh

@@ -23,13 +23,23 @@ rm -rf "${coverage_profile}"
 # integration::run_daemon_test_cases starts cases.
 integration::run_daemon_test_cases() {
   echo "start pouch daemon integration test..."
-  local code
+  local code=0
+  local job_id=$1
 
   cp -rf "${REPO_BASE}/test/tls" /tmp/
 
   set +e
-  "${REPO_BASE}/bin/pouchd-integration-test" -test.v -check.v
-  code=$?
+  pushd "${REPO_BASE}/test"
+  local testcases
+  testcases=$(cat "${REPO_BASE}/test/testcase.list.${job_id}")
+  for one in ${testcases}; do
+    go test -check.v -check.f "${one}"
+    ret=$?
+    if [[ ${ret} -ne 0 ]]; then
+      code=${ret}
+    fi
+  done
+
   integration::stop_local_persist
   integration::stop_pouchd
   set -e
@@ -47,6 +57,7 @@ integration::run_daemon_test_cases() {
 
 main() {
   local cmd flags
+  local job_id=$1
   cmd="pouchd-integration"
   flags=" -test.coverprofile=${coverage_profile} DEVEL"
   flags="${flags} --debug --enable-lxcfs --add-runtime runv=runv"
@@ -66,7 +77,7 @@ main() {
     cat "${pouchd_log}"
     exit ${code}
   fi
-  integration::run_daemon_test_cases
+  integration::run_daemon_test_cases "${job_id}"
 }
 
 main "$@"

hack/testing/utils.sh

@@ -12,7 +12,10 @@ integration::run_local_persist_background() {
 # integration::stop_local_persist stop local-persist.
 integration::stop_local_persist() {
   echo "stop local-persist volume plugin..."
-  set +e; pkill local-persist; set -e
+  set +e
+  pkill local-persist
+  rm -rf /run/docker/plugins/local-persist.sock
+  set -e
 }
 
 # integration::run_mount_lxcfs_background runs lxcfs mount to /var/lib/lxcfs.
@@ -23,8 +26,8 @@ integration::run_mount_lxcfs_background() {
 
 # integration::stop_mount_lxcfs stop lxcfs mount.
 integration::stop_mount_lxcfs() {
-    echo "stop lxcfs /var/lib/lxcfs..."
-    set +e; pkill lxcfs; rm -rf /run/lxcfs.pid; set -e
+  echo "stop lxcfs /var/lib/lxcfs..."
+  set +e; pkill lxcfs; rm -rf /run/lxcfs.pid; set -e
 }
 
 # integration::run_pouchd_background runs pouchd in background.
@@ -59,7 +62,7 @@ integration::ping_pouchd() {
       break
     elif (( $((timeout--)) == 0 ));then
       echo "failed to start pouch daemon in background!"
-      exit 1
+      return 1
     fi
     sleep 1
   done