Merge pull request #1750 from fuweid/refactor_1582

refactor: reorder hack scripts

.circleci/config.yml

@@ -30,29 +30,12 @@ jobs:
     steps:
       - checkout
       - run: 
-          name: install golint
-          command: go get -u golang.org/x/lint/golint
-      - run: 
-          name: install swagger-0.12.0
-          command: |
-            wget --quiet -O /bin/swagger https://github.com/go-swagger/go-swagger/releases/download/0.12.0/swagger_linux_amd64
-            chmod +x /bin/swagger
+          name: install gometalinter linters tools
+          command: hack/install/install_ci_related.sh
       - run: 
           name: use check target in Makefile to check code
           command: make check
-
-  unit-test:
-    docker:
-      - image: golang:1.9.1
-    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
-    steps:
-      - checkout
-      - run:
-          name: unit test
-          command: make unit-test
-      - run:
-          name: upload code coverage report
-          command: bash <(curl -s https://codecov.io/bash)
+
 notify:
   webhooks:
     - url: http://47.96.190.121:6788/circleci_notifications
@@ -63,4 +46,3 @@ workflows:
     jobs:
       - markdownlint-misspell-shellcheck
       - code-check
-      - unit-test

.gitignore

@@ -9,3 +9,5 @@ coverage.txt
 .bashrc
 .idea
 .DS_Store
+bin/
+coverage/

.gometalinter.json

@@ -0,0 +1,18 @@
+{
+  "Enable": [
+    "gofmt",
+    "goimports",
+    "golint",
+    "misspell",
+    "vet"
+  ],
+  "Deadline": "2m",
+  "Sort": ["linter", "severity", "path", "line"],
+  "Exclude": [
+    ".*\\.pb\\.go",
+	"pkg/jsonstream/image_progress\\.go:.*::error: unrecognized printf verb 'r'"
+  ],
+  "EnableGC": true,
+  "WarnUnmatchedDirective": true,
+  "Vendor": true
+}

.travis.yml

@@ -14,22 +14,34 @@ notifications:
     on_failure: always
     on_error: always
 
-before_install:
-  |
-    git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.md$)' || {
-      echo travis doesn\'t run for document-only changed
-      exit
-    }
-
-    sudo apt-get update -qq
-    sudo apt-get install -y -qq autoconf automake
-
 env:
-  - TEST_SUITE=integration-test
-  - TEST_SUITE=cri-test
-
-script:
-  - sudo make -e ${TEST_SUITE}
-
+  - TEST_SUITE=unittest
+  - TEST_SUITE=integrationtest
+  - TEST_SUITE=critest
+
+script: |
+  if [[ "${TEST_SUITE}" = "unittest" ]]; then
+    hack/install/install_ci_related.sh
+    make unit-test
+    make coverage
+  elif [[ "${TEST_SUITE}" = "integrationtest" ]]; then
+    make build
+    make build-daemon-integration
+    TEST_FLAGS= make build-integration-test
+    sudo env "PATH=$PATH" make install
+
+    sudo env "PATH=$PATH" make download_dependencies
+    sudo env "PATH=$PATH" make integration-test
+    make coverage
+  else
+    make build
+    TEST_FLAGS= make build-daemon-integration
+    sudo env "PATH=$PATH" make install
+
+    sudo env "PATH=$PATH" make download_dependencies
+    sudo env "PATH=$PATH" make cri-test
+    make coverage
+  fi
+  
 after_success:
-- bash <(curl -s https://codecov.io/bash) || echo Codecov_Did_Not_Collect_Coverage_Reports
+  - bash <(curl -s https://codecov.io/bash) -cF ${TEST_SUITE} -y .codecov.yml

Makefile

@@ -1,115 +1,160 @@
-# Go parameters
-GOBUILD=go build
-GOCLEAN=go clean
-GOTEST=go test
-GOPACKAGES=$(shell go list ./... | grep -v /vendor/ | sed 's/^_//')
-
-# Binary name of CLI and Daemon
-BINARY_NAME=pouchd
+# TEST_FLAGS used as flags of go test.
+TEST_FLAGS ?= -v --race
+
+# DAEMON_BINARY_NAME is the name of binary of daemon.
+DAEMON_BINARY_NAME=pouchd
+
+# CLI_BINARY_NAME is the name of binary of pouch client.
 CLI_BINARY_NAME=pouch
 
-# Base path used to install pouch & pouchd
-DESTDIR=/usr/local
+# DAEMON_INTEGRATION_BINARY_NAME is the name of test binary of daemon.
+DAEMON_INTEGRATION_BINARY_NAME=pouchd-integration
 
-.PHONY: build
-build: server client
+# INTEGRATION_TESTCASE_BINARY_NAME is the name of binary of integration cases.
+INTEGRATION_TESTCASE_BINARY_NAME=pouchd-integration-test
 
-.PHONY: pre
-pre:
-	@./hack/build pre
+# DEST_DIR is base path used to install pouch & pouchd
+DEST_DIR=/usr/local
 
-.PHONY: server
-server: pre modules
-	@./hack/build server
+# the following variables used for the daemon build
 
-.PHONY: client
-client: pre
-	@./hack/build client
+# API_VERSION is used for Daemon API Version in go build.
+API_VERSION="1.24"
 
-.PHONY: testserver
-testserver: pre modules
-	@./hack/build testserver
+# VERSION is used for Daemon Release Version in go build.
+VERSION ?= "1.0.0-rc1"
 
-.PHONY: clean
-clean:
-	$(GOCLEAN)
-	rm -f $(BINARY_NAME)
-	rm -f $(CLI_BINARY_NAME)
-	./hack/build clean
-	./hack/module --clean
+# GIT_COMMIT is used for Daemon GitCommit in go build.
+GIT_COMMIT=$(shell git describe --dirty --always --tags 2> /dev/null || true)
 
-.PHONY: check
-check: pre fmt lint vet validate-swagger
+# BUILD_TIME is used for Daemon BuildTime in go build.
+BUILD_TIME=$(shell date --rfc-3339 s 2> /dev/null | sed -e 's/ /T/')
 
-.PHONY: fmt
-fmt: ## run go fmt
-	@echo $@
-	@which gofmt
-	@test -z "$$(gofmt -s -l . 2>/dev/null | grep -Fv 'vendor/' | grep -v ".pb.go$$" | tee /dev/stderr)" || \
-		(echo "please format Go code with 'gofmt -s -w'" && false)
-	@test -z "$$(find . -path ./vendor -prune -o ! -name timestamp.proto ! -name duration.proto -name '*.proto' -type f -exec grep -Hn -e "^ " {} \; | tee /dev/stderr)" || \
-		(echo "please indent proto files with tabs only" && false)
-	@test -z "$$(find . -path ./vendor -prune -o -name '*.proto' -type f -exec grep -Hn "Meta meta = " {} \; | grep -v '(gogoproto.nullable) = false' | tee /dev/stderr)" || \
-		(echo "meta fields in proto files must have option (gogoproto.nullable) = false" && false)
-
-.PHONY: lint
-lint: ## run go lint
+VERSION_PKG=github.com/alibaba/pouch
+DEFAULT_LDFLAGS="-X ${VERSION_PKG}/version.GitCommit=${GIT_COMMIT} \
+		  -X ${VERSION_PKG}/version.Version=${VERSION} \
+		  -X ${VERSION_PKG}/version.ApiVersion=${API_VERSION} \
+		  -X ${VERSION_PKG}/version.BuildTime=${BUILD_TIME}"
+
+# COVERAGE_PACKAGES is the coverage we care about.
+COVERAGE_PACKAGES=$(shell go list ./... | \
+				  grep -v github.com/alibaba/pouch$$ | \
+				  grep -v github.com/alibaba/pouch/storage/volume/examples/demo | \
+				  grep -v github.com/alibaba/pouch/test | \
+				  grep -v github.com/alibaba/pouch/cli | \
+				  grep -v github.com/alibaba/pouch/cri/apis | \
+				  grep -v github.com/alibaba/pouch/apis/types )
+
+COVERAGE_PACKAGES_LIST=$(shell echo $(COVERAGE_PACKAGES) | tr " " ",")
+
+build: build-daemon build-cli ## build PouchContainer both daemon and cli binaries
+
+build-daemon: modules ## build PouchContainer daemon binary
 	@echo $@
-	@which golint
-	@test -z "$$(golint ./... | grep -Fv 'vendor/' | grep -v ".pb.go:" | tee /dev/stderr)"
+	@mkdir -p bin
+	GOOS=linux go build -ldflags ${DEFAULT_LDFLAGS} -o bin/${DAEMON_BINARY_NAME} -tags 'selinux' 
 
-.PHONY: vet
-vet: ## run go vet
+build-cli: ## build PouchContainer cli binary
 	@echo $@
-	@test -z "$$(./hack/build vet)"
+	@mkdir -p bin
+	go build -o bin/${CLI_BINARY_NAME} github.com/alibaba/pouch/cli
 
-.PHONY: unit-test
-unit-test: pre modules ## run go test
+build-daemon-integration: modules ## build PouchContainer Daemon Integration Testing binary
 	@echo $@
-	@./hack/build unit-test
+	@mkdir -p bin
+	go test -c ${TEST_FLAGS} \
+		-cover -covermode=atomic -coverpkg ${COVERAGE_PACKAGES_LIST} \
+		-o bin/${DAEMON_INTEGRATION_BINARY_NAME}
 
-.PHONY: validate-swagger
-validate-swagger: ## run swagger validate
+build-integration-test: modules ## build PouchContainer Integration test-case binary
 	@echo $@
-	@swagger validate apis/swagger.yml
+	@mkdir -p bin
+	go test -c \
+		-o bin/${INTEGRATION_TESTCASE_BINARY_NAME} github.com/alibaba/pouch/test
 
-.PHONY: modules
-modules:
+modules: ## run modules to generate volume related code
+	@echo $@
 	@./hack/module --clean
 	@./hack/module --add-volume=github.com/alibaba/pouch/storage/volume/modules/tmpfs
 	@./hack/module --add-volume=github.com/alibaba/pouch/storage/volume/modules/local
 
-# build binaries
-# install them to /usr/local/bin/
-# remove binaries
-.PHONY: install
-install: build ## build and install binary into /usr/local/bin
+install: ## install pouch and pouchd binary into /usr/local/bin
 	@echo $@
-	@echo "installing $(BINARY_NAME) and $(CLI_BINARY_NAME) to $(DESTDIR)/bin"
-	@mkdir -p $(DESTDIR)/bin
-	@install $(BINARY_NAME) $(DESTDIR)/bin
-	@install $(CLI_BINARY_NAME) $(DESTDIR)/bin
+	@mkdir -p $(DEST_DIR)/bin
+	install bin/$(CLI_BINARY_NAME) $(DEST_DIR)/bin
+	install bin/$(DAEMON_BINARY_NAME) $(DEST_DIR)/bin
 
-.PHONY: uninstall
 uninstall: ## uninstall pouchd and pouch binary
 	@echo $@
-	@rm -f $(addprefix $(DESTDIR)/bin/,$(notdir $(BINARY_NAME)))
-	@rm -f $(addprefix $(DESTDIR)/bin/,$(notdir $(CLI_BINARY_NAME)))
+	@rm -f $(addprefix $(DEST_DIR)/bin/,$(notdir $(DAEMON_BINARY_NAME)))
+	@rm -f $(addprefix $(DEST_DIR)/bin/,$(notdir $(CLI_BINARY_NAME)))
+
+.PHONY: download_dependencies
+download_dependencies:
+	@echo $@
+	hack/install/install_ci_related.sh
+	hack/install/install_containerd.sh
+	hack/install/install_dumb_init.sh
+	hack/install/install_local_persist.sh
+	hack/install/install_lxcfs.sh
+	hack/install/install_nsenter.sh
+	hack/install/install_runc.sh
+
+.PHONY: clean
+clean: ## clean to remove bin/* and files created by module
+	@go clean
+	@rm -f bin/*
+	@rm -rf coverage/*
+	@./hack/module --clean
+
+
+.PHONY: check
+check: gometalinter validate-swagger ## run all linters
+
+.PHONY: validate-swagger
+validate-swagger: ## run swagger validate
+	@echo $@
+	./hack/validate_swagger.sh
+
+# gometalinter consumes .gometalinter.json as config.
+.PHONY: gometalinter
+gometalinter: ## run gometalinter for go source code
+	@echo $@
+	gometalinter --config .gometalinter.json ./...
+
+
+.PHONY: unit-test
+unit-test: modules ## run go unit-test
+	@echo $@
+	@mkdir -p coverage
+	@( for pkg in ${COVERAGE_PACKAGES}; do \
+		go test ${TEST_FLAGS} \
+			-cover -covermode=atomic \
+			-coverprofile=coverage/unit-test-`echo $$pkg | tr "/" "_"`.out \
+			$$pkg || exit; \
+	done )
 
-# For integration-test and test, PATH is not set under sudo, then we set up path mannually.
-# Ref https://unix.stackexchange.com/questions/83191/how-to-make-sudo-preserve-path
 .PHONY: integration-test
-integration-test: ## build binary and run integration-test
-	@bash -c "env PATH=$(PATH) hack/make.sh build integration-test"
+integration-test: ## run daemon integration-test
+	@echo $@
+	@mkdir -p coverage
+	./hack/testing/run_daemon_integration.sh
 
 .PHONY: cri-test
-cri-test: ## build binary and run cri-test
-	@bash -c "env PATH=$(PATH) hack/make.sh build cri-test"
+cri-test: ## run v1 alpha2 cri-test
+	@echo $@
+	@mkdir -p coverage
+	./hack/testing/run_daemon_cri_integration.sh
 
 .PHONY: test
-test: ## run the build integration-test cri-test
-	@bash -c "env PATH=$(PATH) hack/make.sh build integration-test cri-test"
+test: unit-test integration-test cri-test ## run the unit-test, integration-test and cri-test
+
+.PHONY: coverage
+coverage: ## combine coverage after test
+	@echo $@
+	@gocovmerge coverage/* > coverage.txt
+
 
 .PHONY: help
 help: ## this help
-	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-28s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)

client/image_list.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"context"
+
 	"github.com/alibaba/pouch/apis/types"
 )
 

hack/codegen/swagger.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly SWAGGER_VERSION=0.12.0
+
+# swagger::check_version checks the command and the version.
+swagger::check_version() {
+  local has_installed version
+
+  has_installed="$(command -v swagger || echo false)"
+  if [[ "${has_installed}" = "false" ]]; then
+    echo false
+    return
+  fi
+
+  version="$(swagger version | head -n 1 | cut -d " " -f 2)"
+  if [[ "${SWAGGER_VERSION}" != "${version}" ]]; then
+    echo false
+    return
+  fi
+  echo true
+}
+
+# swagger::install installs the swagger binary.
+swagger::install() {
+  echo ">>>> install swagger-${SWAGGER_VERSION} <<<<"
+  local url
+  url="https://github.com/go-swagger/go-swagger/releases/download"
+  url="${url}/${SWAGGER_VERSION}/swagger_linux_amd64"
+
+  wget --quiet -O /usr/local/bin/swagger "${url}"
+  chmod +x /usr/local/bin/swagger
+}