[CE-223] Decouple the local image gen from setup

Now we have images from dockerhub for x86 platform. So in the setup step, we can pull the images from dockerhub. And also we decouple the image pulling from start step, but adding notice needing the image locally. Change-Id: I9ab310c9cc76a352fcfbd9354bfc05db46e42450 Signed-off-by: Baohua Yang <yangbaohua@gmail.com>
hyperledger · Jan 10, 2018 · bf06e57 · bf06e57
1 parent b14ec7f
commit bf06e57
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 85 deletions.
diff --git a/Makefile b/Makefile
@@ -7,15 +7,15 @@
 # This makefile defines the following targets
 #
 #   - all (default):  Builds all targets and runs all tests/checks
-#   - checks:         Runs all tests/checks, will be triggered by CI
+#   - checks:         Setup as master node, and runs all tests/checks, will be triggered by CI
 #   - clean:          Cleans the build area
 #   - doc:            Start a local web service to explore the documentation
-#   - docker[-clean]: Ensures all docker images are available[/cleaned]
+#   - docker[-clean]: Build/clean docker images locally
 #   - help:           Output the help instructions for each command
 #   - log:            Check the recent log output of all services
 #   - restart:        Stop the cello service and then start
-#   - setup-master:   Setup the host as a master node
-#   - setup-worker:   Setup the host as a worker node
+#   - setup-master:   Setup the host as a master node, install pkg and download docker images
+#   - setup-worker:   Setup the host as a worker node, install pkg and download docker images
 #   - start:          Start the cello service
 #   - stop:           Stop the cello service, and remove all service containers
 
@@ -100,7 +100,7 @@ build/docker/baseimage/$(DUMMY): build/docker/baseimage/$(DUMMY)
 build/docker/nginx/$(DUMMY): build/docker/nginx/$(DUMMY)
 build/docker/mongo/$(DUMMY): build/docker/mongo/$(DUMMY)
 
-build/docker/%/$(DUMMY):
+build/docker/%/$(DUMMY): ##@Build an image locally
 	$(eval TARGET = ${patsubst build/docker/%/$(DUMMY),%,${@}})
 	$(eval IMG_NAME = $(BASENAME)-$(TARGET))
 	@mkdir -p $(@D)
@@ -110,12 +110,10 @@ build/docker/%/$(DUMMY):
 		| sed -e 's|_NS_|$(DOCKER_NS)|g' \
 		| sed -e 's|_TAG_|$(IMG_TAG)|g' \
 		> $(@D)/Dockerfile
-	if [ "$$(docker images -q $(IMG_NAME) 2> /dev/null)" == "" ]; then \
-		docker build -f $(@D)/Dockerfile \
-			-t $(IMG_NAME) \
-			-t $(IMG_NAME):$(IMG_TAG) \
-			. ; \
-	fi
+	docker build -f $(@D)/Dockerfile \
+		-t $(IMG_NAME) \
+		-t $(IMG_NAME):$(IMG_TAG) \
+		. ; \
 	@touch $@
 
 build/docker/%/.push: build/docker/%/$(DUMMY)
@@ -124,7 +122,7 @@ build/docker/%/.push: build/docker/%/$(DUMMY)
 		--password=$(DOCKER_HUB_PASSWORD)
 	@docker push $(BASENAME)-$(patsubst build/docker/%/.push,%,$@):$(IMG_TAG)
 
-docker: $(patsubst %,build/docker/%/$(DUMMY),$(DOCKER_IMAGES))
+docker: $(patsubst %,build/docker/%/$(DUMMY),$(DOCKER_IMAGES)) ##@Generate docker images locally
 
 docker-clean: image-clean ##@Clean all existing images
 
@@ -148,7 +146,7 @@ changelog: ##@Update the changelog.md file in the root folder
 	#bash scripts/changelog.sh bd0c6db v$(PREV_VERSION)
 	bash scripts/changelog.sh v$(PREV_VERSION) HEAD
 
-doc: ##@Create local online documentation
+doc: ##@Create local online documentation and start serve
 	pip install mkdocs
 	mkdocs serve
 
@@ -183,9 +181,9 @@ initial-env: ##@Configuration Initial Configuration for dashboard
 	$(SED) 's/\(WEBROOT=\).*/\1${WEBROOT}/' .env
 	$(SED) 's/\(THEME=\).*/\1${THEME}/' .env
 
-start: docker ##@Service Start service
+start: ##@Service Start service
 	@$(MAKE) $(START_OPTIONS)
-	echo "Start all services..."
+	echo "Start all services... docker images must exist local now, otherwise, run `make setup-master first` !"
 	docker-compose up -d --no-recreate
 
 stop: ##@Service Stop service
@@ -196,7 +194,7 @@ stop: ##@Service Stop service
 
 restart: stop start ##@Service Restart service
 
-setup-master: docker ##@Environment Setup dependency for master node
+setup-master: ##@Environment Setup dependency for master node
 	cd scripts/master_node && bash setup.sh
 
 setup-worker: ##@Environment Setup dependency for worker node

diff --git a/docs/setup_master.md b/docs/setup_master.md
@@ -31,6 +31,8 @@ $ git clone http://gerrit.hyperledger.org/r/cello && cd cello
 
 For the first time running, please setup the master node with the [setup.sh](https://github.com/hyperledger/cello/blob/master/scripts/master_node/setup.sh).
 
+Just run (safe to repeat it):
+
 ```sh
 $ make setup-master
 ```
@@ -48,7 +50,7 @@ $ make start
 
 To stop or restart the whole services, run `make stop` or `make restart`.
 
-#### Redploy a service
+#### Redeploy a service
 To redeploy one specific service, e.g., dashboard, please run
 
 ```sh
@@ -83,8 +85,8 @@ Please keep it safe by backups or using more high-available solutions.
 
 In MacOS, Docker cannot mount local path from host by default. Hence for mongo container data volume, users need to:
 
-* Make sure the `/opt/cello` path exists locally, and it is writable for the current user account. `make setup-master` should handle this already.
-* Add the path to sharing list in the preference of [Docker for MacOS](https://docs.docker.com/docker-for-mac/install/), to make it mountable by container.
+* Make sure the `/opt/cello` path exists locally, and it is writable for the current user account. Simply just run `make setup-master`.
+* Add the path to `File Sharing` list in the preference of [Docker for MacOS](https://docs.docker.com/docker-for-mac/install/), to make it mountable by container.
 
 ## Cello Baseimage
 ![BaseImage](imgs/cello_baseimage.png)
@@ -94,7 +96,7 @@ required for supporting a Hyperledger Cello environment. The build process is ge
 inefficient to JIT assemble these components on demand. Hence bundled into baseimage and subsequently cached on
 the public repositories, so they can be simply consumed without requiring a local build cycle.
 
-### Usuage
+### Usage
 * "make docker" will build the docker images and commit it to your local environment; e.g. "hyperledger/cello-baseimage".
 The docker image is also tagged with architecture and release details.
 

diff --git a/scripts/master_node/download_images.sh b/scripts/master_node/download_images.sh
@@ -25,13 +25,23 @@ fi
 
 echo_b "Downloading the docker images for master node"
 
+# TODO: will be removed after we have the user dashboard image
+echo_b "Check node:9.2 image."
+[ -z "$(docker images -q node:9.2 2> /dev/null)" ] && { echo "pulling node:9.2"; docker pull node:9.2; }
 
+# docker image
 ARCH=$(uname -m)
+VERSION=latest
 
-docker pull node:9.2 \
-	&& docker pull hyperledger/cello-baseimage:${ARCH}-latest \
-	&& docker pull hyperledger/cello-mongo:${ARCH}-latest \
-	&& docker pull hyperledger/cello-nginx:${ARCH}-latest \
-	&& docker tag hyperledger/cello-baseimage:${ARCH}-latest hyperledger/cello-baseimage \
-	&& docker tag hyperledger/cello-mongo:${ARCH}-latest hyperledger/cello-mongo \
-	&& docker tag hyperledger/cello-nginx:${ARCH}-latest hyperledger/cello-nginx
+for IMG in baseimage mongo nginx ; do
+	HC_IMG=hyperledger/cello-${IMG}
+	if [ -z "$(docker images -q ${HC_IMG} 2> /dev/null)" ]; then  # not exist
+		echo_b "Pulling ${HC_IMG}:${ARCH}-${VERSION} from dockerhub"
+		docker pull ${HC_IMG}:${ARCH}-${VERSION}
+		docker tag ${HC_IMG}:${ARCH}-${VERSION} ${HC_IMG}
+	else
+		echo_g "${HC_IMG} already exist locally"
+	fi
+done
+
+echo_g "All Image downloaded "
diff --git a/scripts/master_node/setup.sh b/scripts/master_node/setup.sh
@@ -40,70 +40,77 @@ get_distribution() {
 	echo "${distribution//\"}"
 }
 
+# Install necessary software including curl, python-pip, tox, docker
+install_software() {
+	case $DISTRO in
+		ubuntu)
+			sudo apt-get update && sudo apt-get install -y curl python-pip tox;
+			command -v docker >/dev/null 2>&1 || { echo_r >&2 "No docker-engine found, try installing"; curl -sSL https://get.docker.com/ | sh; sudo service docker restart; }
+			command -v docker-compose >/dev/null 2>&1 || { echo_r >&2 "No docker-compose found, try installing"; sudo pip install 'docker-compose>=1.17.0'; }
+			;;
+		linuxmint)
+			sudo apt-get install apt-transport-https ca-certificates -y
+			sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+			sudo echo deb https://apt.dockerproject.org/repo ubuntu-xenial main >> /etc/apt/sources.list.d/docker.list
+			sudo apt-get update
+			sudo apt-get purge lxc-docker
+			sudo apt-get install python-pip
+			sudo apt-get install linux-image-extra-$(uname -r) -y
+			sudo apt-get install docker-engine cgroup-lite apparmor -y
+			sudo service docker start
+			;;
+		debian)
+			sudo apt-get install apt-transport-https ca-certificates -y
+			sudo sh -c "echo deb https://apt.dockerproject.org/repo debian-jessie main > /etc/apt/sources.list.d/docker.list"
+			sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+			sudo apt-get update
+			sudo apt-cache policy docker-engine
+			sudo apt-get install docker-engine curl python-pip  -y
+			sudo service docker start
+			;;
+		centos)
+			sudo yum install -y epel-release yum-utils
+			sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+			sudo yum makecache fast
+			sudo yum update && sudo yum install -y docker-ce python-pip
+			sudo systemctl enable docker
+			sudo systemctl start docker
+			;;
+		fedora)
+			sudo dnf -y update
+			sudo dnf -y install  docker python-pip --allowerasing
+			sudo systemctl enable docker
+			sudo systemctl start docker
+			;;
+		opensuse)
+			sudo zypper refresh
+			sudo zypper install docker docker-compose python-pip
+			sudo systemctl restart docker
+			;;
+		*)
+			echo "Linux distribution not identified !!! skipping docker & pip installation"
+			;;
+	esac
+}
+
 USER=`whoami`
 DISTRO=$(get_distribution)
 DB_DIR=/opt/${PROJECT}/mongo
 
+echo_b "Check python-pip, tox, curl and docker-engine for $DISTRO"
+NEED_INSTALL="false"
+for software in pip tox curl docker docker-compose; do
+	command -v ${software} >/dev/null 2>&1 || { NEED_INSTALL="true"; break; }
+done
+[ $NEED_INSTALL = "true" ] && install_software
 
-echo_b "Install python-pip, tox, and docker-engine"
-case $DISTRO in
-	ubuntu)
-		sudo apt-get update && sudo apt-get install -y curl python-pip tox;
-		echo_b "Checking to install Docker-engine..."
-		command -v docker >/dev/null 2>&1 || { echo_r >&2 "No docker-engine found, try installing"; curl -sSL https://get.docker.com/ | sh; sudo service docker restart; }
-    ;;
-	linuxmint)
-		sudo apt-get install apt-transport-https ca-certificates -y
-    sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-		sudo echo deb https://apt.dockerproject.org/repo ubuntu-xenial main >> /etc/apt/sources.list.d/docker.list
-		sudo apt-get update
-		sudo apt-get purge lxc-docker
-		sudo apt-get install python-pip
-		sudo apt-get install linux-image-extra-$(uname -r) -y
-		sudo apt-get install docker-engine cgroup-lite apparmor -y
-		sudo service docker start
-    ;;
-  debian)
-		sudo apt-get install apt-transport-https ca-certificates -y
-		sudo sh -c "echo deb https://apt.dockerproject.org/repo debian-jessie main > /etc/apt/sources.list.d/docker.list"
-		sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-		sudo apt-get update
-		sudo apt-cache policy docker-engine
-		sudo apt-get install docker-engine curl python-pip  -y
-		sudo service docker start
-		;;
-  centos)
-		sudo yum install -y epel-release yum-utils
-		sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-		sudo yum makecache fast
-		sudo yum update && sudo yum install -y docker-ce python-pip
-		sudo systemctl enable docker
-		sudo systemctl start docker
-		;;
-  fedora)
-		sudo dnf -y update
-		sudo dnf -y install  docker python-pip --allowerasing
-		sudo systemctl enable docker
-		sudo systemctl start docker
-		;;
-  opensuse)
-		sudo zypper refresh
-		sudo zypper install docker docker-compose python-pip
-		sudo systemctl restart docker
-		;;
-  *)
-		echo "Linux distribution not identified !!! skipping docker & pip installation"
-		;;
-esac
-
-echo_b "Add existing user to docker group"
+echo_b "Add existing user ${USER} to docker group"
 sudo usermod -aG docker ${USER}
 
-echo_b "Checking to install Docker-compose..."
-command -v docker-compose >/dev/null 2>&1 || { echo_r >&2 "No docker-compose found, try installing"; sudo pip install 'docker-compose>=1.17.0'; }
-
-[ `sudo docker ps -qa|wc -l` -gt 0 ] \
-	&& echo_r "Warn: existing containers may cause unpredictable failure, suggest to clean them using docker rm"
+if [ `sudo docker ps -qa|wc -l` -gt 0 ]; then
+	echo_r "Warn: existing containers may cause unpredictable failure, suggest to clean them (docker rm)"
+	docker ps -a
+fi
 
 echo_b "Download required Docker images..."
 bash ./download_images.sh
@@ -115,4 +122,4 @@ echo_b "Checking local mounted database path ${DB_DIR}..."
 	&& sudo chown -R ${USER}:${USER} ${DB_DIR}
 
 echo_g "Setup done, please logout and login again."
-echo_g "It's safe to run this script repeatedly. Just re-run if it fails."
+echo_g "It's safe to run this script repeatedly."