Skip to content

Commit bf06e57

Browse files
yeasyyeasy
committed
[CE-223] Decouple the local image gen from setup
Now we have images from dockerhub for x86 platform. So in the setup step, we can pull the images from dockerhub. And also we decouple the image pulling from start step, but adding notice needing the image locally. Change-Id: I9ab310c9cc76a352fcfbd9354bfc05db46e42450 Signed-off-by: Baohua Yang <yangbaohua@gmail.com>
1 parent b14ec7f commit bf06e57

File tree

4 files changed

+102
-85
lines changed

4 files changed

+102
-85
lines changed

‎Makefile

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,15 @@
77
# This makefile defines the following targets
88
#
99
# - all (default): Builds all targets and runs all tests/checks
10-
# - checks: Runs all tests/checks, will be triggered by CI
10+
# - checks: Setup as master node, and runs all tests/checks, will be triggered by CI
1111
# - clean: Cleans the build area
1212
# - doc: Start a local web service to explore the documentation
13-
# - docker[-clean]: Ensures all docker images are available[/cleaned]
13+
# - docker[-clean]: Build/clean docker images locally
1414
# - help: Output the help instructions for each command
1515
# - log: Check the recent log output of all services
1616
# - restart: Stop the cello service and then start
17-
# - setup-master: Setup the host as a master node
18-
# - setup-worker: Setup the host as a worker node
17+
# - setup-master: Setup the host as a master node, install pkg and download docker images
18+
# - setup-worker: Setup the host as a worker node, install pkg and download docker images
1919
# - start: Start the cello service
2020
# - stop: Stop the cello service, and remove all service containers
2121

@@ -100,7 +100,7 @@ build/docker/baseimage/$(DUMMY): build/docker/baseimage/$(DUMMY)
100100
build/docker/nginx/$(DUMMY): build/docker/nginx/$(DUMMY)
101101
build/docker/mongo/$(DUMMY): build/docker/mongo/$(DUMMY)
102102

103-
build/docker/%/$(DUMMY):
103+
build/docker/%/$(DUMMY): ##@Build an image locally
104104
$(eval TARGET = ${patsubst build/docker/%/$(DUMMY),%,${@}})
105105
$(eval IMG_NAME = $(BASENAME)-$(TARGET))
106106
@mkdir -p $(@D)
@@ -110,12 +110,10 @@ build/docker/%/$(DUMMY):
110110
| sed -e 's|_NS_|$(DOCKER_NS)|g' \
111111
| sed -e 's|_TAG_|$(IMG_TAG)|g' \
112112
> $(@D)/Dockerfile
113-
if [ "$$(docker images -q $(IMG_NAME) 2> /dev/null)" == "" ]; then \
114-
docker build -f $(@D)/Dockerfile \
115-
-t $(IMG_NAME) \
116-
-t $(IMG_NAME):$(IMG_TAG) \
117-
. ; \
118-
fi
113+
docker build -f $(@D)/Dockerfile \
114+
-t $(IMG_NAME) \
115+
-t $(IMG_NAME):$(IMG_TAG) \
116+
. ; \
119117
@touch $@
120118

121119
build/docker/%/.push: build/docker/%/$(DUMMY)
@@ -124,7 +122,7 @@ build/docker/%/.push: build/docker/%/$(DUMMY)
124122
--password=$(DOCKER_HUB_PASSWORD)
125123
@docker push $(BASENAME)-$(patsubst build/docker/%/.push,%,$@):$(IMG_TAG)
126124

127-
docker: $(patsubst %,build/docker/%/$(DUMMY),$(DOCKER_IMAGES))
125+
docker: $(patsubst %,build/docker/%/$(DUMMY),$(DOCKER_IMAGES)) ##@Generate docker images locally
128126

129127
docker-clean: image-clean ##@Clean all existing images
130128

@@ -148,7 +146,7 @@ changelog: ##@Update the changelog.md file in the root folder
148146
#bash scripts/changelog.sh bd0c6db v$(PREV_VERSION)
149147
bash scripts/changelog.sh v$(PREV_VERSION) HEAD
150148

151-
doc: ##@Create local online documentation
149+
doc: ##@Create local online documentation and start serve
152150
pip install mkdocs
153151
mkdocs serve
154152

@@ -183,9 +181,9 @@ initial-env: ##@Configuration Initial Configuration for dashboard
183181
$(SED) 's/\(WEBROOT=\).*/\1${WEBROOT}/' .env
184182
$(SED) 's/\(THEME=\).*/\1${THEME}/' .env
185183

186-
start: docker ##@Service Start service
184+
start: ##@Service Start service
187185
@$(MAKE) $(START_OPTIONS)
188-
echo "Start all services..."
186+
echo "Start all services... docker images must exist local now, otherwise, run `make setup-master first` !"
189187
docker-compose up -d --no-recreate
190188

191189
stop: ##@Service Stop service
@@ -196,7 +194,7 @@ stop: ##@Service Stop service
196194

197195
restart: stop start ##@Service Restart service
198196

199-
setup-master: docker ##@Environment Setup dependency for master node
197+
setup-master: ##@Environment Setup dependency for master node
200198
cd scripts/master_node && bash setup.sh
201199

202200
setup-worker: ##@Environment Setup dependency for worker node

‎docs/setup_master.md

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ $ git clone http://gerrit.hyperledger.org/r/cello && cd cello
3131

3232
For the first time running, please setup the master node with the [setup.sh](https://github.com/hyperledger/cello/blob/master/scripts/master_node/setup.sh).
3333

34+
Just run (safe to repeat it):
35+
3436
```sh
3537
$ make setup-master
3638
```
@@ -48,7 +50,7 @@ $ make start
4850

4951
To stop or restart the whole services, run `make stop` or `make restart`.
5052

51-
#### Redploy a service
53+
#### Redeploy a service
5254
To redeploy one specific service, e.g., dashboard, please run
5355

5456
```sh
@@ -83,8 +85,8 @@ Please keep it safe by backups or using more high-available solutions.
8385

8486
In MacOS, Docker cannot mount local path from host by default. Hence for mongo container data volume, users need to:
8587

86-
* Make sure the `/opt/cello` path exists locally, and it is writable for the current user account. `make setup-master` should handle this already.
87-
* Add the path to sharing list in the preference of [Docker for MacOS](https://docs.docker.com/docker-for-mac/install/), to make it mountable by container.
88+
* Make sure the `/opt/cello` path exists locally, and it is writable for the current user account. Simply just run `make setup-master`.
89+
* Add the path to `File Sharing` list in the preference of [Docker for MacOS](https://docs.docker.com/docker-for-mac/install/), to make it mountable by container.
8890

8991
## Cello Baseimage
9092
![BaseImage](imgs/cello_baseimage.png)
@@ -94,7 +96,7 @@ required for supporting a Hyperledger Cello environment. The build process is ge
9496
inefficient to JIT assemble these components on demand. Hence bundled into baseimage and subsequently cached on
9597
the public repositories, so they can be simply consumed without requiring a local build cycle.
9698

97-
### Usuage
99+
### Usage
98100
* "make docker" will build the docker images and commit it to your local environment; e.g. "hyperledger/cello-baseimage".
99101
The docker image is also tagged with architecture and release details.
100102

‎scripts/master_node/download_images.sh

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,23 @@ fi
2525

2626
echo_b "Downloading the docker images for master node"
2727

28+
# TODO: will be removed after we have the user dashboard image
29+
echo_b "Check node:9.2 image."
30+
[ -z "$(docker images -q node:9.2 2> /dev/null)" ] && { echo "pulling node:9.2"; docker pull node:9.2; }
2831

32+
# docker image
2933
ARCH=$(uname -m)
34+
VERSION=latest
3035

31-
docker pull node:9.2 \
32-
&& docker pull hyperledger/cello-baseimage:${ARCH}-latest \
33-
&& docker pull hyperledger/cello-mongo:${ARCH}-latest \
34-
&& docker pull hyperledger/cello-nginx:${ARCH}-latest \
35-
&& docker tag hyperledger/cello-baseimage:${ARCH}-latest hyperledger/cello-baseimage \
36-
&& docker tag hyperledger/cello-mongo:${ARCH}-latest hyperledger/cello-mongo \
37-
&& docker tag hyperledger/cello-nginx:${ARCH}-latest hyperledger/cello-nginx
36+
for IMG in baseimage mongo nginx ; do
37+
HC_IMG=hyperledger/cello-${IMG}
38+
if [ -z "$(docker images -q ${HC_IMG} 2> /dev/null)" ]; then # not exist
39+
echo_b "Pulling ${HC_IMG}:${ARCH}-${VERSION} from dockerhub"
40+
docker pull ${HC_IMG}:${ARCH}-${VERSION}
41+
docker tag ${HC_IMG}:${ARCH}-${VERSION} ${HC_IMG}
42+
else
43+
echo_g "${HC_IMG} already exist locally"
44+
fi
45+
done
46+
47+
echo_g "All Image downloaded "

‎scripts/master_node/setup.sh

Lines changed: 65 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -40,70 +40,77 @@ get_distribution() {
4040
echo "${distribution//\"}"
4141
}
4242

43+
# Install necessary software including curl, python-pip, tox, docker
44+
install_software() {
45+
case $DISTRO in
46+
ubuntu)
47+
sudo apt-get update && sudo apt-get install -y curl python-pip tox;
48+
command -v docker >/dev/null 2>&1 || { echo_r >&2 "No docker-engine found, try installing"; curl -sSL https://get.docker.com/ | sh; sudo service docker restart; }
49+
command -v docker-compose >/dev/null 2>&1 || { echo_r >&2 "No docker-compose found, try installing"; sudo pip install 'docker-compose>=1.17.0'; }
50+
;;
51+
linuxmint)
52+
sudo apt-get install apt-transport-https ca-certificates -y
53+
sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
54+
sudo echo deb https://apt.dockerproject.org/repo ubuntu-xenial main >> /etc/apt/sources.list.d/docker.list
55+
sudo apt-get update
56+
sudo apt-get purge lxc-docker
57+
sudo apt-get install python-pip
58+
sudo apt-get install linux-image-extra-$(uname -r) -y
59+
sudo apt-get install docker-engine cgroup-lite apparmor -y
60+
sudo service docker start
61+
;;
62+
debian)
63+
sudo apt-get install apt-transport-https ca-certificates -y
64+
sudo sh -c "echo deb https://apt.dockerproject.org/repo debian-jessie main > /etc/apt/sources.list.d/docker.list"
65+
sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
66+
sudo apt-get update
67+
sudo apt-cache policy docker-engine
68+
sudo apt-get install docker-engine curl python-pip -y
69+
sudo service docker start
70+
;;
71+
centos)
72+
sudo yum install -y epel-release yum-utils
73+
sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
74+
sudo yum makecache fast
75+
sudo yum update && sudo yum install -y docker-ce python-pip
76+
sudo systemctl enable docker
77+
sudo systemctl start docker
78+
;;
79+
fedora)
80+
sudo dnf -y update
81+
sudo dnf -y install docker python-pip --allowerasing
82+
sudo systemctl enable docker
83+
sudo systemctl start docker
84+
;;
85+
opensuse)
86+
sudo zypper refresh
87+
sudo zypper install docker docker-compose python-pip
88+
sudo systemctl restart docker
89+
;;
90+
*)
91+
echo "Linux distribution not identified !!! skipping docker & pip installation"
92+
;;
93+
esac
94+
}
95+
4396
USER=`whoami`
4497
DISTRO=$(get_distribution)
4598
DB_DIR=/opt/${PROJECT}/mongo
4699

100+
echo_b "Check python-pip, tox, curl and docker-engine for $DISTRO"
101+
NEED_INSTALL="false"
102+
for software in pip tox curl docker docker-compose; do
103+
command -v ${software} >/dev/null 2>&1 || { NEED_INSTALL="true"; break; }
104+
done
105+
[ $NEED_INSTALL = "true" ] && install_software
47106

48-
echo_b "Install python-pip, tox, and docker-engine"
49-
case $DISTRO in
50-
ubuntu)
51-
sudo apt-get update && sudo apt-get install -y curl python-pip tox;
52-
echo_b "Checking to install Docker-engine..."
53-
command -v docker >/dev/null 2>&1 || { echo_r >&2 "No docker-engine found, try installing"; curl -sSL https://get.docker.com/ | sh; sudo service docker restart; }
54-
;;
55-
linuxmint)
56-
sudo apt-get install apt-transport-https ca-certificates -y
57-
sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
58-
sudo echo deb https://apt.dockerproject.org/repo ubuntu-xenial main >> /etc/apt/sources.list.d/docker.list
59-
sudo apt-get update
60-
sudo apt-get purge lxc-docker
61-
sudo apt-get install python-pip
62-
sudo apt-get install linux-image-extra-$(uname -r) -y
63-
sudo apt-get install docker-engine cgroup-lite apparmor -y
64-
sudo service docker start
65-
;;
66-
debian)
67-
sudo apt-get install apt-transport-https ca-certificates -y
68-
sudo sh -c "echo deb https://apt.dockerproject.org/repo debian-jessie main > /etc/apt/sources.list.d/docker.list"
69-
sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
70-
sudo apt-get update
71-
sudo apt-cache policy docker-engine
72-
sudo apt-get install docker-engine curl python-pip -y
73-
sudo service docker start
74-
;;
75-
centos)
76-
sudo yum install -y epel-release yum-utils
77-
sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
78-
sudo yum makecache fast
79-
sudo yum update && sudo yum install -y docker-ce python-pip
80-
sudo systemctl enable docker
81-
sudo systemctl start docker
82-
;;
83-
fedora)
84-
sudo dnf -y update
85-
sudo dnf -y install docker python-pip --allowerasing
86-
sudo systemctl enable docker
87-
sudo systemctl start docker
88-
;;
89-
opensuse)
90-
sudo zypper refresh
91-
sudo zypper install docker docker-compose python-pip
92-
sudo systemctl restart docker
93-
;;
94-
*)
95-
echo "Linux distribution not identified !!! skipping docker & pip installation"
96-
;;
97-
esac
98-
99-
echo_b "Add existing user to docker group"
107+
echo_b "Add existing user ${USER} to docker group"
100108
sudo usermod -aG docker ${USER}
101109

102-
echo_b "Checking to install Docker-compose..."
103-
command -v docker-compose >/dev/null 2>&1 || { echo_r >&2 "No docker-compose found, try installing"; sudo pip install 'docker-compose>=1.17.0'; }
104-
105-
[ `sudo docker ps -qa|wc -l` -gt 0 ] \
106-
&& echo_r "Warn: existing containers may cause unpredictable failure, suggest to clean them using docker rm"
110+
if [ `sudo docker ps -qa|wc -l` -gt 0 ]; then
111+
echo_r "Warn: existing containers may cause unpredictable failure, suggest to clean them (docker rm)"
112+
docker ps -a
113+
fi
107114

108115
echo_b "Download required Docker images..."
109116
bash ./download_images.sh
@@ -115,4 +122,4 @@ echo_b "Checking local mounted database path ${DB_DIR}..."
115122
&& sudo chown -R ${USER}:${USER} ${DB_DIR}
116123

117124
echo_g "Setup done, please logout and login again."
118-
echo_g "It's safe to run this script repeatedly. Just re-run if it fails."
125+
echo_g "It's safe to run this script repeatedly."

0 commit comments

Comments
 (0)