[CE-446] Support async task in flask server

Add celery worker for async task in backend.
Remove unused installation for operator dashboard.
Add rabbitmq service for celery task.

Change-Id: I485a18bfcc476e4e802ff9e66a92d8dd94c0e013
Signed-off-by: Haitao Yue <hightall@me.com>

.makerc/operator-dashboard

@@ -8,6 +8,9 @@ export NPM_REGISTRY?=https://registry.npmjs.org
 export DEV?=False
 # whether enable user active, if enable user must be active to use user dashboard
 export ENABLE_EMAIL_ACTIVE?=False
+export RABBITMQ_DEFAULT_USER?=cello
+export RABBITMQ_DEFAULT_PASS?=cello
+export RABBITMQ_DEFAULT_VHOST?=cello
 
 NPM_REGISTRY_REPLACE=$(subst $(SLASH),$(REPLACE_SLASH),$(NPM_REGISTRY))
 

Makefile

@@ -38,9 +38,9 @@ BASENAME ?= $(DOCKER_NS)/cello
 VERSION ?= 0.9.0
 IS_RELEASE=false
 
-DOCKER_BASE_x86_64=ubuntu:xenial
-DOCKER_BASE_ppc64le=ppc64le/ubuntu:xenial
-DOCKER_BASE_s390x=s390x/debian:jessie
+DOCKER_BASE_x86_64=python:3.6
+DOCKER_BASE_ppc64le=ppc64le/python:3.6
+DOCKER_BASE_s390x=s390x/python:3.6
 DOCKER_BASE=$(DOCKER_BASE_$(ARCH))
 BASE_VERSION ?= $(ARCH)-$(VERSION)
 

docker-compose-dev.yml

@@ -36,23 +36,35 @@ services:
   #    - PORT=8080
   #    - USERNAME=admin
   #    - PASSWORD=pass
+#  redis:
+#    image: redis
+  rabbitmq:
+    image: rabbitmq
+    environment:
+      - RABBITMQ_DEFAULT_USER=$RABBITMQ_DEFAULT_USER
+      - RABBITMQ_DEFAULT_PASS=$RABBITMQ_DEFAULT_PASS
+      - RABBITMQ_DEFAULT_VHOST=$RABBITMQ_DEFAULT_VHOST
 
   # cello dashboard service for network operator
   operator-dashboard:
     image: hyperledger/cello-operator-dashboard
     container_name: cello-operator-dashboard
     hostname: cello-operator-dashboard
     restart: unless-stopped
+    links:
+      - rabbitmq
     environment:
       - MONGO_URL=mongodb://cello-mongo:27017  # used by pymongo, deprecate soon
       - MONGO_HOST=mongo
       - MONGO_DB=dev
       - MONGODB_PORT=27017
-      - DEBUG=True    # in debug mode, service will auto-restart
+      - DEBUG=True   # in debug mode, service will auto-restart
       - LOG_LEVEL=$LOG_LEVEL  # what level log will be output
       - STATIC_FOLDER=$STATIC_FOLDER
       - TEMPLATE_FOLDER=$TEMPLATE_FOLDER
       - ENABLE_EMAIL_ACTIVE=$ENABLE_EMAIL_ACTIVE
+      - BROKER=amqp://$RABBITMQ_DEFAULT_USER:$RABBITMQ_DEFAULT_PASS@rabbitmq:5672/$RABBITMQ_DEFAULT_VHOST
+      - BACKEND=amqp://$RABBITMQ_DEFAULT_USER:$RABBITMQ_DEFAULT_PASS@rabbitmq:5672/$RABBITMQ_DEFAULT_VHOST
     ports:
       - "8080:8080"
     volumes:  # This should be removed in product env

docker-compose.yml

@@ -36,6 +36,12 @@ services:
   #    - PORT=8080
   #    - USERNAME=admin
   #    - PASSWORD=pass
+  rabbitmq:
+    image: rabbitmq
+    environment:
+    - RABBITMQ_DEFAULT_USER=$RABBITMQ_DEFAULT_USER
+    - RABBITMQ_DEFAULT_PASS=$RABBITMQ_DEFAULT_PASS
+    - RABBITMQ_DEFAULT_VHOST=$RABBITMQ_DEFAULT_VHOST
 
   # cello dashboard service for network operator
   operator-dashboard:
@@ -53,6 +59,8 @@ services:
       - STATIC_FOLDER=$STATIC_FOLDER
       - TEMPLATE_FOLDER=$TEMPLATE_FOLDER
       - ENABLE_EMAIL_ACTIVE=$ENABLE_EMAIL_ACTIVE
+      - BROKER=amqp://$RABBITMQ_DEFAULT_USER:$RABBITMQ_DEFAULT_PASS@rabbitmq:5672/$RABBITMQ_DEFAULT_VHOST
+      - BACKEND=amqp://$RABBITMQ_DEFAULT_USER:$RABBITMQ_DEFAULT_PASS@rabbitmq:5672/$RABBITMQ_DEFAULT_VHOST
     ports:
       - "8080:8080"
     volumes:

docker/baseimage/install.sh

@@ -8,88 +8,5 @@ set -x
 # Based thie file on https://github.com/docker-library/mongo/blob/master/3.4/Dockerfile &
 # https://docs.mongodb.com/manual/tutorial/install-mongodb-enterprise-on-ubuntu/#install-mongodb-enterprise
 
-# ----------------------------------------------------------------
-# Install mongo
-# ----------------------------------------------------------------
-
-groupadd -r mongodb && useradd -r -g mongodb mongodb
-
-apt-get update \
-&& apt-get install -y --no-install-recommends ca-certificates jq numactl sudo\
-&& rm -rf /var/lib/apt/lists/*
-
-# grab gosu for easy step-down from root
-export GOSU_VERSION=1.10
-
-set -x \
-	&& apt-get update && apt-get install -y --no-install-recommends wget && rm -rf /var/lib/apt/lists/* \
-	&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
-	&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
-	&& export GNUPGHOME="$(mktemp -d)" \
-	&& gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
-	&& gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
-	&& rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \
-	&& chmod +x /usr/local/bin/gosu \
-	&& gosu nobody true
-
-mkdir /docker-entrypoint-initdb.d
-
-# Add GPG Keys & update apt sources
-
-apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6
-
-echo "deb [ arch=amd64,arm64,ppc64el,s390x ] http://repo.mongodb.com/apt/ubuntu xenial/mongodb-enterprise/3.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-enterprise.list
-
-if [ $(dpkg --print-architecture) == "s390x" ]; then
-	echo "deb [ arch=s390x ] http://deb.debian.org/debian stable main" >> /etc/apt/sources.list
-fi
-
-apt-get update
-
-export MONGO_PACKAGE=mongodb-enterprise
-# export MONGO_REPO=repo.mongodb.com
-# export MONGO_PACKAGE=${MONGO_PACKAGE} MONGO_REPO=${MONGO_REPO}
-
-export MONGO_MAJOR=3.4
-export MONGO_VERSION=3.4.9
-
-apt-get install -y \
-		${MONGO_PACKAGE}=$MONGO_VERSION \
-		${MONGO_PACKAGE}-server=$MONGO_VERSION \
-		${MONGO_PACKAGE}-shell=$MONGO_VERSION \
-		${MONGO_PACKAGE}-mongos=$MONGO_VERSION \
-		${MONGO_PACKAGE}-tools=$MONGO_VERSION
-
-mkdir -p /data/db /data/configdb \
-&& chown -R mongodb:mongodb /data/db /data/configdb
-
-# ----------------------------------------------------------------
-# Install NodeJS
-# ----------------------------------------------------------------
-NODE_VER=8.9.0
-
-ARCH=`uname -m | sed 's|i686|x86|' | sed 's|x86_64|x64|'`
-NODE_PKG=node-v$NODE_VER-linux-$ARCH.tar.gz
-SRC_PATH=/tmp/$NODE_PKG
-
-# First remove any prior packages downloaded in case of failure
-cd /tmp
-rm -f node*.tar.gz
-wget --quiet https://nodejs.org/dist/v$NODE_VER/$NODE_PKG
-cd /usr/local && sudo tar --strip-components 1 -xzf $SRC_PATH
-rm -f /tmp/node*.tar.gz
-
-# ----------------------------------------------------------------
-# Install python3 and pip
-# ----------------------------------------------------------------
-if [[ $ARCH = 'ppc64le' ]];then
-apt-get install build-essential libssl-dev libffi-dev python3-dev libxslt-dev python3 -y
-else
-apt-get install python3 -y
-fi
-
-update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-cd /tmp
-wget https://bootstrap.pypa.io/get-pip.py
-python get-pip.py
-rm get-pip.py
+#set -x \
+#	&& apt-get update && apt-get install -y --no-install-recommends circus && rm -rf /var/lib/apt/lists/*

docker/operator-dashboard/Dockerfile.in

@@ -5,4 +5,4 @@ RUN	cd /app/ && \
 	pip install -r requirements.txt && \
 	rm -rf /tmp/cello
 
-CMD if [ "$DEBUG" = "True" ]; then python dashboard.py ; else gunicorn -w 1 --worker-class eventlet -b 0.0.0.0:8080 dashboard:app ;fi
+CMD bash /app/celery.sh && if [ "$DEBUG" = "True" ]; then python dashboard.py ; else gunicorn -w 1 --worker-class eventlet -b 0.0.0.0:8080 dashboard:app ;fi

src/celery.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#
+# Copyright 2009-2017 SAP SE or an SAP affiliate company.
+# All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+holdup -t 120 tcp://rabbitmq:5672
+
+export C_FORCE_ROOT=yes
+cd /app
+celery worker -A dashboard.celery --autoscale=20,3 -l info -f /var/log/celery.log -D

src/dashboard.py

@@ -20,6 +20,7 @@
     bp_login, bp_user_api, bp_user_view, front_rest_user_v2
 from modules.user import User
 from sockets.custom import CustomSockets
+from extensions import celery
 
 logger = logging.getLogger(__name__)
 logger.setLevel(LOG_LEVEL)
@@ -34,6 +35,8 @@
 app.config.from_object('config.DevelopmentConfig')
 app.config.from_envvar('CELLO_CONFIG_FILE', silent=True)
 
+celery.conf.update(app.config)
+
 connect(app.config.get("MONGODB_DB", "dashboard"),
         host=app.config.get("MONGODB_HOST", "mongo"),
         username=app.config.get("MONGODB_USERNAME", ""),

src/extensions.py

@@ -0,0 +1,13 @@
+# Copyright IBM Corp, All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+from celery import Celery
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+BROKER = os.environ.get("BROKER", "")
+BACKEND = os.environ.get("BACKEND", "")
+
+celery = Celery('cello-celery', broker=BROKER, backend=BACKEND)

src/requirements.txt

@@ -21,3 +21,6 @@ marshmallow>=2.13.6,<=2.14.0
 kubernetes==5.0.0
 flask-socketio>=2.9.0,<=2.9.6
 eventlet>=0.22,<=0.22.1
+celery>=4.2.0,<=4.2.1
+redis>=2.10.0,<=2.10.6
+holdup>1.5.0,<=1.6.0

src/resources/cluster_api.py

@@ -21,6 +21,7 @@
     FabricPreNetworkConfig, FabricV1NetworkConfig
 
 from modules import cluster_handler, host_handler
+from tasks import release_cluster
 
 logger = logging.getLogger(__name__)
 logger.setLevel(LOG_LEVEL)
@@ -140,10 +141,9 @@ def cluster_release(r):
     if not cluster_id:
         logger.warning("No cluster_id is given")
         return make_fail_resp("No cluster_id is given")
-    if cluster_handler.release_cluster(cluster_id):
-        return make_ok_resp()
+    release_cluster.delay(cluster_id)
 
-    return make_fail_resp("cluster release failed")
+    return make_ok_resp()
 
 
 @front_rest_v2.route('/cluster_op', methods=['GET', 'POST'])

src/tasks.py

@@ -0,0 +1,16 @@
+# Copyright IBM Corp, All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+from extensions import celery
+import logging
+from modules import cluster_handler
+
+logger = logging.getLogger(__name__)
+
+
+@celery.task()
+def release_cluster(cluster_id):
+    if cluster_handler.release_cluster(cluster_id):
+        logger.info("release cluster successfully")
+        return True