From 2b35193467f6cb20225739198c4d5e9d4971e6d7 Mon Sep 17 00:00:00 2001 From: Haitao Yue Date: Thu, 30 Aug 2018 23:54:22 +0800 Subject: [PATCH] [CE-446] Support async task in flask server Add celery worker for async task in backend. Remove unused installation for operator dashboard. Add rabbitmq service for celery task. Change-Id: I485a18bfcc476e4e802ff9e66a92d8dd94c0e013 Signed-off-by: Haitao Yue --- .makerc/operator-dashboard | 3 + Makefile | 6 +- docker-compose-dev.yml | 14 +++- docker-compose.yml | 8 +++ docker/baseimage/install.sh | 87 +------------------------ docker/operator-dashboard/Dockerfile.in | 2 +- src/celery.sh | 12 ++++ src/dashboard.py | 3 + src/extensions.py | 13 ++++ src/requirements.txt | 3 + src/resources/cluster_api.py | 6 +- src/tasks.py | 16 +++++ 12 files changed, 80 insertions(+), 93 deletions(-) create mode 100644 src/celery.sh create mode 100644 src/extensions.py create mode 100644 src/tasks.py diff --git a/.makerc/operator-dashboard b/.makerc/operator-dashboard index d9deb2063..e544cce8b 100644 --- a/.makerc/operator-dashboard +++ b/.makerc/operator-dashboard @@ -8,6 +8,9 @@ export NPM_REGISTRY?=https://registry.npmjs.org export DEV?=False # whether enable user active, if enable user must be active to use user dashboard export ENABLE_EMAIL_ACTIVE?=False +export RABBITMQ_DEFAULT_USER?=cello +export RABBITMQ_DEFAULT_PASS?=cello +export RABBITMQ_DEFAULT_VHOST?=cello NPM_REGISTRY_REPLACE=$(subst $(SLASH),$(REPLACE_SLASH),$(NPM_REGISTRY)) diff --git a/Makefile b/Makefile index a8b0e125c..72ad01ec8 100755 --- a/Makefile +++ b/Makefile @@ -38,9 +38,9 @@ BASENAME ?= $(DOCKER_NS)/cello VERSION ?= 0.9.0 IS_RELEASE=false -DOCKER_BASE_x86_64=ubuntu:xenial -DOCKER_BASE_ppc64le=ppc64le/ubuntu:xenial -DOCKER_BASE_s390x=s390x/debian:jessie +DOCKER_BASE_x86_64=python:3.6 +DOCKER_BASE_ppc64le=ppc64le/python:3.6 +DOCKER_BASE_s390x=s390x/python:3.6 DOCKER_BASE=$(DOCKER_BASE_$(ARCH)) BASE_VERSION ?= $(ARCH)-$(VERSION) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 43b45a4e6..b4fde8629 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -36,6 +36,14 @@ services: # - PORT=8080 # - USERNAME=admin # - PASSWORD=pass +# redis: +# image: redis + rabbitmq: + image: rabbitmq + environment: + - RABBITMQ_DEFAULT_USER=$RABBITMQ_DEFAULT_USER + - RABBITMQ_DEFAULT_PASS=$RABBITMQ_DEFAULT_PASS + - RABBITMQ_DEFAULT_VHOST=$RABBITMQ_DEFAULT_VHOST # cello dashboard service for network operator operator-dashboard: @@ -43,16 +51,20 @@ services: container_name: cello-operator-dashboard hostname: cello-operator-dashboard restart: unless-stopped + links: + - rabbitmq environment: - MONGO_URL=mongodb://cello-mongo:27017 # used by pymongo, deprecate soon - MONGO_HOST=mongo - MONGO_DB=dev - MONGODB_PORT=27017 - - DEBUG=True # in debug mode, service will auto-restart + - DEBUG=True # in debug mode, service will auto-restart - LOG_LEVEL=$LOG_LEVEL # what level log will be output - STATIC_FOLDER=$STATIC_FOLDER - TEMPLATE_FOLDER=$TEMPLATE_FOLDER - ENABLE_EMAIL_ACTIVE=$ENABLE_EMAIL_ACTIVE + - BROKER=amqp://$RABBITMQ_DEFAULT_USER:$RABBITMQ_DEFAULT_PASS@rabbitmq:5672/$RABBITMQ_DEFAULT_VHOST + - BACKEND=amqp://$RABBITMQ_DEFAULT_USER:$RABBITMQ_DEFAULT_PASS@rabbitmq:5672/$RABBITMQ_DEFAULT_VHOST ports: - "8080:8080" volumes: # This should be removed in product env diff --git a/docker-compose.yml b/docker-compose.yml index c81254e28..fc5503cdd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,12 @@ services: # - PORT=8080 # - USERNAME=admin # - PASSWORD=pass + rabbitmq: + image: rabbitmq + environment: + - RABBITMQ_DEFAULT_USER=$RABBITMQ_DEFAULT_USER + - RABBITMQ_DEFAULT_PASS=$RABBITMQ_DEFAULT_PASS + - RABBITMQ_DEFAULT_VHOST=$RABBITMQ_DEFAULT_VHOST # cello dashboard service for network operator operator-dashboard: @@ -53,6 +59,8 @@ services: - STATIC_FOLDER=$STATIC_FOLDER - TEMPLATE_FOLDER=$TEMPLATE_FOLDER - ENABLE_EMAIL_ACTIVE=$ENABLE_EMAIL_ACTIVE + - BROKER=amqp://$RABBITMQ_DEFAULT_USER:$RABBITMQ_DEFAULT_PASS@rabbitmq:5672/$RABBITMQ_DEFAULT_VHOST + - BACKEND=amqp://$RABBITMQ_DEFAULT_USER:$RABBITMQ_DEFAULT_PASS@rabbitmq:5672/$RABBITMQ_DEFAULT_VHOST ports: - "8080:8080" volumes: diff --git a/docker/baseimage/install.sh b/docker/baseimage/install.sh index ccd75a106..78375a0ad 100644 --- a/docker/baseimage/install.sh +++ b/docker/baseimage/install.sh @@ -8,88 +8,5 @@ set -x # Based thie file on https://github.com/docker-library/mongo/blob/master/3.4/Dockerfile & # https://docs.mongodb.com/manual/tutorial/install-mongodb-enterprise-on-ubuntu/#install-mongodb-enterprise -# ---------------------------------------------------------------- -# Install mongo -# ---------------------------------------------------------------- - -groupadd -r mongodb && useradd -r -g mongodb mongodb - -apt-get update \ -&& apt-get install -y --no-install-recommends ca-certificates jq numactl sudo\ -&& rm -rf /var/lib/apt/lists/* - -# grab gosu for easy step-down from root -export GOSU_VERSION=1.10 - -set -x \ - && apt-get update && apt-get install -y --no-install-recommends wget && rm -rf /var/lib/apt/lists/* \ - && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ - && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \ - && export GNUPGHOME="$(mktemp -d)" \ - && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ - && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \ - && rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \ - && chmod +x /usr/local/bin/gosu \ - && gosu nobody true - -mkdir /docker-entrypoint-initdb.d - -# Add GPG Keys & update apt sources - -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6 - -echo "deb [ arch=amd64,arm64,ppc64el,s390x ] http://repo.mongodb.com/apt/ubuntu xenial/mongodb-enterprise/3.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-enterprise.list - -if [ $(dpkg --print-architecture) == "s390x" ]; then - echo "deb [ arch=s390x ] http://deb.debian.org/debian stable main" >> /etc/apt/sources.list -fi - -apt-get update - -export MONGO_PACKAGE=mongodb-enterprise -# export MONGO_REPO=repo.mongodb.com -# export MONGO_PACKAGE=${MONGO_PACKAGE} MONGO_REPO=${MONGO_REPO} - -export MONGO_MAJOR=3.4 -export MONGO_VERSION=3.4.9 - -apt-get install -y \ - ${MONGO_PACKAGE}=$MONGO_VERSION \ - ${MONGO_PACKAGE}-server=$MONGO_VERSION \ - ${MONGO_PACKAGE}-shell=$MONGO_VERSION \ - ${MONGO_PACKAGE}-mongos=$MONGO_VERSION \ - ${MONGO_PACKAGE}-tools=$MONGO_VERSION - -mkdir -p /data/db /data/configdb \ -&& chown -R mongodb:mongodb /data/db /data/configdb - -# ---------------------------------------------------------------- -# Install NodeJS -# ---------------------------------------------------------------- -NODE_VER=8.9.0 - -ARCH=`uname -m | sed 's|i686|x86|' | sed 's|x86_64|x64|'` -NODE_PKG=node-v$NODE_VER-linux-$ARCH.tar.gz -SRC_PATH=/tmp/$NODE_PKG - -# First remove any prior packages downloaded in case of failure -cd /tmp -rm -f node*.tar.gz -wget --quiet https://nodejs.org/dist/v$NODE_VER/$NODE_PKG -cd /usr/local && sudo tar --strip-components 1 -xzf $SRC_PATH -rm -f /tmp/node*.tar.gz - -# ---------------------------------------------------------------- -# Install python3 and pip -# ---------------------------------------------------------------- -if [[ $ARCH = 'ppc64le' ]];then -apt-get install build-essential libssl-dev libffi-dev python3-dev libxslt-dev python3 -y -else -apt-get install python3 -y -fi - -update-alternatives --install /usr/bin/python python /usr/bin/python3 10 -cd /tmp -wget https://bootstrap.pypa.io/get-pip.py -python get-pip.py -rm get-pip.py +#set -x \ +# && apt-get update && apt-get install -y --no-install-recommends circus && rm -rf /var/lib/apt/lists/* diff --git a/docker/operator-dashboard/Dockerfile.in b/docker/operator-dashboard/Dockerfile.in index 91bacc435..6a629b098 100644 --- a/docker/operator-dashboard/Dockerfile.in +++ b/docker/operator-dashboard/Dockerfile.in @@ -5,4 +5,4 @@ RUN cd /app/ && \ pip install -r requirements.txt && \ rm -rf /tmp/cello -CMD if [ "$DEBUG" = "True" ]; then python dashboard.py ; else gunicorn -w 1 --worker-class eventlet -b 0.0.0.0:8080 dashboard:app ;fi +CMD bash /app/celery.sh && if [ "$DEBUG" = "True" ]; then python dashboard.py ; else gunicorn -w 1 --worker-class eventlet -b 0.0.0.0:8080 dashboard:app ;fi diff --git a/src/celery.sh b/src/celery.sh new file mode 100644 index 000000000..c023f7d07 --- /dev/null +++ b/src/celery.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Copyright 2009-2017 SAP SE or an SAP affiliate company. +# All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +holdup -t 120 tcp://rabbitmq:5672 + +export C_FORCE_ROOT=yes +cd /app +celery worker -A dashboard.celery --autoscale=20,3 -l info -f /var/log/celery.log -D \ No newline at end of file diff --git a/src/dashboard.py b/src/dashboard.py index 6d7546202..606ce4f7d 100644 --- a/src/dashboard.py +++ b/src/dashboard.py @@ -20,6 +20,7 @@ bp_login, bp_user_api, bp_user_view, front_rest_user_v2 from modules.user import User from sockets.custom import CustomSockets +from extensions import celery logger = logging.getLogger(__name__) logger.setLevel(LOG_LEVEL) @@ -34,6 +35,8 @@ app.config.from_object('config.DevelopmentConfig') app.config.from_envvar('CELLO_CONFIG_FILE', silent=True) +celery.conf.update(app.config) + connect(app.config.get("MONGODB_DB", "dashboard"), host=app.config.get("MONGODB_HOST", "mongo"), username=app.config.get("MONGODB_USERNAME", ""), diff --git a/src/extensions.py b/src/extensions.py new file mode 100644 index 000000000..4b50e79af --- /dev/null +++ b/src/extensions.py @@ -0,0 +1,13 @@ +# Copyright IBM Corp, All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +from celery import Celery +import logging +import os + +logger = logging.getLogger(__name__) +BROKER = os.environ.get("BROKER", "") +BACKEND = os.environ.get("BACKEND", "") + +celery = Celery('cello-celery', broker=BROKER, backend=BACKEND) diff --git a/src/requirements.txt b/src/requirements.txt index 11c957bb5..aa1d33a89 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -21,3 +21,6 @@ marshmallow>=2.13.6,<=2.14.0 kubernetes==5.0.0 flask-socketio>=2.9.0,<=2.9.6 eventlet>=0.22,<=0.22.1 +celery>=4.2.0,<=4.2.1 +redis>=2.10.0,<=2.10.6 +holdup>1.5.0,<=1.6.0 diff --git a/src/resources/cluster_api.py b/src/resources/cluster_api.py index 10ccee147..96ba2524f 100644 --- a/src/resources/cluster_api.py +++ b/src/resources/cluster_api.py @@ -21,6 +21,7 @@ FabricPreNetworkConfig, FabricV1NetworkConfig from modules import cluster_handler, host_handler +from tasks import release_cluster logger = logging.getLogger(__name__) logger.setLevel(LOG_LEVEL) @@ -140,10 +141,9 @@ def cluster_release(r): if not cluster_id: logger.warning("No cluster_id is given") return make_fail_resp("No cluster_id is given") - if cluster_handler.release_cluster(cluster_id): - return make_ok_resp() + release_cluster.delay(cluster_id) - return make_fail_resp("cluster release failed") + return make_ok_resp() @front_rest_v2.route('/cluster_op', methods=['GET', 'POST']) diff --git a/src/tasks.py b/src/tasks.py new file mode 100644 index 000000000..459a1ab9d --- /dev/null +++ b/src/tasks.py @@ -0,0 +1,16 @@ +# Copyright IBM Corp, All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +from extensions import celery +import logging +from modules import cluster_handler + +logger = logging.getLogger(__name__) + + +@celery.task() +def release_cluster(cluster_id): + if cluster_handler.release_cluster(cluster_id): + logger.info("release cluster successfully") + return True