restful server zeppelin interface (#618)

* adapt ci * trigger ci * trigger ci * trigger ci
arctern-io · May 19, 2020 · 7561969 · 7561969
1 parent d85dfbe
commit 7561969
Show file tree

Hide file tree

Showing 40 changed files with 1,106 additions and 1,679 deletions.
diff --git a/ci/scripts/conda/cpu/arctern-webserver/build_arctern_webserver.sh b/ci/scripts/conda/cpu/arctern-webserver/build_arctern_webserver.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ "$BUILD_ARCTERN_WEBSERVER" == '1' ]; then
+    echo "Building arctern-webserver..."
+    conda build conda/recipes/arctern-webserver -c defaults -c conda-forge
+fi
diff --git a/ci/scripts/conda/cpu/build.sh b/ci/scripts/conda/cpu/build.sh
@@ -79,6 +79,9 @@ source ci/scripts/conda/cpu/arctern/build_arctern.sh
 logger "Build conda pkg for arctern-spark..."
 source ci/scripts/conda/cpu/arctern-spark/build_arctern-spark.sh
 
+logger "Build conda pkg for arctern-webserver..."
+source ci/scripts/conda/cpu/arctern-webserver/build_arctern_webserver.sh
+
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################

diff --git a/ci/scripts/conda/cpu/prebuild.sh b/ci/scripts/conda/cpu/prebuild.sh
@@ -3,6 +3,8 @@
 export BUILD_LIBARCTERN=1
 export BUILD_ARCTERN=1
 export BUILD_ARCTERN_SPARK=1
+export BUILD_ARCTERN_WEBSERVER=1
 export UPLOAD_LIBARCTERN=1
 export UPLOAD_ARCTERN=1
 export UPLOAD_ARCTERN_SPARK=1
+export UPLOAD_ARCTERN_WEBSERVER=1
diff --git a/ci/scripts/conda/cpu/upload_anaconda.sh b/ci/scripts/conda/cpu/upload_anaconda.sh
@@ -49,3 +49,15 @@ if [ "$UPLOAD_ARCTERN_SPARK" == "1" ]; then
     echo ${ARCTERN_SPARK_FILE}
     anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-arctern} ${LABEL_OPTION} --force ${ARCTERN_SPARK_FILE}
 fi
+
+
+if [ "$UPLOAD_ARCTERN_WEBSERVER" == "1" ]; then
+    export ARCTERN_WEBSERVER_FILE=`conda build conda/recipes/arctern-webserver -c conda-forge -c defaults --output`
+    LABEL_OPTION="--label main"
+    echo "LABEL_OPTION=${LABEL_OPTION}"
+
+    test -e ${ARCTERN_WEBSERVER_FILE}
+    echo "Upload arctern-webserver..."
+    echo ${ARCTERN_WEBSERVER_FILE}
+    anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-arctern} ${LABEL_OPTION} --force ${ARCTERN_WEBSERVER_FILE}
+fi
diff --git a/ci/scripts/conda/cpu/upload_package.sh b/ci/scripts/conda/cpu/upload_package.sh
@@ -7,7 +7,7 @@ if [ -z "$UPLOAD_PACKAGE_FILE_KEY" ]; then
     return 0
 fi
 
-if [[ "$UPLOAD_LIBARCTERN" == "1" || "$UPLOAD_ARCTERN" == "1" || "$UPLOAD_ARCTERN_SPARK" == "1" ]]; then
+if [[ "$UPLOAD_LIBARCTERN" == "1" || "$UPLOAD_ARCTERN" == "1" || "$UPLOAD_ARCTERN_SPARK" == "1" || "$UPLOAD_ARCTERN_WEBSERVER" == "1" ]]; then
     if [ -d ${CONDA_PREFIX}/conda-bld ];then
         tar -zcf ./conda-bld.tar.gz -C ${CONDA_PREFIX}/ conda-bld
         curl -u${JFROG_USENAME:-arctern}:${UPLOAD_PACKAGE_FILE_KEY} -T ./conda-bld.tar.gz ${ARTFACTORY_URL}/conda-bld.tar.gz

diff --git a/ci/scripts/conda/gpu/arctern-webserver/build_arctern_webserver.sh b/ci/scripts/conda/gpu/arctern-webserver/build_arctern_webserver.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ "$BUILD_ARCTERN_WEBSERVER" == '1' ]; then
+    echo "Building arctern-webserver..."
+    conda build conda/recipes/arctern-webserver -c defaults -c conda-forge
+fi
diff --git a/ci/scripts/conda/gpu/build.sh b/ci/scripts/conda/gpu/build.sh
@@ -76,6 +76,9 @@ source ci/scripts/conda/gpu/arctern/build_arctern.sh
 logger "Build conda pkg for arctern-spark..."
 source ci/scripts/conda/gpu/arctern-spark/build_arctern-spark.sh
 
+logger "Build conda pkg for arctern-webserver..."
+source ci/scripts/conda/cpu/arctern-webserver/build_arctern_webserver.sh
+
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################

diff --git a/ci/scripts/conda/gpu/prebuild.sh b/ci/scripts/conda/gpu/prebuild.sh
@@ -4,6 +4,8 @@ export CUDA_VERSION=${CUDA_VERSION:="10.0"}
 export BUILD_LIBARCTERN=1
 export BUILD_ARCTERN=1
 export BUILD_ARCTERN_SPARK=1
+export BUILD_ARCTERN_WEBSERVER=1
 export UPLOAD_LIBARCTERN=1
 export UPLOAD_ARCTERN=1
 export UPLOAD_ARCTERN_SPARK=1
+export UPLOAD_ARCTERN_WEBSERVER=1
diff --git a/ci/scripts/conda/gpu/upload_anaconda.sh b/ci/scripts/conda/gpu/upload_anaconda.sh
@@ -51,3 +51,14 @@ if [ "$UPLOAD_ARCTERN_SPARK" == "1" ]; then
     echo ${ARCTERN_SPARK_FILE}
     anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-arctern} ${LABEL_OPTION} --force ${ARCTERN_SPARK_FILE}
 fi
+
+if [ "$UPLOAD_ARCTERN_WEBSERVER" == "1" ]; then
+    export ARCTERN_WEBSERVER_FILE=`conda build conda/recipes/arctern-webserver -c conda-forge -c defaults --output`
+    LABEL_OPTION="--label main"
+    echo "LABEL_OPTION=${LABEL_OPTION}"
+
+    test -e ${ARCTERN_WEBSERVER_FILE}
+    echo "Upload arctern-webserver..."
+    echo ${ARCTERN_WEBSERVER_FILE}
+    anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-arctern} ${LABEL_OPTION} --force ${ARCTERN_WEBSERVER_FILE}
+fi
diff --git a/ci/scripts/conda/gpu/upload_package.sh b/ci/scripts/conda/gpu/upload_package.sh
@@ -7,7 +7,7 @@ if [ -z "$UPLOAD_PACKAGE_FILE_KEY" ]; then
     return 0
 fi
 
-if [[ "$UPLOAD_LIBARCTERN" == "1" || "$UPLOAD_ARCTERN" == "1" || "$UPLOAD_ARCTERN_SPARK" == "1" ]]; then
+if [[ "$UPLOAD_LIBARCTERN" == "1" || "$UPLOAD_ARCTERN" == "1" || "$UPLOAD_ARCTERN_SPARK" == "1" || "$UPLOAD_ARCTERN_WEBSERVER" == "1" ]]; then
     if [ -d ${CONDA_PREFIX}/conda-bld ];then
         tar -zcf ./conda-bld.tar.gz -C ${CONDA_PREFIX}/ conda-bld
         curl -u${JFROG_USENAME:-arctern}:${UPLOAD_PACKAGE_FILE_KEY} -T ./conda-bld.tar.gz ${ARTFACTORY_URL}/conda-bld.tar.gz

diff --git a/conda/recipes/arctern-webserver/build.sh b/conda/recipes/arctern-webserver/build.sh
@@ -0,0 +1,4 @@
+cd ./gui/server
+
+python setup.py build
+python setup.py install
diff --git a/conda/recipes/arctern-webserver/meta.yaml b/conda/recipes/arctern-webserver/meta.yaml
@@ -0,0 +1,47 @@
+# Usage:
+#   conda build . -c defaults -c conda-forge
+
+{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
+# {% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
+{% set git_revision_count=environ.get('GIT_DESCRIBE_NUMBER', 0) %}
+{% set git_commit_id=environ.get('GIT_COMMIT_ID', '') %}
+package:
+  name: arctern-webserver
+  version: {{ version }}
+
+source:
+  path: ../../..
+
+# ref: https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#build-section
+build:
+  number: {{ git_revision_count }}
+  string: {{ git_commit_id }}_{{ git_revision_count }}
+
+requirements:
+  build:
+    - python ===3.7.6
+    - setuptools
+    - flask ===1.1.1
+    - waitress ===1.4.3
+    - flask-httpauth ===3.3.0
+    - flask-cors ===3.0.8
+    - requests ===2.23.0
+
+  run:
+    - python ===3.7.6
+    - flask ===1.1.1
+    - waitress ===1.4.3
+    - flask-httpauth ===3.3.0
+    - flask-cors ===3.0.8
+    - requests ===2.23.0
+
+test:
+  commands:
+    - python -c "import arctern_server"
+
+about:
+  home: http://www.github.com/zilliztech/arctern
+  license: Apache-2.0
+  license_family: Apache
+  license_file: LICENSE
+  summary:  Web server for Arctern
diff --git a/docker/test_env/docker-compose.yaml b/docker/test_env/docker-compose.yaml
@@ -33,7 +33,7 @@ services:
       - ../../gui/server:/arctern/gui/server
     working_dir: /arctern/gui/server/arctern_server/tests/restful
     command: &flask-test
-      ["pytest", "--host", "flask", "--port", "8080", "--config", "/arctern/gui/server/arctern_server/db.json", "-p", "no:cacheprovider", "-s"]
+      ["pytest", "-p", "no:cacheprovider", "-s"]
     deploy:
       resources:
         limits:

diff --git a/gui/server/arctern_server/README.md b/gui/server/arctern_server/README.md
@@ -1,6 +1,6 @@
 # arctern-gui-server
 
-一个简易的RESTFUL接口的web服务器，采用python语言编写，主要是为了使用spark服务。
+一个简易的 RESTFUL 接口的web服务器，采用python语言编写，主要是为了使用spark服务。
 
 ## 代码结构
 

diff --git a/gui/server/arctern_server/app/account.py b/gui/server/arctern_server/app/account.py
diff --git a/gui/server/arctern_server/app/common/config.py b/gui/server/arctern_server/app/common/config.py
@@ -22,3 +22,7 @@ def optionxform(self, optionstr):
 INSTANCE = MyConf()
 INSTANCE.read(os.path.split(os.path.realpath(__file__))[0]
               + '/../../config.ini')
+
+ZEPPELEN_HOST = INSTANCE.get("zeppelin", "zeppelin-host", fallback="127.0.0.1")
+ZEPPELEN_PORT = INSTANCE.get("zeppelin", "zeppelin-port", fallback=8888)
+ZEPPELEN_PREFIX = "http://" + ZEPPELEN_HOST + ":" + ZEPPELEN_PORT
diff --git a/gui/server/arctern_server/app/common/db.py b/gui/server/arctern_server/app/common/db.py