server package refactor

.gitignore

@@ -159,6 +159,6 @@ tests/draw_map/test*.png
 node_modules
 
 #/gui/server
-gui/server/log.txt
-gui/server/metastore_db/
-gui/server/spark-warehouse/
+gui/server/arctern_server/log.txt
+gui/server/arctern_server/metastore_db/
+gui/server/arctern_server/spark-warehouse/

docker/flask/runtime/Dockerfile

@@ -18,12 +18,13 @@ RUN cd /arctern/spark/pyspark && \
 
 COPY gui/server /arctern/gui/server
 
-RUN cd /arctern/gui/server && \
+RUN cd /arctern/gui/server/arctern_server && \
     python3.7 -m pip --no-cache-dir install -r requirements.txt
 
 EXPOSE 8080
 
-WORKDIR /arctern/gui/server
+WORKDIR /arctern/gui/server/arctern_server
 
 # use login shell when running the container
+ENV PYTHONPATH=/arctern/gui/server:$PYTHONPATH
 CMD ["python", "manage.py", "-r"]

docker/test_env/docker-compose.yaml

@@ -31,9 +31,9 @@ services:
       - PYTHONDONTWRITEBYTECODE=1
     volumes:
       - ../../gui/server:/arctern/gui/server
-    working_dir: /arctern/gui/server/tests/restful
+    working_dir: /arctern/gui/server/arctern_server/tests/restful
     command: &flask-test
-      ["pytest", "--host", "flask", "--port", "8080", "--config", "/arctern/gui/server/db.json", "-p", "no:cacheprovider", "-s"]
+      ["pytest", "--host", "flask", "--port", "8080", "--config", "/arctern/gui/server/arctern_server/db.json", "-p", "no:cacheprovider", "-s"]
     deploy:
       resources:
         limits:

docker/test_env/restful-pytest.dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.7-alpine
 
-COPY gui/server/tests/restful/requirements.txt /requirements.txt
+COPY gui/server/arctern_server/tests/restful/requirements.txt /requirements.txt
 
 RUN pip3 install --no-cache-dir \
     -r /requirements.txt

gui/server/MANIFEST.in

@@ -0,0 +1,3 @@
+include arctern_server/config.ini
+include arctern_server/data/0_5M_nyc_taxi_and_building.csv
+include arctern_server/data/account.db

gui/server/README.md → gui/server/arctern_server/README.md

@@ -39,6 +39,7 @@ python setup.py install
 运行一下命令可以启动web服务
 
 ```bash
+export PYTHONPATH=arctern/gui/server:$PYTHONPATH
 python manage.py -r
 ```
 
@@ -56,6 +57,7 @@ python manage.py -r
 如果期望服务器启动，自动加载数据
 
 ```bash
+export PYTHONPATH=arctern/gui/server:$PYTHONPATH
 python manage.py -r -c path/to/db.json
 ```
 
@@ -168,7 +170,7 @@ token: yes
         {
             "name": "old_nyc_taxi",
             "format": "csv",
-            "path": "/home/ljq/work/arctern/gui/server/data/0_5M_nyc_taxi_and_building.csv",
+            "path": "/home/ljq/work/arctern/gui/server/arctern_server/data/0_5M_nyc_taxi_and_building.csv",
             "options": {
                 "header": "True",
                 "delimiter": ","
@@ -215,7 +217,7 @@ token: yes
 举个例子：
 
 ```shell
-curl -X POST -H "Content-Type: application/json" -H "Authorization: Token YourToken" -d @./arctern/gui/server/db.json http://127.0.0.1:8080/load
+curl -X POST -H "Content-Type: application/json" -H "Authorization: Token YourToken" -d @./arctern/gui/server/arctern_server/db.json http://127.0.0.1:8080/load
 ```
 
 ### /dbs 获取数据库列表
@@ -645,14 +647,15 @@ curl --location --request POST 'http://localhost:8080/db/query' \
 启动server：
 
 ```shell
-cd arctern/gui/server
+cd arctern/gui/server/arctern_server
+export PYTHONPATH=arctern/gui/server:$PYTHONPATH
 python manage.py -r 
 ```
 
 运行restful api test：
 
 ```shell
-cd arctern/gui/server/tests/restful
+cd arctern/gui/server/arctern_server/tests/restful
 pytest --host=localhost --port=8080 --config=../../db.json
 ```
 

gui/server/app/codegen.py → gui/server/arctern_server/app/codegen.py

@@ -16,13 +16,13 @@
 def generate_session_code(session_name="spark"):
     uid = str(uuid.uuid1()).replace("-", "")
     app_name = "app_" + uid
-    from app.common import config as app_config
+    from arctern_server.app.common import config as app_config
     master_addr = app_config.INSTANCE.get("spark", "master-addr", fallback="local[*]")
     import socket
     localhost_ip = socket.gethostbyname(socket.gethostname())
 
-    session_code = 'from arctern.util.vega import vega_choroplethmap, vega_heatmap, vega_pointmap, vega_weighted_pointmap, vega_fishnetmap\n'
-    session_code += 'from arctern_pyspark import choroplethmap, heatmap, pointmap, weighted_pointmap, fishnetmap\n'
+    session_code = 'from arctern.util.vega import vega_choroplethmap, vega_heatmap, vega_pointmap, vega_weighted_pointmap, vega_icon, vega_fishnetmap\n'
+    session_code += 'from arctern_pyspark import choroplethmap, heatmap, pointmap, weighted_pointmap, icon_viz, fishnetmap\n'
     session_code += 'from arctern_pyspark import register_funcs\n'
     session_code += 'from pyspark.sql import SparkSession\n'
     session_code += '{} = SparkSession.builder'.format(session_name)
@@ -148,6 +148,17 @@ def generate_weighted_map_code(sql, params, session_name='spark'):
     )
     return sql_code, vega_code
 
+def generate_icon_viz_code(sql, params, session_name='spark'):
+    sql_code = generate_run_sql_code(sql, session_name)
+    vega_code = 'vega_icon({}, {}, {}, "{}", "{}")'.format(
+        int(params.get('width')),
+        int(params.get('height')),
+        params.get('bounding_box'),
+        params.get('icon_path'),
+        params.get('coordinate_system')
+    )
+    return sql_code, vega_code
+
 def generate_fishnetmap_code(sql, params, session_name='spark'):
     sql_code = generate_run_sql_code(sql, session_name)
     vega_code = 'vega_fishnetmap({}, {}, {}, {}, {}, {}, {}, "{}", "{}")'.format(

gui/server/app/common/spark.py → gui/server/arctern_server/app/common/spark.py

@@ -16,7 +16,7 @@
 
 from pyspark.sql import SparkSession
 
-from app.common import db
+from arctern_server.app.common import db
 from arctern_pyspark import register_funcs
 
 

gui/server/app/common/token.py → gui/server/arctern_server/app/common/token.py

@@ -18,8 +18,8 @@
 from itsdangerous import TimedJSONWebSignatureSerializer as Serializer
 from itsdangerous import BadSignature, SignatureExpired
 
-from app import account
-from app.common import utils
+from arctern_server.app import account
+from arctern_server.app.common import utils
 
 # -H "Authorization: Token <jws-token>"
 AUTH = HTTPTokenAuth(scheme='Token')

gui/server/app/scope.py → gui/server/arctern_server/app/scope.py

@@ -16,8 +16,8 @@
 
 from flask import Blueprint, jsonify, request
 
-from app.common import log
-from app import codegen
+from arctern_server.app.common import log
+from arctern_server.app import codegen
 
 API = Blueprint("scope_api", __name__)
 
@@ -39,7 +39,7 @@ def create_scope():
     else:
         scope = request.json.get("scope")
         if scope in _SCOPE:
-            return jsonify(status="error", code=-1, message="sorry, scope_id exists!")
+            return jsonify(status="error", code=-1, message="sorry, scope {} already exists!".format(scope))
     if scope is None:
         scope = str(uuid.uuid1()).replace("-", "")
     _SCOPE[scope] = dict()
@@ -97,9 +97,9 @@ def load_file():
         exec(load_code, _SCOPE[scope])
     return jsonify(status='success', code=200, message='load table successfully!')
 
-@API.route('/savetable', methods=['POST'])
+@API.route('/savefile', methods=['POST'])
 def save_table():
-    log.INSTANCE.info("POST /savetable: {}".format(request.json))
+    log.INSTANCE.info("POST /savefile: {}".format(request.json))
 
     scope = request.json.get('scope')
     log.INSTANCE.info("scope: {}".format(scope))
@@ -196,6 +196,7 @@ def render(payload, render_type):
         "heatmap": codegen.generate_heatmap_code,
         "choroplethmap": codegen.generate_choropleth_map_code,
         "weighted_pointmap": codegen.generate_weighted_map_code,
+        "icon_viz": codegen.generate_icon_viz_code,
         "fishnetmap": codegen.generate_fishnetmap_code,
     }
 
@@ -263,6 +264,15 @@ def weighted_pointmap():
         result=result,
     )
 
+@API.route('/icon_viz', methods=['POST'])
+def icon_viz():
+    status, code, result = render(request.json, 'icon_viz')
+    return jsonify(
+        status=status,
+        code=code,
+        result=result,
+    )
+
 @API.route('/fishnetmap', methods=['POST'])
 def fishnetmap():
     status, code, result = render(request.json, 'fishnetmap')

gui/server/app/service.py → gui/server/arctern_server/app/service.py

@@ -22,8 +22,8 @@
 from arctern.util.vega import vega_choroplethmap, vega_heatmap, vega_pointmap, vega_weighted_pointmap, vega_icon, vega_fishnetmap
 from arctern_pyspark import choroplethmap, heatmap, pointmap, weighted_pointmap, icon_viz, fishnetmap
 
-from app import account
-from app.common import spark, token, utils, db, log
+from arctern_server.app import account
+from arctern_server.app.common import spark, token, utils, db, log
 
 API = Blueprint('app_api', __name__)
 

gui/server/arctern-server → gui/server/arctern_server/arctern-server

@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 """
 Copyright (C) 2019-2020 Zilliz. All rights reserved.
 
@@ -18,7 +20,7 @@ import logging
 import getopt
 import sys
 
-import arctern_server
+from arctern_server import manage
 
 if __name__ == '__main__':
     IS_DEBUG = True
@@ -40,12 +42,12 @@ if __name__ == '__main__':
         OPTS, ARGS = getopt.getopt(sys.argv[1:], 'hri:p:c:', ['logfile=', 'loglevel='])
     except getopt.GetoptError as _e:
         print("Error '{}' occured. Arguments {}.".format(str(_e), _e.args))
-        usage()
+        manage.usage()
         sys.exit(2)
 
     for opt, arg in OPTS:
         if opt == '-h':
-            usage()
+            manage.usage()
             sys.exit()
         elif opt == '-r':
             IS_DEBUG = False
@@ -60,4 +62,4 @@ if __name__ == '__main__':
         elif opt == '--loglevel':
             LOG_LEVEL = _LEVEL_DICT_.get(arg, logging.DEBUG)
 
-    arctern_server.manage.main(IS_DEBUG, IP, PORT, JSON_CONFIG, LOG_FILE, LOG_LEVEL)
+    manage.main(IS_DEBUG, IP, PORT, JSON_CONFIG, LOG_FILE, LOG_LEVEL)

gui/server/config.ini → gui/server/arctern_server/config.ini

@@ -1,3 +1,2 @@
 [spark]
-# adapt ci
 master-addr = spark://spark-master:7077

gui/server/db.json → gui/server/arctern_server/db.json

@@ -15,7 +15,7 @@
         {
             "name": "old_nyc_taxi",
             "format": "csv",
-            "path": "/arctern/gui/server/data/0_5M_nyc_taxi_and_building.csv",
+            "path": "/arctern/gui/server/arctern_server/data/0_5M_nyc_taxi_and_building.csv",
             "options": {
                 "header": "True",
                 "delimiter": ","

gui/server/json/load_nyc_taxi.json → gui/server/arctern_server/json/load_nyc_taxi.json

@@ -4,7 +4,7 @@
         {
             "name": "old_nyc_taxi",
             "format": "csv",
-            "path": "/arctern/gui/server/data/0_5M_nyc_taxi_and_building.csv",
+            "path": "/arctern/gui/server/arctern_server/data/0_5M_nyc_taxi_and_building.csv",
             "options": {
                 "header": "True",
                 "delimiter": ","

gui/server/manage.py → gui/server/arctern_server/manage.py

@@ -25,9 +25,9 @@
 from flask import Flask, jsonify
 from flask_cors import CORS
 
-from app import service as app_service
-from app import scope as app_scope
-from app.common import log
+from arctern_server.app import service as app_service
+from arctern_server.app import scope as app_scope
+from arctern_server.app.common import log
 
 APP = Flask(__name__)
 

gui/server/restful-api.md → gui/server/arctern_server/restful-api.md

@@ -318,7 +318,7 @@ curl --location --request POST 'http://localhost:8080/loadfile' \
 #### Request
 
 - Method: **POST**
-- URL: `/savetable`
+- URL: `/savefile`
 - Headers:
     - `Content-Type: application/json`
 - Body:
@@ -358,9 +358,9 @@ curl --location --request POST 'http://localhost:8080/loadfile' \
 ```python
 import requests
 
-url = "http://localhost:8080/savetable"
+url = "http://localhost:8080/savefile"
 
-payload = "```json\n{\n    \"scope\": \"scope_name\",\n    \"session\": \"session_name\", \n    \"tables\": [\n        {\n            \"sql\": \"select * from table_name\",\n            \"format\": \"csv\",\n            \"path\": \"/path/to/data.csv\",\n            \"options\": {\n                \"header\": \"True\",\n                \"delimiter\": \",\"\n            }\n        }\n    ]\n}"
+payload = "{\n    \"scope\": \"scope_name\",\n    \"session\": \"session_name\", \n    \"tables\": [\n        {\n            \"sql\": \"select * from table_name\",\n            \"format\": \"csv\",\n            \"path\": \"/path/to/data.csv\",\n            \"options\": {\n                \"header\": \"True\",\n                \"delimiter\": \",\"\n            }\n        }\n    ]\n}"
 headers = {
   'Content-Type': 'application/json'
 }
@@ -371,9 +371,9 @@ print(response.text.encode('utf8'))
 ```
 
 ```shell
-curl --location --request POST 'http://localhost:8080/savetable' \
+curl --location --request POST 'http://localhost:8080/savefile' \
 --header 'Content-Type: application/json' \
---data-raw '```json
+--data-raw '
 {
     "scope": "scope_name",
     "session": "session_name",