Manager cofiguration rework

README.md

@@ -50,7 +50,7 @@ Before installing Hydrosphere Serving, please install its prerequisites:
 * [Docker 18.0+](https://docs.docker.com/install/);
 * [Docker Compose 1.23+](https://docs.docker.com/compose/install/#install-compose).
 
-To install Hydrosphere Serving, follow the instructions below:
+To deploy the Hydrosphere platform from master branch, follow the instructions below:
 
 1. Download the latest release version from the [releases](https://github.com/Hydrospheredata/hydro-serving/releases) page;
     ```sh 
@@ -67,6 +67,23 @@ To install Hydrosphere Serving, follow the instructions below:
     docker-compose up
     ```
 
+To deploy the Hydrosphere platform from the master branch, follow the instructions below:
+
+1. Clone umbrella repository locally.
+    ```sh
+    git clone ...
+    cd hydro-serving
+    ```
+1. Build the docker-compose.yaml.
+    ```sh
+    ./build-compose.sh
+    docker-compose up
+    ```
+1. Set up an environment.
+    ```sh
+    docker-compose up
+    ```
+
 To check installation, open http://localhost/. By default Hydrosphere UI is available at port 80.
 
 **Note**, other installation options are described in the [documentation](https://hydrosphere.io/serving-docs/latest/index.html).

build-compose.sh

@@ -0,0 +1,15 @@
+docker-compose \
+    -f docker-compose/docker-compose.service.template \
+    -f docker-compose/hydro-serving-manager.service.template \
+    -f docker-compose/hydro-serving-gateway.service.template \
+    -f docker-compose/hydro-serving-ui.service.template \
+    -f docker-compose/sonar.service.template \
+    -f docker-compose/hydro-auto-od.service.template \
+    -f docker-compose/hydro-root-cause.service.template \
+    -f docker-compose/hydro-stat.service.template \
+    -f docker-compose/hydro-visualization.service.template \
+    -f docker-compose/minio.service.template \
+    -f docker-compose/mongo.service.template \
+    -f docker-compose/postgresql.service.template \
+    -f docker-compose/prometheus-am.service.template \
+    config > docker-compose.yaml

docker-compose.yaml

@@ -11,31 +11,36 @@ services:
   auto-od:
     container_name: auto-od
     depends_on:
-    - minio
-    - mongodb
+      minio:
+        condition: service_started
+      mongodb:
+        condition: service_started
     environment:
       AWS_ACCESS_KEY_ID: minio
       AWS_SECRET_ACCESS_KEY: minio123
-      DEBUG: "True"
+      CLUSTER_ENDPOINT: http://managerui:8080
+      DEBUG: "False"
+      DEFAULT_RUNTIME: hydrosphere/serving-runtime-python-3.6
+      DEFAULT_TIMEOUT: '120'
       GRPC_PORT: '5001'
-      HS_CLUSTER_ADDRESS: http://managerui:8080
       MONGO_URL: mongodb
       S3_ENDPOINT: http://minio:9000
-    image: hydrosphere/auto-od:7cec5d2ebde759e1001b0e8d9a3578e13572e96e
+    image: hydrosphere/auto-od
     networks:
       extnet: {}
       hydronet: {}
   gateway:
     container_name: gateway
     depends_on:
-    - manager
+      manager:
+        condition: service_started
     environment:
       API_GATEWAY_GRPC_PORT: '9090'
       API_GATEWAY_HOST: managerui
       API_GATEWAY_HTTP_PORT: '8080'
       GRPC_PORT: '9091'
       HTTP_PORT: '9090'
-    image: hydrosphere/serving-gateway:9703a1d87126ca95552d9c3dbdd3b7a5ac1a3013
+    image: hydrosphere/serving-gateway
     networks:
       hydronet: {}
     ports:
@@ -46,11 +51,11 @@ services:
   manager:
     container_name: manager
     depends_on:
-    - postgres
+      postgres:
+        condition: service_started
     environment:
-      DATABASE_HOST: postgres
-      NETWORK_NAME: demo_hydronet
-    image: hydrosphere/serving-manager:4f524ed07b803ed4d733462119b488d5a4e8512c
+      CUSTOM_CONFIG: /app/config/dc.conf
+    image: hydrosphere/serving-manager
     networks:
       extnet: {}
       hydronet: {}
@@ -60,15 +65,21 @@ services:
     - published: 19091
       target: 9091
     volumes:
+    - ./docker-compose/dc.conf:/app/config/dc.conf:ro
     - /var/run/docker.sock:/var/run/docker.sock:rw
   managerui:
     container_name: managerui
     depends_on:
-    - gateway
-    - manager
-    - rootcause
-    - serving-sonar
-    - visualization
+      gateway:
+        condition: service_started
+      manager:
+        condition: service_started
+      rootcause:
+        condition: service_started
+      serving-sonar:
+        condition: service_started
+      visualization:
+        condition: service_started
     environment:
       AUTO_OD_GRPC_PORT: '5001'
       AUTO_OD_HOST: auto-od
@@ -81,14 +92,13 @@ services:
       MONITORING_GRPC_PORT: '9091'
       MONITORING_HOST: serving-sonar
       MONITORING_HTTP_PORT: '9090'
-      OSS: ''
       PROMETHEUS_AM_HOST: prometheus-am
       ROOTCAUSE_HOST: rootcause
       ROOTCAUSE_HTTP_PORT: '5000'
       STAT_HOST: stat
       STAT_PORT: '5000'
       VISUALIZATION_HOST: visualization
-    image: hydrosphere/hydro-serving-ui:f5836e0968c6e4b7f9c5a943f0073f34e11f8148
+    image: hydrosphere/hydro-serving-ui
     networks:
       extnet: {}
       hydronet: {}
@@ -153,21 +163,24 @@ services:
     command: python app.py
     container_name: rootcause
     depends_on:
-    - mongodb
-    - rootcause-worker
-    - serving-sonar
+      mongodb:
+        condition: service_started
+      rootcause-worker:
+        condition: service_started
+      serving-sonar:
+        condition: service_started
     environment:
       AWS_ACCESS_KEY_ID: minio
       AWS_SECRET_ACCESS_KEY: minio123
-      DEBUG: "True"
+      DEBUG: "False"
       GRPC_UI_ADDRESS: managerui
       HTTP_UI_ADDRESS: http://managerui:8080
       MONGO_AUTH_DB: admin
       MONGO_PORT: '27017'
       MONGO_URL: mongodb
       ROOTCAUSE_DB_NAME: rootcause
       S3_ENDPOINT: http://minio:9000
-    image: hydrosphere/hydro-root-cause:256d6e3ef012d45be79e7287dde54ca2764d9cfd
+    image: hydrosphere/hydro-root-cause
     networks:
       extnet: {}
       hydronet: {}
@@ -178,26 +191,29 @@ services:
     command: celery -A app.celery worker -l info -O fair -Q rootcause
     container_name: rootcause-worker
     depends_on:
-    - mongodb
-    - serving-sonar
+      mongodb:
+        condition: service_started
+      serving-sonar:
+        condition: service_started
     environment:
       AWS_ACCESS_KEY_ID: minio
       AWS_SECRET_ACCESS_KEY: minio123
-      DEBUG: "True"
+      DEBUG: "False"
       GRPC_UI_ADDRESS: managerui:9090
       HTTP_UI_ADDRESS: http://managerui:8080
       MONGO_AUTH_DB: admin
       MONGO_PORT: '27017'
       MONGO_URL: mongodb
       ROOTCAUSE_DB_NAME: rootcause
       S3_ENDPOINT: http://minio:9000
-    image: hydrosphere/hydro-root-cause:256d6e3ef012d45be79e7287dde54ca2764d9cfd
+    image: hydrosphere/hydro-root-cause
     networks:
       hydronet: {}
   serving-sonar:
     container_name: serving-sonar
     depends_on:
-    - postgres
+      postgres:
+        condition: service_started
     environment:
       ALERTING_FRONTEND_URL: http://localhost/
       ALERTING_MANAGER_URL: prometheus-am:9093
@@ -216,7 +232,7 @@ services:
       STORAGE_PATH_STYLE_ACCESS: "true"
       STORAGE_S3_IMPL: org.apache.hadoop.fs.s3a.S3AFileSystem
       STORAGE_SECRET_KEY: minio123
-    image: hydrosphere/sonar:79b47921b8f8eb37c9935101054fb10b871ccc40
+    image: hydrosphere/sonar
     networks:
       extnet: {}
       hydronet: {}
@@ -228,15 +244,17 @@ services:
   stat:
     container_name: stat
     depends_on:
-    - minio
-    - mongodb
+      minio:
+        condition: service_started
+      mongodb:
+        condition: service_started
     environment:
       AWS_ACCESS_KEY_ID: minio
       AWS_SECRET_ACCESS_KEY: minio123
       HTTP_UI_ADDRESS: http://managerui:8080
       MONGO_URL: mongodb
       S3_ENDPOINT: http://minio:9000
-    image: hydrosphere/stat:343d76a3b01b73b5170d5ae602b0066fe0eeb211
+    image: hydrosphere/stat
     networks:
       extnet: {}
       hydronet: {}
@@ -247,19 +265,21 @@ services:
     command: python app.py
     container_name: visualization
     depends_on:
-    - minio
-    - mongodb
+      minio:
+        condition: service_started
+      mongodb:
+        condition: service_started
     environment:
       AWS_ACCESS_KEY_ID: minio
       AWS_SECRET_ACCESS_KEY: minio123
       AWS_STORAGE_ENDPOINT: http://minio:9000
-      DEBUG: "True"
+      DEBUG: "False"
       GRPC_PORT: '5003'
       GRPC_PROXY_ADDRESS: managerui:9090
       HTTP_PROXY_ADDRESS: http://managerui:8080
       MONGO_URL: mongodb
       SECURE: "False"
-    image: hydrosphere/hydro-visualization:7dc486f6d226b2ab326b978f934ca7c12cdf0bf8
+    image: hydrosphere/hydro-visualization
     networks:
       extnet: {}
       hydronet: {}
@@ -270,19 +290,22 @@ services:
     command: celery -A app.celery worker -l info -Q visualization
     container_name: visualization-worker
     depends_on:
-    - minio
-    - mongodb
-    - visualization
+      minio:
+        condition: service_started
+      mongodb:
+        condition: service_started
+      visualization:
+        condition: service_started
     environment:
       AWS_ACCESS_KEY_ID: minio
       AWS_SECRET_ACCESS_KEY: minio123
       AWS_STORAGE_ENDPOINT: http://minio:9000
-      DEBUG: "True"
+      DEBUG: "False"
       GRPC_PROXY_ADDRESS: managerui:9090
       HTTP_PROXY_ADDRESS: http://managerui:8080
       MONGO_URL: mongodb
       SECURE: "False"
-    image: hydrosphere/hydro-visualization:7dc486f6d226b2ab326b978f934ca7c12cdf0bf8
+    image: hydrosphere/hydro-visualization
     networks:
       hydronet: {}
 version: '3.5'

docker-compose/dc.conf

@@ -0,0 +1,14 @@
+include "application.conf"
+
+database {
+  jdbc-url = "jdbc:postgresql://postgres:5432/docker"
+  username = "docker"
+  password = "docker"
+}
+
+docker-repository.type = "local"
+
+cloud-driver {
+  type = "docker"
+  network-name = "demo_hydronet"
+}