docker-compose.yml

# Copyright 2021 The Dataomnis Authors. All rights reserved.
# Use of this source code is governed by a Apache license
# that can be found in the LICENSE file.

version: '3'

networks:
  dataomnis-net:

services:
  dataomnis-db:
    container_name: "dataomnis-db"
    image: "mysql:8.0.11"
    command: --default-authentication-plugin=mysql_native_password
    environment:
      - MYSQL_ROOT_PASSWORD=${MYSQL_PASSWORD}
#      - MYSQL_USER=${MYSQL_USER} # there is no need to set root that created by default with the password specified by MYSQL_PASSWORD
#      - MYSQL_PASSWORD=${MYSQL_PASSWORD}
    volumes:
      - ${LOCAL_VOLUME_PATH}/mysql:/var/lib/mysql
    ports:
      - "3306:${MYSQL_PORT}" # for test & debug
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - dataomnis-db

  dataomnis-db-init:
    container_name: "dataomnis-db-init"
    image: ${IMAGE_REPO}/flyway:${IMAGE_TAG}
    entrypoint: "sh"
    command: -c "/flyway/ddl/ddl_init.sh --connect-timeout=5"
    environment:
      - MYSQL_HOST=${MYSQL_HOST}
      - MYSQL_PORT=${MYSQL_PORT}
      - MYSQL_PASSWORD=${MYSQL_PASSWORD}
      - MYSQL_USER=${MYSQL_USER}
    links:
      - dataomnis-db:dataomnis-db
    depends_on:
      - dataomnis-db
    networks:
      dataomnis-net:
        aliases:
          - dataomnis-db-init

  dataomnis-db-ctrl:
    image: ${IMAGE_REPO}/flyway:${IMAGE_TAG}
    container_name: "dataomnis-db-ctrl"
    command: -url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DATABASE} -user=${MYSQL_USER} -password=${MYSQL_PASSWORD} -validateOnMigrate=false migrate
    links:
      - dataomnis-db:dataomnis-db
    depends_on:
      - dataomnis-db-init
    networks:
      dataomnis-net:
        aliases:
          - dataomnis-ctrl

  dataomnis-etcd:
    image: "quay.io/coreos/etcd:latest"
    container_name: "dataomnis-etcd"
    volumes:
      - ${LOCAL_VOLUME_PATH}/etcd:/etcd-data
    command: "/usr/local/bin/etcd --data-dir=/etcd-data --name dataomnis-etcd --initial-advertise-peer-urls http://0.0.0.0:2380 --listen-peer-urls http://0.0.0.0:2380 --advertise-client-urls http://0.0.0.0:2379 --listen-client-urls http://0.0.0.0:2379 --initial-cluster dataomnis-etcd=http://0.0.0.0:2380"
    ports:
      - "2379:${ETCD_SERVER_PORT}" # for test & debug
      - "2380:${ETCD_PEER_PORT}" # for test & debug
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - dataomnis-etcd

  dataomnis-jaeger:
    image: "jaegertracing/all-in-one:1.22"
    container_name: "dataomnis-jaeger"
    ports:
      - "5775:5775/udp" # for test & debug
      - "6831:6831/udp" # for test & debug
      - "6832:6832/udp"
      - "5778:5778"
      - "16686:16686"
      - "14268:14268"
      - "14250:14250"
      - "9411:9411"
    environment:
      - COLLECTOR_ZIPKIN_HTTP_PORT=9411
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - dataomnis-jaeger

  apiglobal:
    image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
    container_name: "dataomnis-apiglobal"
    command: "apiglobal start -c ${DATAOMNIS_CONF}/apiglobal.yaml"
    ports:
      - "${API_GLOBAL_PORT}:${API_GLOBAL_PORT}"
    links:
      - apiserver
      - account
    depends_on:
      - apiserver
      - dataomnis-jaeger
      - account
    environment:
      - API_GLOBAL_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
      - API_GLOBAL_READ_TIMEOUT=${API_SERVER_READ_TIMEOUT}
      - API_GLOBAL_WRITE_TIMEOUT=${API_SERVER_WRITE_TIMEOUT}
      - API_GLOBAL_DLE_TIMEOUT=${API_SERVER_DLE_TIMEOUT}
      - API_GLOBAL_EXIT_TIMEOUT=${API_SERVER_EXIT_TIMEOUT}
      - API_GLOBAL_HTTP_SERVER_ADDRESS=0.0.0.0:${API_GLOBAL_PORT}
      - API_GLOBAL_TRACER_SERVICE_NAME=apiglobal
      - API_GLOBAL_TRACER_LOCAL_AGENT=dataomnis-jaeger:${TRACER_PORT}
      - API_GLOBAL_REGIONS=staging:http://apiserver:9001|zh_cn:开发测试区,en_us:staging mock:http://apiserver:9001|zh_cn:模拟测试用,en_us:mock
      - API_GLOBAL_ACCOUNT_SERVER_ADDRESS=account:${ACCOUNT_PORT}
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - apiglobal

  apiserver:
    image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
    container_name: "dataomnis-apiserver"
    command: "apiserver start -c ${DATAOMNIS_CONF}/apiserver.yaml"
    ports:
      - "${API_SERVER_PORT}:${API_SERVER_PORT}"
    links:
      - spacemanager
      - scheduler
      - jobmanager
      - resourcemanager
      - account
      - enginemanager
    depends_on:
      - spacemanager
      - scheduler
      - jobmanager
      - dataomnis-jaeger
      - account
      - enginemanager
    environment:
      - API_SERVER_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
      - API_SERVER_READ_TIMEOUT=${API_SERVER_READ_TIMEOUT}
      - API_SERVER_WRITE_TIMEOUT=${API_SERVER_WRITE_TIMEOUT}
      - API_SERVER_DLE_TIMEOUT=${API_SERVER_DLE_TIMEOUT}
      - API_SERVER_EXIT_TIMEOUT=${API_SERVER_EXIT_TIMEOUT}
      - API_SERVER_HTTP_SERVER_ADDRESS=0.0.0.0:${API_SERVER_PORT}
      - API_SERVER_TRACER_SERVICE_NAME=apiserver
      - API_SERVER_TRACER_LOCAL_AGENT=dataomnis-jaeger:${TRACER_PORT}
      - API_SERVER_SPACE_MANAGER_ADDRESS=spacemanager:${SPACE_MANAGER_PORT}
      - API_SERVER_SPACE_MANAGER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - API_SERVER_SPACE_MANAGER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - API_SERVER_SCHEDULER_ADDRESS=scheduler:${SCHEDULER_PORT}
      - API_SERVER_SCHEDULER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - API_SERVER_SCHEDULER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - API_SERVER_JOB_MANAGER_ADDRESS=jobmanager:${JOB_MANAGER_PORT}
      - API_SERVER_JOB_MANAGER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - API_SERVER_JOB_MANAGER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - API_SERVER_OBSERVER_MANAGER_ADDRESS=observer:{OBSERVER_PORT}
      - API_SERVER_OBSERVER_MANAGER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - API_SERVER_OBSERVER_MANAGER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - API_SERVER_RESOURCE_MANAGER_ADDRESS=resourcemanager:${RESOURCE_MANAGER_PORT}
      - API_SERVER_RESOURCE_MANAGER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - API_SERVER_RESOURCE_MANAGER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - API_SERVER_ACCOUNT_SERVER_ADDRESS=account:${ACCOUNT_PORT}
      - API_SERVER_ACCOUNT_SERVER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - API_SERVER_ACCOUNT_SERVER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - API_SERVER_ENGINE_MANAGER_ADDRESS=enginemanager:${ENGINE_MANAGER_PORT}
      - API_SERVER_ENGINE_MANAGER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - API_SERVER_ENGINE_MANAGER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - apiserver

  spacemanager:
    image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
    container_name: "dataomnis-spacemanager"
    command: "spacemanager start -c ${DATAOMNIS_CONF}/spacemanager.yaml"
    ports:
      - "${SPACE_MANAGER_PORT}:${SPACE_MANAGER_PORT}"
    links:
      - dataomnis-db:dataomnis-db
    depends_on:
      - dataomnis-db-ctrl
      - dataomnis-jaeger
    environment:
      - SPACE_MANAGER_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
      - SPACE_MANAGER_GRPC_SERVER_ADDRESS=0.0.0.0:${SPACE_MANAGER_PORT}
      - SPACE_MANAGER_GRPC_SERVER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - SPACE_MANAGER_GRPC_SERVER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - SPACE_MANAGER_METRICS_ENABLED=${METRICS_ENABLED}
      - SPACE_MANAGER_METRICS_SERVER_ADDRESS=0.0.0.0:${SPACE_MANAGER_METRICS_PORT}
      - SPACE_MANAGER_MYSQL_HOSTS=dataomnis-db:${MYSQL_PORT}
      - SPACE_MANAGER_MYSQL_DATABASE=${MYSQL_DATABASE}
      - SPACE_MANAGER_MYSQL_PASSWORD=${MYSQL_PASSWORD}
      - SPACE_MANAGER_MYSQL_LOG_LEVEL=${MYSQL_LOG_LEVEL}
      - SPACE_MANAGER_TRACER_SERVICE_NAME=spacemanager
      - SPACE_MANAGER_TRACER_LOCAL_AGENT=dataomnis-jaeger:${TRACER_PORT}
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - spacemanager

  scheduler:
    image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
    container_name: "dataomnis-scheduler"
    command: "scheduler start -c ${DATAOMNIS_CONF}/scheduler.yaml"
    ports:
      - "${SCHEDULER_PORT}:${SCHEDULER_PORT}"
    links:
      - dataomnis-db:dataomnis-db
      - dataomnis-etcd:dataomnis-etcd
    depends_on:
      - dataomnis-db-ctrl
      - dataomnis-etcd
      - dataomnis-jaeger
    environment:
      - SCHEDULER_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
      - SCHEDULER_GRPC_SERVER_ADDRESS=0.0.0.0:${SCHEDULER_PORT}
      - SCHEDULER_GRPC_SERVER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - SCHEDULER_GRPC_SERVER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - SCHEDULER_METRICS_ENABLED=${METRICS_ENABLED}
      - SCHEDULER_METRICS_SERVER_ADDRESS=0.0.0.0:${SCHEDULER_METRICS_PORT}
      - SCHEDULER_MYSQL_HOSTS=dataomnis-db:${MYSQL_PORT}
      - SCHEDULER_MYSQL_DATABASE=${MYSQL_DATABASE}
      - SCHEDULER_MYSQL_PASSWORD=${MYSQL_PASSWORD}
      - SCHEDULER_MYSQL_LOG_LEVEL=${MYSQL_LOG_LEVEL}
      - SCHEDULER_ETCD_ENDPOINTS=dataomnis-etcd:${ETCD_SERVER_PORT}
      - SCHEDULER_ETCD_DIAL_TIMEOUT=5s
      - SCHEDULER_TRACER_SERVICE_NAME=scheduler
      - SCHEDULER_TRACER_LOCAL_AGENT=dataomnis-jaeger:${TRACER_PORT}
      - SCHEDULER_JOB_MANAGER_ADDRESS=jobmanager:${JOB_MANAGER_PORT}
      - SCHEDULER_JOB_MANAGER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - SCHEDULER_JOB_MANAGER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - scheduler

  jobmanager:
    image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
    container_name: "dataomnis-jobmanager"
    command: "jobmanager start -c ${DATAOMNIS_CONF}/jobmanager.yaml"
    ports:
      - "${JOB_MANAGER_PORT}:${JOB_MANAGER_PORT}"
    links:
      - dataomnis-db:dataomnis-db
    depends_on:
      - dataomnis-db-ctrl
      - dataomnis-jaeger
    environment:
      - JOB_MANAGER_ZEPPELIN_ADDRESS=zeppelin:${ZEPPELIN_NODEPORT}
      - JOB_MANAGER_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
      - JOB_MANAGER_GRPC_SERVER_ADDRESS=0.0.0.0:${JOB_MANAGER_PORT}
      - JOB_MANAGER_GRPC_SERVER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - JOB_MANAGER_GRPC_SERVER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - JOB_MANAGER_METRICS_ENABLED=${METRICS_ENABLED}
      - JOB_MANAGER_METRICS_SERVER_ADDRESS=0.0.0.0:${JOB_MANAGER_METRICS_PORT}
      - JOB_MANAGER_MYSQL_HOSTS=${MYSQL_HOST}:${MYSQL_PORT}
      - JOB_MANAGER_MYSQL_DATABASE=${MYSQL_DATABASE}
      - JOB_MANAGER_MYSQL_USERS=${MYSQL_USER}
      - JOB_MANAGER_MYSQL_PASSWORD=${MYSQL_PASSWORD}
      - JOB_MANAGER_MYSQL_LOG_LEVEL=${MYSQL_LOG_LEVEL}
      - JOB_MANAGER_TRACER_SERVICE_NAME=jobmanager
      - JOB_MANAGER_TRACER_LOCAL_AGENT=dataomnis-jaeger:${TRACER_PORT}
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - jobmanager

  resourcemanager:
    image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
    container_name: "dataomnis-resourcemanager"
    command: "resourcemanager start -c ${DATAOMNIS_CONF}/resourcemanager.yaml"
    ports:
      - "${RESOURCE_MANAGER_PORT}:${RESOURCE_MANAGER_PORT}"
    links:
      - dataomnis-db:dataomnis-db
    depends_on:
      - dataomnis-db-ctrl
    environment:
      - RESOURCE_MANAGER_HDFS_SERVER=hadoop:9000
      - RESOURCE_MANAGER_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
      - RESOURCE_MANAGER_GRPC_SERVER_ADDRESS=0.0.0.0:${RESOURCE_MANAGER_PORT}
      - RESOURCE_MANAGER_GRPC_SERVER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - RESOURCE_MANAGER_GRPC_SERVER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - RESOURCE_MANAGER_METRICS_ENABLED=${METRICS_ENABLED}
      - RESOURCE_MANAGER_METRICS_SERVER_ADDRESS=0.0.0.0:${RESOURCE_MANAGER_METRICS_PORT}
      - RESOURCE_MANAGER_MYSQL_HOSTS=dataomnis-db:${MYSQL_PORT}
      - RESOURCE_MANAGER_MYSQL_DATABASE=${MYSQL_DATABASE}
      - RESOURCE_MANAGER_MYSQL_USERS=${MYSQL_USER}
      - RESOURCE_MANAGER_MYSQL_PASSWORD=${MYSQL_PASSWORD}
      - RESOURCE_MANAGER_MYSQL_LOG_LEVEL=${MYSQL_LOG_LEVEL}
      - RESOURCE_MANAGER_TRACER_SERVICE_NAME=resourcemanager
      - RESOURCE_MANAGER_TRACER_LOCAL_AGENT=dataomnis-jaeger:${TRACER_PORT}
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - resourcemanager

  zeppelin:
    image: ${IMAGE_REPO}/zeppelin:${ZEPPELIN_IMAGE_TAG}
    container_name: "dataomnis-zeppelin"
    links:
      - flinkjobmanager:flinkjobmanager
    ports:
      - "${ZEPPELIN_NODEPORT}:8080"
    depends_on:
      - flinkjobmanager
      - flinktaskmanager
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - zeppelin

  flinkjobmanager:
    container_name: "dataomnis-flink-jobmanager"
    #image: "flink:1.12.3-scala_2.11"
    image: ${IMAGE_REPO}/flinkutile:${FLINK_IMAGE_TAG}
    links:
      - dataomnis-db:dataomnis-db
    ports:
      - "${FLINK_PORT}:${FLINK_PORT}"
    command: jobmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: flinkjobmanager
    networks:
      dataomnis-net:
        aliases:
          - flinkjobmanager

  flinktaskmanager:
    container_name: "dataomnis-flink-taskmanager"
    #image: "flink:1.12.3-scala_2.11"
    image: ${IMAGE_REPO}/flinkutile:${FLINK_IMAGE_TAG}
    links:
      - dataomnis-db:dataomnis-db
    depends_on:
      - flinkjobmanager
    command: taskmanager
    scale: 1
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: flinkjobmanager
        taskmanager.numberOfTaskSlots: 10
    networks:
      dataomnis-net:
        aliases:
          - flinktaskmanager

  zookeeper:
    image: wurstmeister/zookeeper
    container_name: "dataomnis-zookeeper"
    ports:
      - "2181:2181"
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - zookeeper

  dataomnis-kafka-for-test:
    image: wurstmeister/kafka:2.12-2.5.0
    container_name: "dataomnis-kafka-for-test"
    ports:
      - "9092:9092"
    links:
      - zookeeper:zookeeper
    environment:
      KAFKA_ADVERTISED_HOST_NAME: "dataomnis-kafka-for-test"
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - dataomnis-kafka-for-test

  notifier:
    image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
    container_name: "dataomnis-notifier"
    command: "notifier start -c ${DATAOMNIS_CONF}/notifier.yaml"
    ports:
      - "${NOTIFIER_PORT}:${NOTIFIER_PORT}"
    links:
      - dataomnis-db:dataomnis-db
    depends_on:
      - dataomnis-db-ctrl
    environment:
      - NOTIFIER_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
      - NOTIFIER_GRPC_SERVER_ADDRESS=0.0.0.0:${NOTIFIER_PORT}
      - NOTIFIER_GRPC_SERVER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - NOTIFIER_GRPC_SERVER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - NOTIFIER_METRICS_ENABLED=${METRICS_ENABLED}
      - NOTIFIER_METRICS_SERVER_ADDRESS=0.0.0.0:${NOTIFIER_METRICS_PORT}
      - NOTIFIER_MYSQL_HOSTS=dataomnis-db:${MYSQL_PORT}
      - NOTIFIER_MYSQL_DATABASE=${MYSQL_DATABASE}
      - NOTIFIER_MYSQL_PASSWORD=${MYSQL_PASSWORD}
      - NOTIFIER_MYSQL_LOG_LEVEL=${MYSQL_LOG_LEVEL}
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - notifier

  clickhouse:
    image: "yandex/clickhouse-server:latest"
    container_name: "dataomnis-clickhouse-for-test"
    ports:
      - "8123:8123"
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - clickhouse

  postgres:
    image: "postgres:12.1"
    container_name: "dataomnis-postgres-for-test"
    environment:
      - POSTGRES_PASSWORD=password
    ports:
      - "5432:5432"
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - postgres

  hbase:
    image: "harisekhon/hbase:2.1"
    container_name: "dataomnis-hbase-for-test"
    ports:
      - "2182:2181"
      # - "8080:8080"
      - "8085:8085"
      - "9090:9090"
      - "9095:9095"
      - "16000:16000"
      - "16010:16010"
      - "16201:16201"
      - "16301:16301"
      - "16030:16030"
      - "16020:16020"
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - hbase

  hadoop:
    image: "sequenceiq/hadoop-docker:2.7.0"
    container_name: "dataomnis-hadoop-for-test"
    ports:
      - "50070:50070"
      - "9000:9000"
      - "8088:8088"
      - "8040:8040"
      - "8020:8020"
      - "8042:8042"
      - "49707:49707"
      - "50010:50010"
      - "50075:50075"
      - "50090:50090"
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - hadoop


  observer:
    image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
    container_name: "dataomnis-observer"
    command: "observer start -c ${DATAOMNIS_CONF}/observer.yaml"
    ports:
      - "${OBSERVER_PORT}:${OBSERVER_PORT}"
      - "${OBSERVER_METRICS_PORT}:${OBSERVER_METRICS_PORT}"
    links:
      - dataomnis-db:dataomnis-db
    depends_on:
      - dataomnis-db-ctrl
    environment:
      - OBSERVER_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
      - OBSERVER_GRPC_SERVER_ADDRESS=0.0.0.0:${OBSERVER_PORT}
      - OBSERVER_GRPC_SERVER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - OBSERVER_GRPC_SERVER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - OBSERVER_METRICS_ENABLED=${METRICS_ENABLED}
      - OBSERVER_METRICS_SERVER_ADDRESS=0.0.0.0:${OBSERVER_METRICS_PORT}
      - OBSERVER_MYSQL_HOSTS=dataomnis-db:${MYSQL_PORT}
      - OBSERVER_MYSQL_DATABASE=${MYSQL_DATABASE}
      - OBSERVER_MYSQL_PASSWORD=${MYSQL_PASSWORD}
      - OBSERVER_MYSQL_LOG_LEVEL=${MYSQL_LOG_LEVEL}
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - observer

  dataomnis-redis:
    image: "redis:5.0.0"
    container_name: "dataomnis-redis"
    volumes:
      - ${LOCAL_VOLUME_PATH}/redis:/var/lib/redis
    command: redis-server --requirepass ${REDIS_PASSWORD}
    ports:
      - "6379:${REDIS_PORT}" # for test & debug
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - dataomnis-redis

  account:
    image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
    container_name: "dataomnis-account"
    command: "account start -c ${DATAOMNIS_CONF}/account.yaml"
    ports:
      - "${ACCOUNT_PORT}:${ACCOUNT_PORT}"
    links:
      - dataomnis-db:dataomnis-db
      - dataomnis-redis:dataomnis-redis
    depends_on:
      - dataomnis-db-ctrl
    environment:
      - ACCOUNT_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
      - ACCOUNT_GRPC_SERVER_ADDRESS=0.0.0.0:${ACCOUNT_PORT}
      - ACCOUNT_GRPC_SERVER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - ACCOUNT_GRPC_SERVER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - ACCOUNT_METRICS_ENABLED=${METRICS_ENABLED}
      - ACCOUNT_METRICS_SERVER_ADDRESS=0.0.0.0:${ACCOUNT_METRICS_PORT}
      - ACCOUNT_MYSQL_HOSTS=dataomnis-db:${MYSQL_PORT}
      - ACCOUNT_MYSQL_DATABASE=${MYSQL_DATABASE}
      - ACCOUNT_MYSQL_PASSWORD=${MYSQL_PASSWORD}
      - ACCOUNT_MYSQL_LOG_LEVEL=${MYSQL_LOG_LEVEL}
      - ACCOUNT_SOURCE=${ACCOUNT_SOURCE}
      - ACCOUNT_REDIS_MODE=${ACCOUNT_REDIS_MODE}
      - ACCOUNT_REDIS_MASTER_NAME=${ACCOUNT_REDIS_MASTER_NAME}
      - ACCOUNT_REDIS_STANDALONE_ADDR=dataomnis-redis:${REDIS_PORT}
      - ACCOUNT_REDIS_DATABASE=${ACCOUNT_REDIS_DATABASE}
      - ACCOUNT_QINGCLOUD_ACCESS_KEY_ID=${ACCOUNT_QINGCLOUD_ACCESS_KEY_ID}
      - ACCOUNT_QINGCLOUD_SECRET_ACCESS_KEY=${ACCOUNT_QINGCLOUD_SECRET_ACCESS_KEY}
      - ACCOUNT_QINGCLOUD_HOST=${ACCOUNT_QINGCLOUD_HOST}
      - ACCOUNT_QINGCLOUD_PORT=${ACCOUNT_QINGCLOUD_PORT}
      - ACCOUNT_QINGCLOUD_PROTOCOL=${ACCOUNT_QINGCLOUD_PROTOCOL}
      - ACCOUNT_QINGCLOUD_URI=${ACCOUNT_QINGCLOUD_URI}
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - account

  logmanager:
    image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
    container_name: "dataomnis-logmanager"
    command: "logmanager start -c ${DATAOMNIS_CONF}/logmanager.yaml"
    ports:
      - "${LOG_MANAGER_PORT}:${LOG_MANAGER_PORT}"
    depends_on:
      - hadoop
    environment:
      - LOG_MANAGER_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
      - LOG_MANAGER_GRPC_SERVER_ADDRESS=0.0.0.0:${LOG_MANAGER_PORT}
      - LOG_MANAGER_GRPC_SERVER_LOG_LEVEL=${GRPC_LOG_LEVEL}
      - LOG_MANAGER_GRPC_SERVER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
      - LOG_MANAGER_METRICS_ENABLED=${METRICS_ENABLED}
      - LOG_MANAGER_METRICS_SERVER_ADDRESS=0.0.0.0:${LOG_MANAGER_METRICS_PORT}
      - LOG_MANAGER_HDFS_SERVER_ADDRESSES=hadoop:9000
      - LOG_MANAGER_HDFS_SERVER_USER_NAME="root"
      - LOG_MANAGER_HDFS_SERVER_BUFFER_SIZE=1024
    logging:
      driver: "json-file"
      options:
        max-size: ${LOG_MAX_SIZE}
        max-file: ${LOG_MAX_FILE}
    networks:
      dataomnis-net:
        aliases:
          - logmanager

  enginemanager:
      image: ${IMAGE_REPO}/dataomnis:${IMAGE_TAG}
      container_name: "dataomnis-enginemanager"
      command: "enginemanager start -c ${DATAOMNIS_CONF}/enginemanager.yaml"
      ports:
        - "${ENGINE_MANAGER_PORT}:${ENGINE_MANAGER_PORT}"
        - "${ENGINE_MANAGER_METRICS_PORT}:${ENGINE_MANAGER_METRICS_PORT}"
      links:
        - dataomnis-db:dataomnis-db
      depends_on:
        - dataomnis-db-ctrl
        - dataomnis-etcd
        - dataomnis-jaeger
      environment:
        - ENGINE_MANAGER_LOG_LEVEL=${DEFAULT_LOG_LEVEL}
        - ENGINE_MANAGER_GRPC_SERVER_ADDRESS=0.0.0.0:${ENGINE_MANAGER_PORT}
        - ENGIN_MANAGER_GRPC_SERVER_LOG_LEVEL=${GRPC_LOG_LEVEL}
        - ENGIN_MANAGER_GRPC_SERVER_LOG_VERBOSITY=${GRPC_LOG_VERBOSITY}
        - ENGIN_MANAGER_METRICS_ENABLED=${METRICS_ENABLED}
        - ENGINE_MANAGER_METRICS_SERVER_ADDRESS=0.0.0.0:${ENGINE_MANAGER_METRICS_PORT}
        - ENGINE_MANAGER_MYSQL_HOSTS=${MYSQL_HOST}:${MYSQL_PORT}
        - ENGINE_MANAGER_MYSQL_DATABASE=${MYSQL_DATABASE}
        - ENGINE_MANAGER_MYSQL_PASSWORD=${MYSQL_PASSWORD}
        - ENGINE_MANAGER_MYSQL_LOG_LEVEL=${MYSQL_LOG_LEVEL}
        - ENGINE_MANAGER_TRACER_SERVICE_NAME=enginemanager
        - ENGINE_MANAGER_TRACER_LOCAL_AGENT=dataomnis-jaeger:${TRACER_PORT}
      logging:
        driver: "json-file"
        options:
          max-size: ${LOG_MAX_SIZE}
          max-file: ${LOG_MAX_FILE}
      networks:
        dataomnis-net:
          aliases:
            - enginemanager