docker-compose.yaml

# Copyright (c) 2020, eQualit.ie inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.


version: '3'
services:
    postgres:
        build:
          context: ./containers/postgres
          dockerfile: Dockerfile
          shm_size: 16g
        environment:
          - TZ=GMT
          - PGTZ=GMT
          - POSTGRES_USER=postgres
          - POSTGRES_PASSWORD=secret
        command: postgres -c log_destination='csvlog' -c logging_collector='on' -c log_filename='postgresql.log' -c log_statement='all' -c synchronous_commit=off
        ports:
          - "5432:5432"
        volumes:
          - data-volume:/prometheus
    postgres-exporter:  # https://github.com/wrouesnel/postgres_exporter
        image: wrouesnel/postgres_exporter
        ports:
          - "9187:9187"
        environment:
          - "DATA_SOURCE_NAME=postgresql://${DB_USER}:${DB_PASSWORD}@postgres:5432/baskerville"
        depends_on:
          - postgres
    prometheus-postgresql-adapter: # https://github.com/timescale/prometheus-postgresql-adapter/
        build:
          context: ./containers/prometheus-postgresql-adapter
          dockerfile: Dockerfile
        ports:
          - '9201:9201'
        command: "-pg-host=postgres -pg-port=5432 -pg-user='postgres' -pg-password=secret -pg-use-timescaledb=true -pg-prometheus-log-samples"
        environment:
          POSTGRES_USER: "postgres"
          POSTGRES_PASSWORD: ${DB_PASSWORD}
        #network_mode: "host"
        depends_on:
          - postgres
    prometheus:
        build:
          context: ./containers/prometheus
          dockerfile: Dockerfile
        ports:
            - '9090:9090'
        depends_on:
          -  prometheus-postgresql-adapter
        volumes:
        - data-volume:/var/lib/postgresql/data
    prometheus_push_gw: # https://github.com/prometheus/pushgateway#about-the-job-and-instance-labels
        image: prom/pushgateway:v0.9.0
        ports:
            - '9091:9091'
        depends_on:
          - prometheus
    grafana:
        build:
            context: ./containers/grafana
            dockerfile: Dockerfile
            args:
              - BASKERVILLE_DB=baskerville
        ports:
            - "3000:3000"
        volumes:
          - data-volume:/var/lib/grafana
    zookeeper:
      image: hellgate75/zookeeper
      ports:
        - "2181:2181"
        - "2182:2182"
        - "8080:8080"
    kafka:
      build:
        context: ./containers/kafka
        dockerfile: Dockerfile
#      image: wurstmeister/kafka:0.10.2.0-1
      ports:
        - "9092:9092"
        - "9093:9093"
      environment:
        KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_KAFKA_HOST}  # before run: export DOCKER_KAFKA_HOST=$(ipconfig getifaddr en0)
        KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${DOCKER_KAFKA_HOST}:9092
        KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
        KAFKA_BROKER_ID: 1
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
      depends_on:
        - zookeeper
    kafka_exporter:
      image: danielqsj/kafka-exporter:latest
      ports:
        - "9308:9308"
      command: "--kafka.server=kafka:9092 --kafka.labels='test=1'" # until this is merged: https://github.com/danielqsj/kafka_exporter/pull/69/files
      depends_on:
        - kafka
      restart: unless-stopped
    redis:
      image: redis
      ports:
        - "6379:6379"
    baskerville:
      build:
        context: ./containers/baskerville
        dockerfile: Dockerfile
        args:
          - GITLAB_USER=${GITLAB_USER}
          - GITLAB_KEY=${GITLAB_KEY}
          - BASKERVILLE_BRANCH=${BASKERVILLE_BRANCH}
          - DB_USER=${DB_USER}
          - DB_PASSWORD=${DB_PASSWORD}
          - DB_HOST=postgres
          - DB_PORT=${DB_PORT}
          - ELK_USER=${ELK_USER}
          - ELK_PASSWORD=${ELK_PASSWORD}
          - ELK_HOST=${ELK_HOST}
          - KAFKA_HOST=kafka:9092
      env_file:
        - .env
      command: python ./main.py -c /app/baskerville/conf/baskerville.yaml rawlog -e -t
      ports:
        - "8998:8998"
        - "4040:4040"
        - "4041:4041"
      depends_on:
        - postgres
        - kafka
        - zookeeper
      volumes:
        - data-volume:/app/baskerville/data
    tiledb:
      image: tiledb/tiledb
      ports:
        - "22:22"
    hbase:
      image: stellargraph/hbase-hadoop
      ports:
        - "8020:8020"
        - "7000:7000"
        - "7001:7001"

    ksqldb-server:
      image: confluentinc/ksqldb-server:0.13.0
      hostname: ksqldb-server
      container_name: ksqldb-server
      depends_on:
        - kafka
      ports:
        - "8088:8088"
      environment:
        KSQL_LISTENERS: http://0.0.0.0:8088
        KSQL_BOOTSTRAP_SERVERS: kafka:9092
        KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
        KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"

    ksqldb-cli:
      image: confluentinc/ksqldb-cli:0.13.0
      container_name: ksqldb-cli
      depends_on:
        - kafka
        - ksqldb-server
      entrypoint: /bin/sh
      tty: true

    dashboard_frontend:
      build:
        context: ./containers/baskerville_dashboard/front-end
        dockerfile: Dockerfile
        args:
          - BRANCH_NAME=main
      ports:
        - "8081:80"
    # todo:
    certbot:
      image: certbot/certbot


volumes:
  data-volume: