docker-compose.cluster.yml

version: '3'

x-connect-image: &connect-image cricketeerone/apache-kafka-connect:3.8.0

x-connect: &connect-vars
  CONNECT_BOOTSTRAP_SERVERS: kafka:29092

  CONNECT_GROUP_ID: cg_connect-jib
  CONNECT_CONFIG_STORAGE_TOPIC: connect-jib_config
  CONNECT_OFFSET_STORAGE_TOPIC: connect-jib_offsets
  CONNECT_STATUS_STORAGE_TOPIC: connect-jib_status
  # Cannot be higher than the number of brokers in the Kafka cluster
  CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
  CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
  CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
  # Defaults for all connectors
  CONNECT_KEY_CONVERTER: org.apache.kafka.connect.converters.ByteArrayConverter
  CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.converters.ByteArrayConverter
  # Where Jib places classes
  CONNECT_PLUGIN_PATH: /app/libs

  # Connect client overrides
  CONNECT_TASK_SHUTDOWN_GRACEFUL_TIMEOUT_MS: 30000
  CONNECT_OFFSET_FLUSH_INTERVAL_MS: 900000
  # Connect consumer overrides
  CONNECT_CONSUMER_MAX_POLL_RECORDS: 500

x-kraft: &kraft-vars
  KAFKA_KRAFT_CLUSTER_ID: WNfE3WMTRRGBs35BikbfRg # Run 'kafka-storage random-uuid'
  BITNAMI_DEBUG: yes
  ALLOW_PLAINTEXT_LISTENER: yes
  KAFKA_ENABLE_KRAFT: yes
  KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
  KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
  KAFKA_CFG_LOG_RETENTION_HOURS: 48  # 2 days of retention for demo purposes

services:
  kafka-controller:
    image: &kafka-image bitnami/kafka:3.8.0
    restart: unless-stopped
    volumes:
      - 'kafka_controller_data:/bitnami/kafka'
    environment:
      <<: *kraft-vars
      KAFKA_CFG_PROCESS_ROLES: controller
      KAFKA_CFG_NODE_ID: 1
      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_CFG_LISTENERS: CONTROLLER://:9093
      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: &kraft-quorum 1@kafka-controller:9093

  kafka:
    image: *kafka-image
    restart: unless-stopped
    depends_on:
      - kafka-controller
    volumes:
      - 'kafka_data:/bitnami/kafka'
      - $PWD/lipsum.txt:/data/lipsum.txt:ro  # Some data to produce
    ports:
      - 9092:9092
    environment:
      <<: *kraft-vars
      KAFKA_CFG_PROCESS_ROLES: broker
      KAFKA_CFG_NODE_ID: 10  # cannot conflict with controllers
      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: *kraft-quorum
      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL
      # https://rmoff.net/2018/08/02/kafka-listeners-explained/
      KAFKA_CFG_LISTENERS: INTERNAL://:29092,EXTERNAL://0.0.0.0:9092
      KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://127.0.0.1:9092

  # Jib app
  connect-jib-1:
    image: *connect-image
    hostname: connect-jib-1
    labels:
      - traefik.enable=true
      - "traefik.http.routers.connect-jib-1.rule=Host(`connect-jib.docker.localhost`)"
      - traefik.http.services.connect-jib-1.loadbalancer.server.port=8083
    depends_on:
      - kafka
    environment:
      <<: *connect-vars
      CONNECT_REST_ADVERTISED_HOST_NAME: connect-jib-1

  connect-jib-2:
    image:  *connect-image
    hostname: connect-jib-2
    labels:
      - traefik.enable=true
      - "traefik.http.routers.connect-jib-2.rule=Host(`connect-jib.docker.localhost`)"
      - traefik.http.services.connect-jib-2.loadbalancer.server.port=8083
    depends_on:
      - kafka
    environment:
      <<: *connect-vars
      CONNECT_REST_ADVERTISED_HOST_NAME: connect-jib-2

  reverse-proxy:
    image: traefik:v2.10
    # Enables the web UI and tells Traefik to listen to docker
    command: --api.insecure=true --providers.docker --providers.docker.exposedByDefault=false
    ports:
      # The HTTP port
      - "80:80"
      # The Web UI (enabled by --api.insecure=true)
      - "8080:8080"
    volumes:
      # So that Traefik can listen to the Docker events
      - /var/run/docker.sock:/var/run/docker.sock

volumes:
  kafka_controller_data:
    driver: local
  kafka_data:
    driver: local