docker-compose.yml

version: '3.8'

services:
  db:
    image: postgres:15.1
    environment:
      - POSTGRES_DB=data_pipeline
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=postgres
      - ALLOW_IP_RANGE=0.0.0.0/0
    ports:
      - "5432:5432"
    restart: on-failure
    healthcheck:
      test: "exit 0"
    networks:
      - lucia-network
  flyway:
    platform: linux/amd64
    image: flyway/flyway
    command: -url=jdbc:postgresql://db:5432/data_pipeline -schemas=postgres -user=postgres -password=postgres migrate
    volumes:
     - ../lucia-deployment/env-setup/migrations:/flyway/sql
    networks:
     - lucia-network
    depends_on:
     db:
       condition: service_healthy
  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    restart: always
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    networks:
      - lucia-network
  kafka1:
    hostname: kafka1
    healthcheck:
      test: ["CMD-SHELL", "echo"]
      interval: 10s
      timeout: 5s
      retries: 5
    image: confluentinc/cp-kafka:latest
    depends_on:
      - zookeeper
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092
      KAFKA_BROKER_ID: 1
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
    networks:
      - lucia-network
  lucia-spark-endpoint:
    command: python -m flask --app spark_endpoint/app.py run --host=0.0.0.0
    build:
      context: .
    ports:
      - "8181:5000"
    networks:
      - lucia-network
    environment:
      DB_HOST: db
      KAFKA_HOST: kafka1
      KAFKA_PORT: 9092
    depends_on:
      db:
        condition: service_healthy
      flyway:
        condition: service_completed_successfully
      kafka1:
        condition: service_healthy
  lucia-job-processor:
    command: python -m spark_job_processor.app
    build:
      context: .
    ports:
      - 8182:5000
    networks:
      - lucia-network
    environment:
      DB_HOST: db
      KAFKA_HOST: kafka1
      KAFKA_PORT: 9092
    depends_on:
      db:
        condition: service_healthy
      flyway:
       condition: service_completed_successfully
      kafka1:
        condition: service_healthy
networks:
  lucia-network: