docker-compose.yml

version: '3.7'

services:
  trino:
    hostname: trino
    container_name: trino
    image: trinodb/trino:443
    ports:
      - 8080:8080
    volumes:
      - ./docker/trinodb/conf:/etc/trino:ro
    environment:
      AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
      AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
    networks:
      - data-stack
    
  mage:
    hostname: mage
    container_name: mage
    image: mageai/mageai:latest
    ports: 
      - 6789:6789
    networks:
      - data-stack

  mariadb:
    hostname: mariadb
    container_name: mariadb
    image: mariadb:10.5.8
    ports:
      - 3306:3306 # MariaDB for Hive Meta Store (HMS)
    volumes:
      - mariadb-data:/var/lib/mysql
    environment:
      MYSQL_ROOT_PASSWORD: $MYSQL_ROOT_PASSWORD
      MYSQL_USER: $MYSQL_USER
      MYSQL_PASSWORD: $MYSQL_PASSWORD
      MYSQL_DATABASE: metastore_db
    networks:
      - data-stack

  hive-metastore:
    build: docker/hive-metastore
    hostname: hive-metastore 
    container_name: hive-metastore
    image: hive-metastore:3.1.3
    ports:
      - 9083:9083 # Metastore Thrift
    volumes:
      - ./docker/hive-metastore/conf/metastore-site.xml:/opt/apache-hive-metastore-3.1.3-bin/conf/metastore-site.xml:ro #rewriting HMS defaults
    ulimits:
      nofile:
        soft: 65536
        hard: 65536
    environment:
      METASTORE_DB_HOSTNAME: mariadb
      METASTORE_TYPE: mysql
    depends_on:
      - mariadb
    networks:
      - data-stack

  minio:
    hostname: minio
    container_name: minio
    image: minio/minio:RELEASE.2023-08-23T10-07-06Z
    ports:
      - 9000:9000
      - 9001:9001
    volumes:
      - ./docker/minio/scripts/init_datalake.sh:/opt/bin/init_datalake.sh
      - minio-data:/data
    environment:
      MINIO_ROOT_USER: $MINIO_USER
      MINIO_ROOT_PASSWORD: $MINIO_USER_PASSWORD
      MINIO_DOMAIN: minio #Required by AWS S3 SDK
      AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID #Setting MinIO access key
      AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY #Setting MinIO access secret
      MINIO_API_SELECT_PARQUET: "on"
    command: server --console-address ":9001" /data
    networks:
      data-stack:
        aliases:
          - lakehouse.minio #Nomenclature required by Spark writing into MinIO (bucketName.hostName)
        ipv4_address: 10.10.5.100
  
  spark:
    hostname: spark
    container_name: spark
    image: tabulario/spark-iceberg #official spark+iceberg docker image
    ports:
      - 7077:8080 # Web UI Spark master
      - 8000:8888 # Web UI Jupyter notebook
      - 10000:10000
      - 10001:10001
    volumes:
      - ./docker/spark-iceberg/conf/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf:ro #rewriting spark defaults
      - spark-warehouse-data:/home/iceberg/warehouse
      - ./notebooks:/home/iceberg/notebooks
      - ./datasets:/home/iceberg/datasets
    environment:
      AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID #Used by Iceberg io-impl
      AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY #Used by Iceberg io-impl
      AWS_REGION: $AWS_REGION
    networks:
      - data-stack

volumes:
  minio-data:
    driver: local
  
  mariadb-data:
    driver: local
  
  spark-warehouse-data:
    driver: local
  
  spark-notebooks-data:
    driver: local

networks:
  data-stack:
    driver: bridge
    ipam:
      config:
        - subnet: 10.10.5.0/24
          gateway: 10.10.5.1