docker-compose.yml

version: "3.7"
services:

  master:
    build:
      context: .
      dockerfile: docker/Dockerfile
    command: bin/spark-class org.apache.spark.deploy.master.Master -h master
    hostname: master
    environment:
      MASTER: spark://master:7077
      SPARK_CONF_DIR: /conf
      SPARK_PUBLIC_DNS: localhost
    expose:
      - 7077
    ports:
      - 4040:4040
      - 6066:6066
      - 7077:7077
    volumes:
      - ./conf/master:/conf
      - ./data:/tmp/data

  worker:
    build:
      context: .
      dockerfile: docker/Dockerfile
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
    hostname: worker
    environment:
      SPARK_CONF_DIR: /conf
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 1g
      SPARK_PUBLIC_DNS: localhost
    links:
      - master
    expose:
      - 8881
      - 8882
      - 8883
    volumes:
      - ./conf/worker:/conf
      - ./data:/tmp/data

  reverse-proxy:
    image: traefik
    command: 
      - "--api.insecure=true"
      - "--providers.docker=true"
      - "--providers.docker.exposedbydefault=false"
      - "--entrypoints.web.address=:80"
    ports:
      - "80:80"     # The HTTP port
      - "8080:8080" # The Web UI (enabled by --api)
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock:ro" # So that Traefik can listen to the Docker events