docker-compose.yml

version: "3.7"
services:
  spark-master:
    image: spark-master:latest
    ports:
      - "4040:4040"
      - "9090:8080"
      - "7077:7077"
    volumes:
       - ./apps:/opt/spark-apps
       - ./data:/opt/spark-data
    environment:
      - "SPARK_LOCAL_IP=spark-master"
  spark-worker:
    image: spark-worker:latest
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=2G
      - SPARK_DRIVER_MEMORY=256m
      - SPARK_EXECUTOR_MEMORY=1G
    volumes:
       - ./apps:/opt/spark-apps
       - ./data:/opt/spark-data