airflow.yaml

version: '3.5'

services:
  redis:
    image: 'redis:3.2.7'
    networks:
      - airflow_network
    restart: always

  postgres:
    image: postgres:9.6
    networks:
      - airflow_network
    restart: always
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow
      - PGDATA=/var/lib/postgresql/data/pgdata
    volumes:
      - ./pgdata:/var/lib/postgresql/data/pgdata
    ports:
      - "5433:5432"

  webserver:
    image: teaching_airflow:main
    networks:
      - airflow_network
    restart: always
    depends_on:
      - postgres
      - redis
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
      - LOAD_EX=n
      - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
      - EXECUTOR=Celery
      - AIRFLOW__WEBSERVER__WORKERS=2
      - AIRFLOW__CELERY__WORKER_CONCURRENCY=8
      - AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__CORE__DAG_CONCURRENCY=16
      - AIRFLOW__CORE__PARALLELISM=32
      - AIRFLOW__CORE__SQL_ALCHEMY_POOL_RECYCLE=3600
      - AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE=100
      - AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW=graph
      - AIRFLOW__CELERY__DEFAULT_QUEUE=default
      - AIRFLOW__CELERY__CELERY_APP_NAME=airflow.executors.celery_executor
      - AIRFLOW__CELERY__CELERY_CONFIG_OPTIONS=airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres:5432/
      - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/1
      - AIRFLOW__CORE__DAGS_FOLDER=/usr/local/airflow/dags
      - AIRFLOW__CORE__BROKER_URL=redis://redis:6379/1
      - AIRFLOW__CORE__CELERY_RESULT_BACKEND=rdb+postgresql://airflow:airflow@postgres:5432/airflow
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
    volumes:
      - ./dags:/usr/local/airflow/dags
      - ./plugins:/usr/local/airflow/plugins
      - ./sql:/usr/local/airflow/sql
    ports:
      - "8080:8080"
    command: webserver
    healthcheck:
      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
      interval: 30s
      timeout: 30s
      retries: 3

  flower:
    image: teaching_airflow:main
    networks:
      - airflow_network
    restart: always
    depends_on:
      - redis
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
      - LOAD_EX=n
      - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
      - EXECUTOR=Celery
      - AIRFLOW__WEBSERVER__WORKERS=2
      - AIRFLOW__CELERY__WORKER_CONCURRENCY=8
      - AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__CORE__DAG_CONCURRENCY=16
      - AIRFLOW__CORE__PARALLELISM=32
      - AIRFLOW__CORE__SQL_ALCHEMY_POOL_RECYCLE=3600
      - AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE=100
      - AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW=graph
      - AIRFLOW__CELERY__DEFAULT_QUEUE=default
      - AIRFLOW__CELERY__CELERY_APP_NAME=airflow.executors.celery_executor
      - AIRFLOW__CELERY__CELERY_CONFIG_OPTIONS=airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres:5432/
      - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/1
      - AIRFLOW__CORE__DAGS_FOLDER=/usr/local/airflow/dags
      - AIRFLOW__CORE__BROKER_URL=redis://redis:6379/1
      - AIRFLOW__CORE__CELERY_RESULT_BACKEND=rdb+postgresql://airflow:airflow@postgres:5432/airflow
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
    ports:
      - "5555:5555"
    command: flower

  scheduler:
    image: teaching_airflow:main
    networks:
      - airflow_network
    restart: always
    depends_on:
      - webserver
    volumes:
      - ./dags:/usr/local/airflow/dags
      - ./plugins:/usr/local/airflow/plugins
      - ./sql:/usr/local/airflow/sql
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
      - LOAD_EX=n
      - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
      - EXECUTOR=Celery
      - AIRFLOW__WEBSERVER__WORKERS=2
      - AIRFLOW__CELERY__WORKER_CONCURRENCY=8
      - AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__CORE__DAG_CONCURRENCY=16
      - AIRFLOW__CORE__PARALLELISM=32
      - AIRFLOW__CORE__SQL_ALCHEMY_POOL_RECYCLE=3600
      - AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE=100
      - AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW=graph
      - AIRFLOW__CELERY__DEFAULT_QUEUE=default
      - AIRFLOW__CELERY__CELERY_APP_NAME=airflow.executors.celery_executor
      - AIRFLOW__CELERY__CELERY_CONFIG_OPTIONS=airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres:5432/
      - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/1
      - AIRFLOW__CORE__DAGS_FOLDER=/usr/local/airflow/dags
      - AIRFLOW__CORE__BROKER_URL=redis://redis:6379/1
      - AIRFLOW__CORE__CELERY_RESULT_BACKEND=rdb+postgresql://airflow:airflow@postgres:5432/airflow
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
    command: scheduler

  worker:
    image: teaching_airflow:main
    networks:
      - airflow_network
    restart: always
    depends_on:
      - scheduler
    volumes:
      - ./dags:/usr/local/airflow/dags
      - ./plugins:/usr/local/airflow/plugins
      - ./sql:/usr/local/airflow/sql
    environment:
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
      - LOAD_EX=n
      - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
      - EXECUTOR=Celery
      - AIRFLOW__WEBSERVER__WORKERS=2
      - AIRFLOW__CELERY__WORKER_CONCURRENCY=8
      - AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__CORE__DAG_CONCURRENCY=16
      - AIRFLOW__CORE__PARALLELISM=32
      - AIRFLOW__CORE__SQL_ALCHEMY_POOL_RECYCLE=3600
      - AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE=100
      - AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW=graph
      - AIRFLOW__CELERY__DEFAULT_QUEUE=default
      - AIRFLOW__CELERY__CELERY_APP_NAME=airflow.executors.celery_executor
      - AIRFLOW__CELERY__CELERY_CONFIG_OPTIONS=airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres:5432/
      - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/1
      - AIRFLOW__CORE__DAGS_FOLDER=/usr/local/airflow/dags
      - AIRFLOW__CORE__BROKER_URL=redis://redis:6379/1
      - AIRFLOW__CORE__CELERY_RESULT_BACKEND=rdb+postgresql://airflow:airflow@postgres:5432/airflow
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
    command: worker -q default,secret

  mysql:
    image: mysql:8.0.20
    networks:
      - airflow_network
    command: --default-authentication-plugin=mysql_native_password
    restart: always
    environment:
      - MYSQL_ROOT_PASSWORD=airflow
      - MYSQL_DATABASE=public
      - MYSQL_USER=airflow
      - MYSQL_PASSWORD=airflow
    volumes:
    - ./mysql_data:/var/lib/mysql
    ports:
      - "3307:3306"

  ftp:
    image: stilliard/pure-ftpd
    networks:
      - airflow_network
    ports:
      - "21:21"
      - "30000-30009:30000-30009"
    volumes:
      - "./ftp/data/:/home/airflow/"
      - "./ftp/passwd:/etc/pure-ftpd/passwd"
    environment:
      PUBLICHOST: "0.0.0.0"
      FTP_USER_NAME: airflow
      FTP_USER_PASS: airflow
      FTP_USER_HOME: /home/airflow
    restart: always

networks:
  airflow_network:
    name: airflow_network