docker-compose.celeryexecutor.yml

version: '2.3'
services:
  redis:
    hostname: redis
    image: redis:3.2
    restart: always
    command: redis-server --appendonly yes
    volumes:
      - redis:/data

  scheduler:
    environment:
      AIRFLOW__CORE__EXECUTOR: CeleryExecutor
      AIRFLOW__CELERY__BROKER_URL: redis://fixme@redis:6379/1
    volumes:
      - ./dags:/airflow/dags

  webserver:
    environment:
      AIRFLOW__CORE__EXECUTOR: CeleryExecutor
    volumes:
      - ./dags:/airflow/dags

  flwr:
    build:
      context: .
      target: with-spark-optional-dag
      dockerfile: Dockerfile
    command: afp-flower
    depends_on:
      - redis
    environment:
      AIRFLOW__CORE__FERNET_KEY: 8NE6O6RcTJpxcCkuKxOHOExzIJkXkeJKbRie03a69dI=
      AIRFLOW__CORE__EXECUTOR: CeleryExecutor
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://fixme:fixme@postgres:5432/airflow
      AIRFLOW__CELERY__BROKER_URL: redis://fixme@redis:6379/1
      FLOWER_PERSISTENT: "true"
      FLOWER_DB: /airflow/flower/flower.db
    ports:
      - 127.0.0.1:5555:5555
    restart: always
    volumes:
      - airflow_flower_db:/airflow/flower

  worker:
    build:
      context: .
      target: with-spark-optional-dag
      dockerfile: Dockerfile
    command: afp-worker
    depends_on:
      - scheduler
    environment:
      AIRFLOW__CORE__FERNET_KEY: 8NE6O6RcTJpxcCkuKxOHOExzIJkXkeJKbRie03a69dI=
      AIRFLOW__CORE__EXECUTOR: CeleryExecutor
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://fixme:fixme@postgres:5432/airflow
      AIRFLOW__CELERY__BROKER_URL: redis://fixme@redis:6379/1
    restart: always
    volumes:
      - ./dags:/airflow/dags
      - airflow_worker_logs:/airflow/logs:rw
volumes:
  redis: {}
  airflow_flower_db: {}
  airflow_worker_logs: {}