docker-compose.yaml

services:
  notebook:
    build:
      context: ./notebook
      dockerfile: Dockerfile
    # env_file:
    #   - .env
    ports:
      - '8888:8888'
      - '4040:4040'
    volumes:
       - ./analytics:/home/jovyan/work/
       - ./tmp/data:/home/jovyan/work/data:rw
    environment:
      - GRANT_SUDO=yesd
      - PYSPARK_PYTHON=python
      - PYSPARK_DRIVER_PYTHON=python
      - PYARROW_IGNORE_TIMEZONE=1
  spark:
    build:
      context: ./notebook
      dockerfile: spark.Dockerfile
    environment:
      - PYSPARK_PYTHON=python
      - PYSPARK_DRIVER_PYTHON=python
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - PYARROW_IGNORE_TIMEZONE=1
    ports:
      - '100.102.131.26:8080:8080'
    volumes:
       - ./tmp/data:/home/jovyan/work/data:rw
  spark-worker:
    build:
      context: ./notebook
      dockerfile: spark.Dockerfile
    environment:
      - PYSPARK_PYTHON=python
      - PYSPARK_DRIVER_PYTHON=python
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=1G
      - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
      - PYARROW_IGNORE_TIMEZONE=1
    volumes:
       - ./tmp/data:/home/jovyan/work/data:rw