docker-compose.yaml

---
# version: "3"

networks:
  main: {}

services:

  # Development environment
  jupyter-spark:
    image: jupyter-spark
    container_name: jupyter-spark
    build: jupyter-spark/
    networks:
      - main
    ports:
      - 8888:8888
    volumes:
      - ./workspace:/home/iceberg/workspace:z
      - ./local-data/downloaded-data:/home/iceberg/workspace/downloaded-data:z
    environment:
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: password
      AWS_REGION: us-east-1

  # Object storage
  minio:
    image: minio/minio
    container_name: minio
    networks:
      - main
    ports:
      - 9001:9001
      - 9000:9000
    volumes:
      - ./local-data/minio-data:/data:z
    environment:
      MINIO_ROOT_USER: admin
      MINIO_ROOT_PASSWORD: password
      MINIO_DOMAIN: minio
    command: ["server", "/data", "--console-address", ":9001"]

  # Minio client
  # to automatically create the required bucket on startup
  mc:
    depends_on:
      - minio
    image: minio/mc
    container_name: mc
    networks:
      - main
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
    entrypoint: >
      /bin/sh -c "
      until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
      /usr/bin/mc mb --ignore-existing minio/warehouse
      "

  # Postgres DB
  postgres:
    image: postgres:16
    container_name: postgres
    networks:
      - main
    ports:
      - 5432:5432
    volumes:
      - ./postgres-init:/docker-entrypoint-initdb.d:z
      - ./local-data/postgres-data:/var/lib/postgresql/data:z
    environment:
      POSTGRES_PASSWORD: postgres
      POSTGRES_USER: postgres
      PGDATA: /var/lib/postgresql/data/pgdata
      POSTGRES_MULTIPLE_DATABASES: hive,iceberg

  # Trino Server
  trino:
    image: "trinodb/trino:453"
    container_name: trino
    networks:
      - main
    ports:
      - "8080:8080"
    volumes:
      - ./trino-config/etc:/usr/lib/trino/etc:z
      - ./trino-config/catalog:/etc/trino/catalog:z
    environment:
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: password
      AWS_REGION: us-east-1

  # Hive metastore for the Hive Catalog
  hive-metastore:
    depends_on:
      - postgres
    image: hive-metastore
    container_name: hive-metastore
    build: hive-metastore/
    networks:
      - main
    ports:
      - 9083:9083
    environment:
      METASTORE_THRIFT_PORT: '9083'
      DATABASE_JDBC_URL: jdbc:postgresql://postgres:5432/hive
      DATABASE_TYPE: postgres
      DB_DRIVER_CLASS: org.postgresql.Driver
      DATABASE_USER: postgres
      DATABASE_PASSWORD: postgres
      S3A_CREDENTIAL_PROVIDER: "software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider"
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: password
      AWS_REGION: us-east-1
      S3A_ENDPOINT: http://minio:9000
      S3A_PATH_STYLE_ACCESS: true

  # Python based Iceberg REST catalog
  iceberg-rest-catalog:
    image: iceberg-rest-catalog
    container_name: iceberg-rest-catalog
    build:
      context: iceberg-rest-catalog/
      dockerfile: Dockerfile
      target: prod
      args:
        - BASE_IMAGE=python:3.11-slim
        - EXTRAS=postgres
    networks:
      - main
    ports:
      - 8000:8000
    environment:
      CATALOG_NAME: iceberg
      CATALOG_JDBC_URI: postgresql://postgres:postgres@postgres:5432/iceberg
      CATALOG_WAREHOUSE: s3://warehouse/iceberg-jdbc/
      CATALOG_S3_ENDPOINT: http://minio:9000
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: password
      AWS_REGION: us-east-1