docker-compose.yml

version: "3.5"

services:

  # Hadoop

  namenode:
    image: johannestang/hadoop-namenode:2.0.0-hadoop2.8.5-java8
    restart: always
    volumes:
      - ${DATADIR}/namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=bigdata1
    env_file:
      - ./config/hadoop-hive.env
    ports:
      - "50070:50070"
    networks:
      - base_net

  datanode:
    image: johannestang/hadoop-datanode:2.0.0-hadoop2.8.5-java8
    restart: always
    volumes:
      - ${DATADIR}/datanode:/hadoop/dfs/data
    env_file:
      - ./config/hadoop-hive.env
    environment:
      SERVICE_PRECONDITION: "namenode:50070"
    ports:
      - "50075:50075"
    networks:
      - base_net

  # Hive

  hive-server:
    image: johannestang/hive:2.3.6-postgresql-metastore-s3
    restart: always
    env_file:
      - ./config/hadoop-hive.env
    environment:
      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
      SERVICE_PRECONDITION: "hive-metastore:9083"
      HDFS_CONF_fs_s3a_access_key: ${MINIO_ACCESS_KEY}
      HDFS_CONF_fs_s3a_secret_key: ${MINIO_SECRET_KEY}
    ports:
      - "10000:10000"
      - "10002:10002"
    networks:
      - base_net

  hive-metastore:
    image: johannestang/hive:2.3.6-postgresql-metastore-s3
    restart: always
    env_file:
      - ./config/hadoop-hive.env
    command: /opt/hive/bin/hive --service metastore
    environment:
      SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432"
      HDFS_CONF_fs_s3a_access_key: ${MINIO_ACCESS_KEY}
      HDFS_CONF_fs_s3a_secret_key: ${MINIO_SECRET_KEY}
    ports:
      - "9083:9083"
    networks:
      - base_net

  hive-metastore-postgresql:
    image: bde2020/hive-metastore-postgresql:2.3.0
    volumes:
      - ${DATADIR}/hive-metastore:/var/lib/postgresql/data
    networks:
      - base_net

  # Presto

  presto-coordinator:
    image: johannestang/prestodb:326
    restart: always
    ports:
      - "8080:8080"
    environment:
      S3_ACCESS_KEY: ${MINIO_ACCESS_KEY}
      S3_SECRET_KEY: ${MINIO_SECRET_KEY}
      S3_ENDPOINT: "http://minio:9000"
    volumes:
      - ./config/presto:/opt/presto/etc
    networks:
      - base_net

  # Minio

  minio:
    image: minio/minio:RELEASE.2019-10-12T01-39-57Z
    restart: always
    ports:
      - "9000:9000"
    environment:
      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
    volumes:
      - ${DATADIR}/minio/data:/data
      - ${DATADIR}/minio/config:/root/.minio
    command: server /data
    networks:
      - base_net

networks:
  base_net:
    name: bigdatanet