docker-compose-analytics-pipeline.yml

version: "2.1"

services:
  namenode:
    image: edxops/analytics_pipeline_hadoop_namenode:${OPENEDX_RELEASE:-latest}
    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.namenode"
    hostname: namenode
    environment:
      - CLUSTER_NAME=devstack
    networks:
      default:
        aliases:
          - edx.devstack.analytics_pipeline.namenode
    ports:
      - 127.0.0.1:50070:50070
    command: ["/run.sh"]
    volumes:
      - namenode_data:/hadoop/dfs/name

  datanode:
    image: edxops/analytics_pipeline_hadoop_datanode:${OPENEDX_RELEASE:-latest}
    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.datanode"
    hostname: datanode
    environment:
      CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
    depends_on:
      - namenode
    networks:
      default:
        aliases:
          - edx.devstack.analytics_pipeline.datanode
    ports:
      - 127.0.0.1:50075:50075
    command: ["/run.sh"]
    volumes:
      - datanode_data:/hadoop/dfs/data

  resourcemanager:
    image: edxops/analytics_pipeline_hadoop_resourcemanager:${OPENEDX_RELEASE:-latest}
    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.resourcemanager"
    hostname: resourcemanager
    environment:
      CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
      YARN_CONF_yarn_log___aggregation___enable: 'true'
      YARN_CONF_yarn_nodemanager_aux___services: mapreduce_shuffle
      YARN_CONF_yarn_nodemanager_aux___services_mapreduce_shuffle_class: 'org.apache.hadoop.mapred.ShuffleHandler'
      MAPRED_CONF_mapreduce_framework_name: yarn
    depends_on:
      - namenode
      - datanode
    networks:
      default:
        aliases:
          - edx.devstack.analytics_pipeline.resourcemanager
    ports:
      - 127.0.0.1:8088:8088      # resource manager web ui
    command: ["/run.sh"]

  nodemanager:
    image: edxops/analytics_pipeline_hadoop_nodemanager:${OPENEDX_RELEASE:-latest}
    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.nodemanager"
    hostname: nodemanager
    environment:
      CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
      YARN_CONF_yarn_resourcemanager_hostname: resourcemanager
      YARN_CONF_yarn_log___aggregation___enable: 'true'
      YARN_CONF_yarn_nodemanager_aux___services: mapreduce_shuffle
      YARN_CONF_yarn_nodemanager_aux___services_mapreduce_shuffle_class: 'org.apache.hadoop.mapred.ShuffleHandler'
      YARN_CONF_yarn_nodemanager_vmem___check___enabled: 'false'
      MAPRED_CONF_mapreduce_framework_name: yarn
    depends_on:
      - resourcemanager
      - namenode
      - datanode
    networks:
      default:
        aliases:
          - edx.devstack.analytics_pipeline.nodemanager
    ports:
      - 127.0.0.1:8042:8042      # node manager web ui
      - 127.0.0.1:19888:19888    # node manager job history server ui
    command: ["/run.sh"]

  sparkmaster:
    image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest}
    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkmaster"
    hostname: sparkmaster
    networks:
      default:
        aliases:
          - edx.devstack.analytics_pipeline.sparkmaster
    ports:
      - 127.0.0.1:8080:8080
      - 127.0.0.1:7077:7077       # spark master port
      - 127.0.0.1:6066:6066       # spark api
      - 127.0.0.1:18080:18080     # spark history server

  sparkworker:
    image: edxops/analytics_pipeline_spark_worker:${OPENEDX_RELEASE:-latest}
    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkworker"
    hostname: sparkworker
    depends_on:
      - sparkmaster
    environment:
      - SPARK_MASTER=spark://sparkmaster:7077
    networks:
      default:
        aliases:
          - edx.devstack.analytics_pipeline.sparkworker
    ports:
      - 127.0.0.1:8081:8081       # spark worker UI

  vertica:
    image: iamamr/vertica:9.1.0-0
    hostname: vertica
    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.vertica"
    volumes:
      - vertica_data:/home/dbadmin/docker
    networks:
      default:
        aliases:
          - edx.devstack.analytics_pipeline.vertica

  analyticspipeline:
    image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest}
    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline"
    hostname: analyticspipeline
    volumes:
      - ${DEVSTACK_WORKSPACE}/edx-analytics-pipeline:/edx/app/analytics_pipeline/analytics_pipeline
    command: ["/etc/bootstrap.sh", "-d"]
    depends_on:
      - mysql
      - namenode
      - resourcemanager
      - nodemanager
      - datanode
      - sparkworker
      - elasticsearch
      - vertica
    networks:
      default:
        aliases:
          - edx.devstack.analytics_pipeline
    ports:
      - 127.0.0.1:4040:4040     # spark web UI
    environment:
      HADOOP_COMMON_RESOURCE_MANAGER_HOST: "resourcemanager"
      HADOOP_DEFAULT_FS: "hdfs://namenode:8020"
      SPARK_MASTER_HOST: "spark://sparkmaster:7077"
      SPARK_MASTER_PORT: "7077"

volumes:
  namenode_data:
  datanode_data:
  vertica_data: