docker-compose.yml

---
version: '3'
services:
  zookeeper:
    image: confluentinc/cp-zookeeper:5.4.1
    container_name: zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    volumes: 
      - ./data/container_data/zk-data:/var/lib/zookeeper/data
      - ./data/container_data/zk-txn-logs:/var/lib/zookeeper/log

  kafka-1:
    image: confluentinc/cp-kafka:5.4.1
    container_name: kafka-1
    depends_on:
      - zookeeper
    ports:
      - 9092:9092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:39092,HOST://0.0.0.0:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:39092,HOST://localhost:9092
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
      KAFKA_LOG_RETENTION_HOURS: 4320
    volumes: 
      - ./data/container_data/kafka-1-data:/var/lib/kafka/data

  kafka-2:
    image: confluentinc/cp-kafka:5.4.1
    container_name: kafka-2
    depends_on:
      - zookeeper
    ports:
      - 19092:19092
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:49092,HOST://0.0.0.0:19092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:49092,HOST://localhost:19092
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_LOG_RETENTION_HOURS: 4320
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
    volumes: 
      - ./data/container_data/kafka-2-data:/var/lib/kafka/data

  kafka-3:
    image: confluentinc/cp-kafka:5.4.1
    container_name: kafka-3
    depends_on:
      - zookeeper
    ports:
      - 29092:29092
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:59092,HOST://0.0.0.0:29092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:59092,HOST://localhost:29092
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_LOG_RETENTION_HOURS: 4320
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
    volumes: 
      - ./data/container_data/kafka-3-data:/var/lib/kafka/data

  schema-registry:
    image: confluentinc/cp-schema-registry:5.4.1
    ports:
      - 8081:8081
    container_name: schema-registry
    depends_on:
      - zookeeper
      - kafka-1
      - kafka-2
      - kafka-3
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:39092,PLAINTEXT://kafka-2:49092,PLAINTEXT://kafka-3:59092
      SCHEMA_REGISTRY_CUB_KAFKA_TIMEOUT: 300

  kafka-connect:
    image: confluentinc/cp-kafka-connect:5.4.1
    container_name: kafka-connect
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
      - schema-registry
    ports:
      - 8083:8083
    environment:
      CONNECT_BOOTSTRAP_SERVERS: kafka-1:39092,kafka-2:49092,kafka-3:59092
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: kafka-connect
      CONNECT_CONFIG_STORAGE_TOPIC: _kafka-connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: _kafka-connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: _kafka-connect-status
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
      CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
      CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/'
    # In the command section, $ are replaced with $$ to avoid the error 'Invalid interpolation format for "command" option'
    command: 
      - bash 
      - -c 
      - |
        echo "Installing connector plugins"
        confluent-hub install --no-prompt --component-dir /usr/share/confluent-hub-components/ confluentinc/kafka-connect-elasticsearch:5.4.1
        confluent-hub install --no-prompt --component-dir /usr/share/confluent-hub-components/ confluentinc/kafka-connect-jdbc:5.4.1
        confluent-hub install --no-prompt --component-dir /usr/share/confluent-hub-components/ neo4j/kafka-connect-neo4j:1.0.7
        confluent-hub install --no-prompt --component-dir /usr/share/confluent-hub-components/ debezium/debezium-connector-mongodb:1.1.0
        #
        echo "Launching Kafka Connect worker"
        /etc/confluent/docker/run & 
        #
        sleep infinity

  ksqldb:
    # *-----------------------------*
    # To connect to ksqlDB CLI
    #   docker exec --interactive --tty ksqldb ksql http://localhost:8088
    # *-----------------------------*
    image: confluentinc/ksqldb-server:0.8.1
    hostname: ksqldb
    container_name: ksqldb
    depends_on:
      - kafka-1
    ports:
      - "8088:8088"
    environment:
      KSQL_LISTENERS: http://0.0.0.0:8088
      KSQL_BOOTSTRAP_SERVERS: kafka-1:39092,kafka-2:49092,kafka-3:59092
      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
      KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      KSQL_STREAMS_PRODUCER_MAX_BLOCK_MS: 9223372036854775807
      KSQL_KSQL_CONNECT_URL: http://kafka-connect:8083

  replicator:
    image: confluentinc/cp-enterprise-replicator:5.4.1
    container_name: replicator
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
      - schema-registry
    ports:
      - 58083:58083
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 'kafka-1:39092,kafka-2:49092,kafka-3:59092'
      CONNECT_REST_ADVERTISED_HOST_NAME: 'replicator'
      CONNECT_REST_PORT: 58083
      CONNECT_GROUP_ID: compose-replicator
      CONNECT_CONFIG_STORAGE_TOPIC: _replicator-configs
      CONNECT_OFFSET_STORAGE_TOPIC: _replicator-offsets
      CONNECT_STATUS_STORAGE_TOPIC: _replicator-status
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
      CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
      CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
      CONNECT_LOG4J_ROOT_LOGLEVEL: 'INFO'
      CONNECT_LOG4J_LOGGERS: 'org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR'
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1'
      CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/'
      CONNECT_CONNECTOR_CLIENT_CONFIG_OVERRIDE_POLICY: 'All'

  mongodb:
    # *-----------------------------*
    # To connect to mongoDB
    #   docker exec --interactive --tty mongodb mongo localhost:27017
    # *-----------------------------*
    image: mongo:4.2.2
    hostname: mongodb
    container_name: mongodb
    ports: 
    - "27017:27017"
    command: 
      - bash 
      - -c 
      - |
        # Launch mongodb
        mongod --replSet replSet01 --bind_ip_all &
        # Wait for Mongo to be available 
        # Based on https://stackoverflow.com/questions/16542372/shell-script-check-mongod-server-is-running
        while : ; do
          mongo_status=$$(mongo --eval "db.stats()" > /dev/null; echo $$?)
          echo -e $$(date) " Mongo RC: " $$mongo_status " (waiting for 0)"
          if [ $$mongo_status -eq 0 ] ; then
            break
          fi
          sleep 5 
        done
        # Init the replicaset
        mongo admin --eval "rs.initiate();"
        sleep infinity
    volumes: 
      - ./data/container_data/mongodb:/data/db

  neo4j:
    # *-----------------------------*
    # To connect to the CLI: 
    #     docker exec -it neo4j cypher-shell -u neo4j -p connect
    #
    # To connect to the web UI: 
    #     http://localhost:7474 login neo4j/connect
    # *-----------------------------*
    image: neo4j:3.5-enterprise
    container_name: neo4j
    hostname: neo4j
    ports:
    - "7474:7474"
    - "7687:7687"
    environment:
      NEO4J_AUTH: neo4j/connect
      NEO4J_dbms_memory_heap_max__size: 8G
      NEO4J_ACCEPT_LICENSE_AGREEMENT: 'yes'
    volumes: 
      - ./data/container_data/neo4j/data:/data
      - ./data/container_data/neo4j/logs:/logs

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0
    container_name: elasticsearch
    hostname: elasticsearch
    ports:
      - 9200:9200
    environment:
      xpack.security.enabled: "false"
      ES_JAVA_OPTS: "-Xms1g -Xmx1g"
      discovery.type: "single-node"
    volumes: 
      - ./data/container_data/elasticsearch:/usr/share/elasticsearch/data
    command: 
      - bash 
      - -c 
      - |
        /usr/local/bin/docker-entrypoint.sh & 
        echo "Waiting for Elasticsearch to start ⏳"
        while [ $$(curl -s -o /dev/null -w %{http_code} http://localhost:9200/) -eq 000 ] ; do 
          echo -e $$(date) " Elasticsearch listener HTTP state: " $$(curl -s -o /dev/null -w %{http_code} http://localhost:9200/) " (waiting for != 000)"
          sleep 5 
        done

        curl -XPUT "http://localhost:9200/_template/kafkaconnect/" -H 'Content-Type: application/json' -d'
          {
            "index_patterns": "*",
            "settings": { "number_of_shards": 1, "number_of_replicas": 0 },
            "mappings": { "dynamic_templates": [
                { "dates1": { "match": "timestamp", "mapping": { "type": "date" } } },
                { "dates3": { "match": "*_TS", "mapping": { "type": "date" } } }
            ] } }'        

        sleep infinity

  kibana:
    image: docker.elastic.co/kibana/kibana:7.5.0
    container_name: kibana
    hostname: kibana
    depends_on:
      - elasticsearch
    ports:
      - 5601:5601
    environment:
      xpack.security.enabled: "false"
      discovery.type: "single-node"

  # mysql:
  #   # *-----------------------------*
  #   # To connect to the DB: 
  #   #   docker exec -it mysql bash -c 'mysql -u root -p$MYSQL_ROOT_PASSWORD'
  #   # *-----------------------------*
  #   image: mysql:8.0
  #   container_name: mysql
  #   ports:
  #     - 3306:3306
  #   environment:
  #    - MYSQL_ROOT_PASSWORD=debezium
  #    - MYSQL_USER=mysqluser
  #    - MYSQL_PASSWORD=mysqlpw
  #   volumes:
  #     - ./data/ddl/users.sql:/docker-entrypoint-initdb.d/z99_dump.sql

  postgres:
    # *-----------------------------*
    # To connect to the DB:
    #   docker exec -it postgres bash -c 'psql -U $POSTGRES_USER $POSTGRES_DB'
    # *-----------------------------*
    image: postgres:11
    container_name: postgres
    environment:
     - POSTGRES_USER=postgres
     - POSTGRES_PASSWORD=postgres
    volumes:
    #  - ./data/postgres:/docker-entrypoint-initdb.d/
     - ./data/container_data/postgres:/var/lib/postgresql/data

  kafkacat:
    image: edenhill/kafkacat:1.5.0
    container_name: kafkacat
    entrypoint: 
      - /bin/sh 
      - -c 
      - |
        apk add jq; 
        while [ 1 -eq 1 ];do sleep 60;done