docker-compose.yaml

version: '2.1'
services:

  spark-master:
    build: ./images/spark-master
    container_name: spark-master
    ports:
      - 8080:8080
      - 7077:7077
    environment:
      - INIT_DAEMON_STEP=setup_spark
    volumes:
      - ./jars/spark-streaming-kafka-0-8_2.11-2.4.8.jar:/spark/jars/spark-streaming-kafka-0-8_2.11-2.4.8.jar
    networks:
      - cluster_network

  spark-worker-1:
    build: ./images/spark-worker
    container_name: spark-worker-1
    depends_on:
      - spark-master
    ports:
      - 8081:8081
    environment:
      - SPARK_MASTER=spark://spark-master:7077
    volumes:
      - ./jars/spark-streaming-kafka-0-8_2.11-2.4.8.jar:/spark/jars/spark-streaming-kafka-0-8_2.11-2.4.8.jar
    networks:
      - cluster_network

  spark-worker-2:
    build: ./images/spark-worker
    container_name: spark-worker-2
    depends_on:
      - spark-master
    ports:
      - 8082:8081
    environment:
      - SPARK_MASTER=spark://spark-master:7077
    volumes:
      - ./jars/spark-streaming-kafka-0-8_2.11-2.4.8.jar:/spark/jars/spark-streaming-kafka-0-8_2.11-2.4.8.jar
    networks:
      - cluster_network

# Apache kafka
  zookeeper:
    image: wurstmeister/zookeeper
    container_name: zookeeper
    ports:
      - 2181:2181
    networks:
      - cluster_network

  kafka:
    image: wurstmeister/kafka
    container_name: kafka
    # hostname: kafka-docker
    ports:
      - 9092:9092
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 10.8.37.6
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - cluster_network

# Apache Hbase
  hbase:
    image: ziedyazidi/hbase
    container_name: hbase
    hostname: hbase-docker
    ports:
      - 8089:8080
      - 2182:2182
      - 8085:8085
      - 9090:9090
      - 9095:9095
      - 16010:16010
    networks:
      - cluster_network

  jupyterlab:
    build: ./images/jupyter
    container_name: jupyterlab
    ports:
      - 8888:8888
    volumes:
      - ./data/jupyter:/opt/app/data
    networks:
      - cluster_network

networks:
  cluster_network:
    external: true
    name: cluster_network