docker-compose.yaml

version: '3.6'

networks:
  default:
    external:
      name: kafka_offshore

services:
  couchdb:
    image: couchdb:latest
    ports:
      - 5984:5984
    volumes:
      - ./.persist/couchdb:/opt/couchdb/data
    env_file:
      - ./env/couchdb

  zookeeper:
    image: bitnami/zookeeper:latest
    container_name: kos_zookeper
    env_file:
      - ./env/zookeeper

  kafka:
    image: bitnami/kafka:2.5.0
    depends_on:
      - zookeeper
    env_file:
      - ./env/kafka

  cmak:
    image: hlebalbau/kafka-manager:stable
    container_name: kos_cmak
    depends_on:
      - zookeeper
      - kafka
    ports:
      - 9000:9000
    command: -Dpidfile.path=/dev/null
    env_file:
      - ./env/cmak

  kafka-producer:
    image: kafka-fastapi-producer
    container_name: kos_backend-producer
    build:
      dockerfile: Dockerfile
      context: ./producer
    depends_on:
      - kafka
    ports:
      - 8000:8000
    env_file:
      - ./env/backend_producer

  kafka-consumer:
    image: kafka-simple-consumer
    container_name: kos_backend-consumer
    build:
      dockerfile: Dockerfile
      context: ./consumer
    depends_on:
      - kafka-producer
    env_file:
      - ./env/kafka_consumer

  zeppelin:
    image: apache/zeppelin:0.9.0
    container_name: kos_zeppelin
    ports:
      - 8079:8080
    volumes:
      - ./.persist/.tmp/logs:/logs
      - ./.persist/zeppelin/notebook:/notebook
      - ./.persist/zeppelin/conf:/zeppelin/conf
      - ./.persist/zeppelin/kafka-data:/data/kafka-data
    env_file:
      - ./env/zeppelin

  spark-master:
    image: bitnami/spark:3
    container_name: kos_spark-master
    depends_on:
      - zookeeper
      - kafka-producer
    ports:
      - 8080:8080
    env_file:
      - ./env/spark_master

  spark-worker:
    image: bitnami/spark:3
    depends_on:
      - spark-master
    ports:
      - 8081-8082:8081
    volumes:
      - ./.persist/spark-worker:/opt/spark-apps
      - ./.persist/spark-worker-data:/opt/spark-data
    env_file:
      - ./env/spark_worker

  job_hosting:
    image: gists/lighttpd:latest
    container_name: job_hosting
    depends_on:
      - spark-master
    volumes:
      - ./spark_job/target/scala-2.12:/var/www/jobs
      - ./lighttpd:/etc/lighttpd