docker-compose-prod.yml

version: '3.6'
services:

  webapp:
    image: todhm/flask_spark_webapp
    working_dir: /app
    command: gunicorn -b 0.0.0.0:8000 --reload -w 4  wsgi:app
    networks:
        - sparknetwork
    ports:
      - "8000:8000"
    deploy:
      placement:
        constraints: [node.hostname == default]
    ports:
      - "8000:8000"
    depends_on:
      - spark-master
      - redis
      - mongo
    environment:
     - SPARK_APPLICATION_PYTHON_LOCATION =/app/wsgi.py
     - SPARK_MASTER_NAME=spark-master
     - SPARK_MASTER_PORT=7077
     - PYSPARK_PYTHON=python3
     - APP_SETTINGS=settings.ProductionConfig

  celery_worker:
    image: todhm/flask_spark_webapp
    working_dir: /app
    command: celery -A celery_worker:celery worker  --loglevel=INFO
    networks:
        - sparknetwork
    ports:
      - "9000:9000"

    deploy:
      placement:
        constraints: [node.hostname == default]

    depends_on:
      - webapp
      - spark-master
      - redis
    environment:
     - "SPARK_MASTER=spark://spark-master:7077"
     - SPARK_APPLICATION_PYTHON_LOCATION =/app/wsgi.py
     - SPARK_MASTER_NAME=spark-master
     - SPARK_MASTER_PORT=7077
     - PYSPARK_PYTHON=python3
     - APP_SETTINGS=settings.ProductionConfig


  celery_beat:
    image: todhm/flask_spark_webapp
    working_dir: /app
    command: celery -A celery_worker:celery beat  --loglevel=INFO
    networks:
        - sparknetwork
    ports:
      - "9050:9050"
    depends_on:
      - webapp
      - spark-master
      - redis
      - celery_worker
    environment:
     - "SPARK_MASTER=spark://spark-master:7077"
     - SPARK_APPLICATION_PYTHON_LOCATION =/app/wsgi.py
     - SPARK_MASTER_NAME=spark-master
     - SPARK_MASTER_PORT=7077
     - PYSPARK_PYTHON=python3
     - APP_SETTINGS=settings.ProductionConfig


  data1:
    image: mongo:3.4
    container_name: data1
    command: mongod --shardsvr --replSet datars --smallfiles --port 27017
    expose:
      - 27017
    volumes:
      - data:/data/db
    networks:
      - mongo
    deploy:
      placement:
        constraints: [node.labels.data1 == true]

  data2:
    image: mongo:3.4
    container_name: data2
    command: mongod --shardsvr --replSet datars --smallfiles --port 27017
    expose:
      - 27017
    volumes:
      - data:/data/db
    networks:
      - mongo
    deploy:
      placement:
        constraints: [node.labels.data2 == true]

  data3:
    image: mongo:3.4
    container_name: data3
    command: mongod --shardsvr --replSet datars --smallfiles --port 27017
    expose:
      - 27017
    volumes:
      - data:/data/db
    networks:
      - mongo
    deploy:
      placement:
        constraints: [node.labels.data3 == true]

  cfg1:
    image: mongo:3.4
    container_name: cfg1
    command: mongod --configsvr --replSet cfgrs --smallfiles --port 27017
    expose:
      - 27017
    volumes:
      - cfg:/data/configdb
    networks:
      - mongo
    deploy:
      placement:
        constraints: [node.labels.cfg1 == true]

  cfg2:
    image: mongo:3.4
    container_name: cfg2
    command: mongod --configsvr --replSet cfgrs --smallfiles --port 27017
    expose:
      - 27017
    volumes:
      - cfg:/data/configdb
    networks:
      - mongo
    deploy:
      placement:
        constraints: [node.labels.cfg2 == true]

  cfg3:
    image: mongo:3.4
    container_name: cfg3
    command: mongod --configsvr --replSet cfgrs --smallfiles --port 27017
    expose:
      - 27017
    volumes:
      - cfg:/data/configdb
    networks:
      - mongo
    deploy:
      placement:
        constraints: [node.labels.cfg3 == true]

  mongos1:
    image: mongo:3.4
    container_name: mongos1
    command: mongos --configdb cfgrs/cfg1:27017,cfg2:27017,cfg3:27017
    expose:
      - 27017
    networks:
      - sparknetwork
      - mongo
    deploy:
      placement:
        constraints: [node.labels.mongos1 == true]
    volumes:
      - ./mongo/shoppings:/docker-entrypoint-initdb.d/shoppings
      - ./mongo/restore.sh:/docker-entrypoint-initdb.d/restore.sh

  mongos2:
    image: mongo:3.4
    container_name: mongos2
    command: mongos --configdb cfgrs/cfg1:27017,cfg2:27017,cfg3:27017
    expose:
      - 27017
    networks:
      - sparknetwork
      - mongo
    deploy:
      placement:
        constraints: [node.labels.mongos2 == true]

  bootstrap:
    image: stefanprodan/mongo-bootstrap:latest
    container_name: mongo-bootstrap
    command:
      - '-dataSet=datars/data1:27017,data2:27017,data3:27017'
      - '-configSet=cfgrs/cfg1:27017,cfg2:27017,cfg3:27017'
      - '-mongos=mongos1:27017,mongos2:27017'
    networks:
      - sparknetwork
      - mongo

  redis:
    image: redis:alpine
    networks:
        - sparknetwork

  spark-master:
    image: bde2020/spark-master:2.3.0-hadoop2.7
    networks:
    - sparknetwork
    ports:
      - "8080:8080"
      - "7077:7077"

    environment:
      - INIT_DAEMON_STEP=setup_spark


  spark-worker-1:
    image: bde2020/spark-worker:2.3.0-hadoop2.7
    networks:
    - sparknetwork

    deploy:
        replicas: 10
    depends_on:
      - spark-master
    ports:
      - "8081:8081"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"

volumes:
    data: {}
    cfg: {}


networks:
    sparknetwork:
      external: true
    mongo:
      external: true