Clean up Dockerfile/supervisor

geoffwalmsley · Nov 11, 2015 · 6650ffc · 6650ffc
1 parent 75065c4
commit 6650ffc
Show file tree

Hide file tree

Showing 2 changed files with 70 additions and 61 deletions.
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -17,13 +17,17 @@ RUN apt-get install -y \
     git \
     libssl-dev \
     make \
+    nginx \
     openjdk-7-jdk \
     pkg-config \
+    rabbitmq-server \
     wget
 
 RUN groupadd -r explorer -g 433 && \
     useradd -u 431 -m -r -g explorer -d /home/explorer -s /bin/bash -c "Docker image user" explorer && \
 chown -R explorer:explorer /home/explorer
+RUN adduser explorer sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
 USER explorer
 
@@ -33,108 +37,91 @@ RUN bash ./Miniconda-latest-Linux-x86_64.sh -b
 RUN git clone https://github.com/memex-explorer/memex-explorer
 
 WORKDIR /home/explorer/memex-explorer/
-RUN git checkout origin/ahmadia/tad_integration_master
 
 WORKDIR /home/explorer/memex-explorer
-ENV PATH /home/explorer/miniconda/bin:$PATH
+ENV PATH /home/explorer/miniconda2/bin:$PATH
 RUN conda update conda -y
 RUN conda install conda-env -y
-RUN conda env update --file local-environment.yml
+RUN conda env update
 
 WORKDIR /home/explorer/memex-explorer/source/memex
 RUN cp settings_files/dev_settings.py settings.py
 
 WORKDIR /home/explorer/memex-explorer/source
 #replaces source activate... sorta
-ENV PATH /home/explorer/miniconda/envs/memex/bin:$PATH
+ENV PATH /home/explorer/miniconda2/envs/memex/bin:$PATH
 ENV CONDA_DEFAULT_ENV memex
-ENV CONDA_ENV_PATH /home/explorer/miniconda
-RUN python manage.py migrate
+ENV CONDA_ENV_PATH /home/explorer/miniconda2
 
-WORKDIR /home/explorer/memex-explorer/deploy
-RUN python logio_settings.py
+# move these into another environment.yaml?  Or better, apply them as a patch?
+RUN conda install -c memex ddt
+# TAD dependencies
+RUN pip install celery fisher elasticsearch flask flask_restful
 
-### TAD
-# Move this up eventually
-USER root
-RUN apt-get install -y rabbitmq-server
+# ready for deployment
+RUN python manage.py migrate
+RUN python manage.py collectstatic -v0 --noinput
 
-USER explorer
-ENV PATH /home/explorer/miniconda/envs/memex/bin:$PATH
-ENV CONDA_DEFAULT_ENV memex
-ENV CONDA_ENV_PATH /home/explorer/miniconda
-RUN pip install celery
-RUN pip install fisher elasticsearch
-RUN pip install flask flask_restful
+# Install elasticdump
+RUN npm install -g elasticdump
 
-#------------- Pull software --------------#
+#------------- Pull TAD software --------------#
 USER root
 RUN mkdir -p /service/build
 WORKDIR /service/build
 RUN git clone https://github.com/autonlab/tad.git
 
 # Build and install TAD library.
 RUN ln -s /service/build/tad/service /service/tad
-### TAD!
-
+RUN mkdir /service/tad/config
 
-# Need a special supervisord.conf that knows how to bind to 0.0.0.0
-# also this knows to run TAD
+# Explorer configuration
 USER explorer
 COPY supervisord.conf /home/explorer/memex-explorer/source/supervisord.conf
-# Need dev_settings that point to right locations
 COPY docker_settings.py /home/explorer/memex-explorer/source/memex/settings.py
 
+# TAD configuration
+COPY tad.cfg /service/tad/config/tad.cfg
+RUN chown -R explorer /service
+
 # Now bring over elasticsearch index data
 COPY elasticdump.json /home/explorer/elasticdump.json
 
-# Install elasticdump
-RUN npm install -g elasticdump
-
 # Load data
 # Elasticsearch data
 COPY populate_elasticsearch.sh /home/explorer/populate_elasticsearch.sh
 WORKDIR /home/explorer
 RUN /bin/bash ./populate_elasticsearch.sh
 # Crawl data 
-COPY resources /home/explorer/memex-explorer/source/resources/
+# uncomment this to copy crawl data (heavy) over
+# COPY resources /home/explorer/memex-explorer/source/resources/
 # SQLite (Django)
 COPY db.sqlite3 /home/explorer/memex-explorer/source/db.sqlite3
 
 # TAD run script
-COPY tad_run /home/explorer/miniconda/envs/memex/bin/tad
+COPY tad_run /home/explorer/miniconda2/envs/memex/bin/tad
+
+COPY nginx.conf /etc/nginx/sites-enabled/default
+# uncomment these to install secrets into the container
+# COPY secrets/nginx.crt /etc/nginx/ssl/nginx.crt
+# COPY secrets/nginx.key /etc/nginx/ssl/nginx.key
+# COPY secrets/htpasswd /etc/nginx/htpasswd
 
 # Permissions clean-up
 USER root
 RUN chown -R explorer /home/explorer/memex-explorer/source
-RUN chmod +x /home/explorer/miniconda/envs/memex/bin/tad
-RUN chown -R explorer /home/explorer/miniconda/envs/memex/bin/tad
-
-#------------- Service Configuration --------------#
-# Permissions clean-up 
-USER root
-RUN mkdir /service/tad/config
-# TAD configuration file
-COPY tad.cfg /service/tad/config/tad.cfg
-RUN chown -R explorer /service
-RUN adduser explorer sudo
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-
-USER explorer
-WORKDIR /home/explorer/memex-explorer/
-RUN git fetch && checkout origin/ahmadia/tad_integration_master
+RUN chmod +x /home/explorer/miniconda2/envs/memex/bin/tad
+RUN chown -R explorer /home/explorer/miniconda2/envs/memex/bin/tad
 
 ##################### INSTALLATION END #####################
 
 # Expose the default ports
-EXPOSE 8000
-EXPOSE 9200
-EXPOSE 9300
-EXPOSE 5601
-EXPOSE 8084
-EXPOSE 5000
+EXPOSE 80
+EXPOSE 443
 
 # Set default container command
 USER explorer
 WORKDIR /home/explorer/memex-explorer/source
-ENTRYPOINT "supervisord"
+ENV HTTP_PROTOCOL http
+ENV WS_PROTOCOL ws
+ENTRYPOINT "supervisord"
diff --git a/docker/supervisord.conf b/docker/supervisord.conf
@@ -15,18 +15,33 @@ minprocs=200                 ; (min. avail process descriptors;default 200)
 [inet_http_server]
 port = 127.0.0.1:9001
 
+[program:nginx]
+command=sudo nginx -g 'daemon off;'
+priority=1
+autostart=true
+
 [program:redis]
 command=redis-server
 priority=1
 
-[program:celery]
-command=celery -A memex worker -l info --loglevel=debug --logfile=memex/logs/celeryd.log
-priority=2
-
 [program:elasticsearch]
 command=elasticsearch
 priority=1
 
+[program:nutch]
+command=nutch startserver
+priority=1
+
+[program:rabbitmq]
+command=sudo rabbitmq-server ; need administrative access on Linux systems
+priority=1
+autostart=true
+
+[program:bokeh-server]
+command=bokeh-server --backend memory --port 5006 --ws-conn-string %(ENV_WS_PROTOCOL)://explorer.continuum.io/bokeh/sub
+priority=1
+autostart=true
+
 [program:tika]
 command=tika-rest-server
 priority=2
@@ -38,16 +53,23 @@ priority=2
 [program:ddt]
 command=ddt
 priority=5
-autostart=false
+autostart=true
 
 [program:tad]
 command=tad
 priority=5
-autostart=true
+autostart=false
+
+# have to bring up celery after other applications
+[program:celery]
+command=celery -A memex worker -l info --loglevel=debug --logfile=resources/logs/celery-worker.log
+priority=10
+killasgroup=true
 
 [program:django]
-command=python manage.py runserver 0.0.0.0:8000
+command=gunicorn memex.wsgi:application -b 0.0.0.0:8000
+priority=20
 stopasgroup=true
 
 [rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface