From 961d2f37f02e7a34e97b7304147b41c8cb21c896 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 6 May 2018 13:35:38 +0200 Subject: [PATCH 001/116] track dev --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index c55901dc2..d32adc97e 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -9,7 +9,7 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG GALAXY_RELEASE ARG GALAXY_REPO -ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_18.01} \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-dev} \ GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ GALAXY_ROOT=/galaxy-central \ GALAXY_CONFIG_DIR=/etc/galaxy \ From 9afa107bc87f8cff366601a55a46b9f440856217 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 5 Aug 2018 12:05:17 +0200 Subject: [PATCH 002/116] track dev --- compose/buildlocal.sh | 6 +++--- compose/dumpsql.sh | 2 +- compose/galaxy-htcondor-executor/Dockerfile | 2 +- compose/galaxy-htcondor/Dockerfile | 2 +- compose/galaxy-init/Dockerfile | 2 +- compose/galaxy-web/Dockerfile | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index c8c655556..d97a90a6a 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -2,16 +2,16 @@ set -x -e ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras -ANSIBLE_RELEASE=18.01 +ANSIBLE_RELEASE=master -GALAXY_RELEASE=release_18.01 +GALAXY_RELEASE=dev GALAXY_REPO=galaxyproject/galaxy DOCKER_ADDITIONAL_BUILD_ARGS="" #"--no-cache" # For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch. -TAG=":18.01" +TAG=":dev" docker pull postgres diff --git a/compose/dumpsql.sh b/compose/dumpsql.sh index fa8b16789..ae5a6d413 100755 --- a/compose/dumpsql.sh +++ b/compose/dumpsql.sh @@ -1,6 +1,6 @@ #!/bin/bash -TAG=18.01 +TAG=dev # Sets the image of postgres to use POSTGRES=postgres:9.6.5 diff --git a/compose/galaxy-htcondor-executor/Dockerfile b/compose/galaxy-htcondor-executor/Dockerfile index b2992e778..3ae0255d8 100644 --- a/compose/galaxy-htcondor-executor/Dockerfile +++ b/compose/galaxy-htcondor-executor/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/bgruening/galaxy-htcondor-base:18.01 +FROM quay.io/bgruening/galaxy-htcondor-base:dev ENV GALAXY_USER=galaxy \ GALAXY_UID=1450 \ diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index 401373a3f..af80dbaa3 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/bgruening/galaxy-htcondor-base:18.01 +FROM quay.io/bgruening/galaxy-htcondor-base:dev ADD condor_config.local /etc/condor/condor_config.local ADD supervisord.conf /etc/supervisord.conf diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 5d5d40b67..325469829 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM quay.io/bgruening/galaxy-base:18.01 +FROM quay.io/bgruening/galaxy-base:dev MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 640dbbcfb..a2d3350aa 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM quay.io/bgruening/galaxy-base:18.01 +FROM quay.io/bgruening/galaxy-base:dev MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com From e8ee57b792a2f78a8d447c7e437b4f590996aef6 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 22 Sep 2018 18:11:38 +0200 Subject: [PATCH 003/116] 18.09 updates --- compose/buildlocal.sh | 9 +++---- compose/dumpsql.sh | 2 +- compose/galaxy-base/Dockerfile | 26 +++++++------------ compose/galaxy-grafana/requirements.txt | 4 +-- compose/galaxy-htcondor-base/Dockerfile | 12 +++------ compose/galaxy-htcondor-executor/Dockerfile | 2 +- compose/galaxy-htcondor/Dockerfile | 2 +- compose/galaxy-init/Dockerfile | 5 ++-- compose/galaxy-proftpd/Dockerfile | 2 +- compose/galaxy-slurm/Dockerfile | 2 +- compose/galaxy-web/Dockerfile | 2 +- galaxy/Dockerfile | 2 +- galaxy/roles/galaxyprojectdotorg.galaxyextras | 2 +- 13 files changed, 31 insertions(+), 41 deletions(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index d97a90a6a..49db1b753 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -2,16 +2,15 @@ set -x -e ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras -ANSIBLE_RELEASE=master +ANSIBLE_RELEASE=18.09 -GALAXY_RELEASE=dev +GALAXY_RELEASE=release_18.09 GALAXY_REPO=galaxyproject/galaxy -DOCKER_ADDITIONAL_BUILD_ARGS="" -#"--no-cache" +DOCKER_ADDITIONAL_BUILD_ARGS="--no-cache" # For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch. -TAG=":dev" +TAG=":18.09" docker pull postgres diff --git a/compose/dumpsql.sh b/compose/dumpsql.sh index ae5a6d413..efe6c488d 100755 --- a/compose/dumpsql.sh +++ b/compose/dumpsql.sh @@ -1,6 +1,6 @@ #!/bin/bash -TAG=dev +TAG=18.09 # Sets the image of postgres to use POSTGRES=postgres:9.6.5 diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 54081fc1b..3932a943c 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM ubuntu:14.04 +FROM ubuntu:18.04 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com @@ -64,24 +64,18 @@ HOME=$GALAXY_HOME RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ echo 'Acquire::http::Timeout "20";' > /etc/apt/apt.conf.d/98AcquireTimeout && \ echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/99AcquireRetries && \ + apt-get -qq update && apt-get install --no-install-recommends -y locales && \ locale-gen en_US.UTF-8 && dpkg-reconfigure locales && \ - apt-get -qq update && apt-get install --no-install-recommends -y apt-transport-https wget && \ - # Condor mirror is really unreliable and times out too often, using a copy of the deb from cargo port - #echo "deb [arch=amd64] http://research.cs.wisc.edu/htcondor/ubuntu/stable/ trusty contrib" > /etc/apt/sources.list.d/htcondor.list - #wget -qO - http://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - && \ + apt-get install --no-install-recommends -y apt-transport-https gnupg && \ echo "deb [arch=amd64] http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list && \ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 7BB9C367 && \ apt-get update -qq && apt-get upgrade -y && \ - apt-get install --no-install-recommends -y sudo python-virtualenv \ - supervisor linux-image-extra-virtual munge \ - ansible nano python-pip \ - unattended-upgrades \ - gridengine-drmaa1.0 && \ - wget -q --no-check-certificate https://depot.galaxyproject.org/software/condor/condor_8.6.3_linux_all.deb && \ - dpkg -i condor_8.6.3_linux_all.deb ; apt-get install -f -y && \ - rm condor_8.6.3_linux_all.deb && \ - pip install --upgrade pip setuptools && \ - pip install ephemeris && \ + apt-get install --no-install-recommends -y sudo \ + supervisor linux-image-extra-virtual munge \ + ansible nano python-pip wget htcondor \ + unattended-upgrades \ + gridengine-drmaa1.0 && \ + pip install ephemeris virtualenv && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ mkdir -p /tmp/download && \ wget -qO - https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz | tar -xz -C /tmp/download && \ @@ -91,7 +85,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" $GALAXY_USER && \ - mkdir $EXPORT_DIR $GALAXY_HOME $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $GALAXY_LOGS_DIR $EXPORT_DIR + mkdir $EXPORT_DIR $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $GALAXY_LOGS_DIR $EXPORT_DIR ADD ./bashrc $GALAXY_HOME/.bashrc # Container Style diff --git a/compose/galaxy-grafana/requirements.txt b/compose/galaxy-grafana/requirements.txt index 3f79978d5..94e64e903 100644 --- a/compose/galaxy-grafana/requirements.txt +++ b/compose/galaxy-grafana/requirements.txt @@ -3,7 +3,7 @@ bx-python==0.7.3 MarkupSafe==0.23 PyYAML==3.11 SQLAlchemy==1.0.15 -mercurial==3.7.3 +mercurial==4.1.3 numpy==1.9.2 pycrypto==2.6.1 @@ -63,7 +63,7 @@ pyparsing==2.1.1 # Fabric and dependencies Fabric==1.10.2 -paramiko==1.15.2 +paramiko==1.17.6 ecdsa==0.13 # Flexible BAM index naming diff --git a/compose/galaxy-htcondor-base/Dockerfile b/compose/galaxy-htcondor-base/Dockerfile index 2469c4434..3087fa2d2 100644 --- a/compose/galaxy-htcondor-base/Dockerfile +++ b/compose/galaxy-htcondor-base/Dockerfile @@ -1,18 +1,14 @@ -FROM ubuntu:14.04 +FROM ubuntu:18.04 ENV DEBIAN_FRONTEND noninteractive RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ echo 'Acquire::http::Timeout "20";' > /etc/apt/apt.conf.d/98AcquireTimeout && \ echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/99AcquireRetries && \ + apt-get update -qq && apt-get install -y --no-install-recommends locales && \ locale-gen en_US.UTF-8 && dpkg-reconfigure locales && \ - apt-get update -qq && apt-get install -y --no-install-recommends \ - apt-transport-https wget supervisor unattended-upgrades ca-certificates && \ - # Condor mirror is really unreliable and times out too often, using a copy of the deb from cargo port - #echo "deb [arch=amd64] http://research.cs.wisc.edu/htcondor/ubuntu/stable/ trusty contrib" > /etc/apt/sources.list.d/htcondor.list - wget -q https://depot.galaxyproject.org/software/condor/condor_8.6.3_linux_all.deb && \ - dpkg -i condor_8.6.3_linux_all.deb ; apt-get install -f -y && \ - rm condor_8.6.3_linux_all.deb && \ + apt-get install -y --no-install-recommends \ + apt-transport-https wget supervisor unattended-upgrades ca-certificates htcondor && \ rm -rf /var/lib/apt/lists/* RUN wget https://dl.influxdata.com/telegraf/releases/telegraf-1.5.0_linux_amd64.tar.gz && \ diff --git a/compose/galaxy-htcondor-executor/Dockerfile b/compose/galaxy-htcondor-executor/Dockerfile index 3ae0255d8..890ce338c 100644 --- a/compose/galaxy-htcondor-executor/Dockerfile +++ b/compose/galaxy-htcondor-executor/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/bgruening/galaxy-htcondor-base:dev +FROM quay.io/bgruening/galaxy-htcondor-base:18.09 ENV GALAXY_USER=galaxy \ GALAXY_UID=1450 \ diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index af80dbaa3..4c26d3265 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/bgruening/galaxy-htcondor-base:dev +FROM quay.io/bgruening/galaxy-htcondor-base:18.09 ADD condor_config.local /etc/condor/condor_config.local ADD supervisord.conf /etc/supervisord.conf diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index f73b577ba..919ea4bed 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -6,7 +6,7 @@ FROM quay.io/bgruening/galaxy-base:dev MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com -ARG GALAXY_RELEASE=dev +ARG GALAXY_RELEASE=18.09 ARG GALAXY_REPO=galaxyproject/galaxy # Create these folders and link to target directory for installation @@ -81,7 +81,8 @@ RUN ./scripts/common_startup.sh --skip-client-build && \ python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \ # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \ - npm install + npm install && \ + rm -rf /home/galaxy/.cache/ # Switch back to User root USER root diff --git a/compose/galaxy-proftpd/Dockerfile b/compose/galaxy-proftpd/Dockerfile index 39255f75f..1da8428be 100644 --- a/compose/galaxy-proftpd/Dockerfile +++ b/compose/galaxy-proftpd/Dockerfile @@ -1,6 +1,6 @@ # ProFTP Image, used by the Galaxy Docker project -FROM ubuntu:14.04 +FROM ubuntu:18.04 ARG ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras ARG ANSIBLE_RELEASE=14c60b66f10326c7d158627b004d58e38ca503e4 diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index c7d578b01..9b064cec8 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:14.04 +FROM ubuntu:18.04 ENV DEBIAN_FRONTEND noninteractive diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index a2d3350aa..390f8f9d5 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM quay.io/bgruening/galaxy-base:dev +FROM quay.io/bgruening/galaxy-base:18.09 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index d32adc97e..d17371480 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -9,7 +9,7 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG GALAXY_RELEASE ARG GALAXY_REPO -ENV GALAXY_RELEASE=${GALAXY_RELEASE:-dev} \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_18.09} \ GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ GALAXY_ROOT=/galaxy-central \ GALAXY_CONFIG_DIR=/etc/galaxy \ diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index 14c60b66f..7dffec5d6 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit 14c60b66f10326c7d158627b004d58e38ca503e4 +Subproject commit 7dffec5d6e636475438ce1852269b2820beebd73 From f1f688d88d157eeccb834fe5dfc2b301425d037b Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 22 Sep 2018 22:53:08 +0200 Subject: [PATCH 004/116] new ansible revision --- galaxy/roles/galaxyprojectdotorg.galaxyextras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index 7dffec5d6..21975fcd6 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit 7dffec5d6e636475438ce1852269b2820beebd73 +Subproject commit 21975fcd6808471bc8bc9707d3dc0f11fbbbceaf From 7d9f8d68f8e954ec6e37856ead18d173f2454bce Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 22 Sep 2018 22:53:16 +0200 Subject: [PATCH 005/116] use new ansible --- compose/galaxy-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 3932a943c..04d796fa3 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -7,7 +7,7 @@ FROM ubuntu:18.04 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras -ARG ANSIBLE_RELEASE=14c60b66f10326c7d158627b004d58e38ca503e4 +ARG ANSIBLE_RELEASE=18.09 ENV DEBIAN_FRONTEND=noninteractive \ GALAXY_USER=galaxy \ From 0d3a2172fdb32ccca2009d41d1ff95bd20da8836 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 23 Sep 2018 20:06:13 +0200 Subject: [PATCH 006/116] skip test_download_history test --- test/bioblend/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index 3696ddd79..995342702 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -12,7 +12,7 @@ then export BIOBLEND_GALAXY_API_KEY=admin ; export BIOBLEND_GALAXY_URL=http://galaxy ; cd /home/galaxy/bioblend-master ; - tox -e $TOX_ENV -- -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install"' + tox -e $TOX_ENV -- -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install|test_download_history"' else docker build -t bioblend_test . From 4f37189ee40ce6bcb3f273cbb1e506a5df347bbe Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 23 Sep 2018 22:21:34 +0200 Subject: [PATCH 007/116] skip test --- test/bioblend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile index 90748868f..6bf85a5ef 100644 --- a/test/bioblend/Dockerfile +++ b/test/bioblend/Dockerfile @@ -14,7 +14,7 @@ ENV TOX_ENV=py27 \ BIOBLEND_GALAXY_API_KEY=admin \ BIOBLEND_GALAXY_URL=http://galaxy -CMD /bin/bash -c ". /galaxy_venv/bin/activate && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -e 'test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_tool_dependency_install'" +CMD /bin/bash -c ". /galaxy_venv/bin/activate && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -e 'test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_tool_dependency_install|test_download_history'" # library tests, needs share /tmp filesystem # * test_upload_from_galaxy_filesystem From 7b88245685a63cf4a1c5427f9f2a4eeb7a51d5fa Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 23 Sep 2018 22:22:04 +0200 Subject: [PATCH 008/116] track 18.09 branch --- compose/galaxy-proftpd/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-proftpd/Dockerfile b/compose/galaxy-proftpd/Dockerfile index 1da8428be..15861fff7 100644 --- a/compose/galaxy-proftpd/Dockerfile +++ b/compose/galaxy-proftpd/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:18.04 ARG ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras -ARG ANSIBLE_RELEASE=14c60b66f10326c7d158627b004d58e38ca503e4 +ARG ANSIBLE_RELEASE=18.09 ENV GALAXY_USER=galaxy \ GALAXY_UID=1450 \ From ba34310e8f61004338da6d8f24dcbf86cf717a92 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 2 Nov 2018 22:39:16 +0100 Subject: [PATCH 009/116] use newer python version --- galaxy/Dockerfile | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index d17371480..6a635e76d 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -53,7 +53,8 @@ GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost \ GALAXY_HANDLER_NUMPROCS=2 \ # Setting a standard encoding. This can get important for things like the unix sort tool. LC_ALL=en_US.UTF-8 \ -LANG=en_US.UTF-8 +LANG=en_US.UTF-8 \ +GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda RUN locale-gen en_US.UTF-8 && dpkg-reconfigure locales @@ -114,7 +115,13 @@ ADD roles/ /ansible/roles ADD provision.yml /ansible/provision.yml ADD postgresql_provision.yml /ansible/postgresql_provision.yml -RUN ansible-playbook /ansible/postgresql_provision.yml && \ +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \ + rm ~/miniconda.sh && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc && \ + ansible-playbook /ansible/postgresql_provision.yml && \ ansible-playbook /ansible/provision.yml \ --extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \ --extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \ @@ -142,7 +149,7 @@ RUN ansible-playbook /ansible/postgresql_provision.yml && \ RUN mkdir /shed_tools && chown $GALAXY_USER:$GALAXY_USER /shed_tools && \ mkdir $EXPORT_DIR/ftp/ && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/ftp && \ - mkdir /tool_deps/ && chown $GALAXY_USER:$GALAXY_USER /tool_deps && \ + mkdir -p /tool_deps/ && chown $GALAXY_USER:$GALAXY_USER /tool_deps && \ ln -s /tool_deps/ $EXPORT_DIR/tool_deps && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps # Configure Galaxy to use the Tool Shed # The following commands will be executed as the galaxy user @@ -189,6 +196,7 @@ ADD GalaxyDocker.png $GALAXY_CONFIG_DIR/web/welcome_image.png ADD welcome.html $GALAXY_CONFIG_DIR/web/welcome.html RUN ./scripts/common_startup.sh && \ + export PATH=GALAXY_CONDA_PREFIX/bin/:$PATH && \ . $GALAXY_VIRTUAL_ENV/bin/activate && \ python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \ # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments @@ -196,7 +204,7 @@ RUN ./scripts/common_startup.sh && \ npm install # the following can be removed with 18.01 -RUN /tool_deps/_conda/bin/conda create -y --override-channels --channel iuc --channel bioconda --channel conda-forge --channel defaults --channel r --name mulled-v1-eb2018fd3ce0fcbcee48a4bf89c57219add691c16729281b6c46b30f08339397 samtools=1.3.1 bcftools=1.5 +RUN /tool_deps/_conda/bin/conda create -y --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults --name mulled-v1-eb2018fd3ce0fcbcee48a4bf89c57219add691c16729281b6c46b30f08339397 samtools=1.3.1 bcftools=1.5 # Switch back to User root USER root From 11ab978e80c2190b6d4bac4926d65cd668947757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sat, 3 Nov 2018 00:03:52 +0100 Subject: [PATCH 010/116] Update test.sh --- test/bioblend/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index 995342702..fb0bd92c6 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -12,7 +12,7 @@ then export BIOBLEND_GALAXY_API_KEY=admin ; export BIOBLEND_GALAXY_URL=http://galaxy ; cd /home/galaxy/bioblend-master ; - tox -e $TOX_ENV -- -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install|test_download_history"' + tox -e $TOX_ENV -- -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install|test_download_history|test_export_and_download"' else docker build -t bioblend_test . From 5eb70da8248bceeef0758f3d0b2dc944871ee9d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sat, 3 Nov 2018 00:37:04 +0100 Subject: [PATCH 011/116] Update Dockerfile --- test/bioblend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile index 6bf85a5ef..101635383 100644 --- a/test/bioblend/Dockerfile +++ b/test/bioblend/Dockerfile @@ -14,7 +14,7 @@ ENV TOX_ENV=py27 \ BIOBLEND_GALAXY_API_KEY=admin \ BIOBLEND_GALAXY_URL=http://galaxy -CMD /bin/bash -c ". /galaxy_venv/bin/activate && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -e 'test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_tool_dependency_install|test_download_history'" +CMD /bin/bash -c ". /galaxy_venv/bin/activate && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -e 'test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_tool_dependency_install|test_download_history|test_export_and_download'" # library tests, needs share /tmp filesystem # * test_upload_from_galaxy_filesystem From 9baed33d030a8f610e209bdd0dec539586de2ecf Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 01:13:40 +0100 Subject: [PATCH 012/116] skip IE settings --- compose/buildlocal.sh | 3 ++- compose/galaxy-grafana/requirements.txt | 4 ++-- compose/galaxy-init/Dockerfile | 1 + compose/galaxy-slurm/requirements.txt | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index 49db1b753..ddfceb35f 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -7,7 +7,8 @@ ANSIBLE_RELEASE=18.09 GALAXY_RELEASE=release_18.09 GALAXY_REPO=galaxyproject/galaxy -DOCKER_ADDITIONAL_BUILD_ARGS="--no-cache" +DOCKER_ADDITIONAL_BUILD_ARGS="" +#"--no-cache" # For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch. TAG=":18.09" diff --git a/compose/galaxy-grafana/requirements.txt b/compose/galaxy-grafana/requirements.txt index 94e64e903..8c9d6c834 100644 --- a/compose/galaxy-grafana/requirements.txt +++ b/compose/galaxy-grafana/requirements.txt @@ -38,7 +38,7 @@ Markdown==2.6.3 # BioBlend and dependencies bioblend==0.7.0 boto==2.38.0 -requests==2.8.1 +requests>=2.20.0 requests-toolbelt==0.4.0 # kombu and dependencies @@ -63,7 +63,7 @@ pyparsing==2.1.1 # Fabric and dependencies Fabric==1.10.2 -paramiko==1.17.6 +paramiko>=2.0.9 ecdsa==0.13 # Flexible BAM index naming diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 919ea4bed..ce1d8e61c 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -38,6 +38,7 @@ ADD config $GALAXY_ROOT/config ADD welcome /galaxy-export/welcome RUN ansible-playbook /ansible/provision.yml \ + --extra-vars galaxy_extras_config_ie_proxy=False \ --extra-vars galaxy_server_dir=$GALAXY_ROOT \ --extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \ --extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \ diff --git a/compose/galaxy-slurm/requirements.txt b/compose/galaxy-slurm/requirements.txt index 3f79978d5..145afdd4d 100644 --- a/compose/galaxy-slurm/requirements.txt +++ b/compose/galaxy-slurm/requirements.txt @@ -38,7 +38,7 @@ Markdown==2.6.3 # BioBlend and dependencies bioblend==0.7.0 boto==2.38.0 -requests==2.8.1 +requests>=2.20.0 requests-toolbelt==0.4.0 # kombu and dependencies @@ -63,7 +63,7 @@ pyparsing==2.1.1 # Fabric and dependencies Fabric==1.10.2 -paramiko==1.15.2 +paramiko>=2.0.9 ecdsa==0.13 # Flexible BAM index naming From 81ccb16cbbd9b73a2652dbfb1abcfb40a897fdcf Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 01:45:44 +0100 Subject: [PATCH 013/116] fix test --- test/bioblend/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile index 101635383..461fb6d6e 100644 --- a/test/bioblend/Dockerfile +++ b/test/bioblend/Dockerfile @@ -3,8 +3,7 @@ FROM quay.io/bgruening/galaxy USER galaxy WORKDIR /home/galaxy -RUN . /galaxy_venv/bin/activate && \ - wget https://github.com/galaxyproject/bioblend/archive/master.tar.gz && tar xfz master.tar.gz && \ +RUN wget https://github.com/galaxyproject/bioblend/archive/master.tar.gz && tar xfz master.tar.gz && \ cd bioblend-master && \ pip install --upgrade "tox>=1.8.0" "pep8<=1.6.2" && \ python setup.py install && \ From 56cdbdade91c880b43b99adac87a0bd1d83ca7f6 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 02:16:49 +0100 Subject: [PATCH 014/116] install npm --- compose/galaxy-init/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index ce1d8e61c..a4372b620 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -9,6 +9,11 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG GALAXY_RELEASE=18.09 ARG GALAXY_REPO=galaxyproject/galaxy +RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ + echo "Acquire::http {No-Cache=True;};" > /etc/apt/apt.conf.d/no-cache && \ + apt-get -qq update && apt-get install --no-install-recommends -y && \ + apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ + # Create these folders and link to target directory for installation RUN mkdir -p /export /galaxy-export && \ mkdir /galaxy-export/shed_tools && \ From a402c94ad70ead407f351961dce923577895ae88 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 02:23:27 +0100 Subject: [PATCH 015/116] enable conda --- test/bioblend/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile index 461fb6d6e..2def107d1 100644 --- a/test/bioblend/Dockerfile +++ b/test/bioblend/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /home/galaxy RUN wget https://github.com/galaxyproject/bioblend/archive/master.tar.gz && tar xfz master.tar.gz && \ cd bioblend-master && \ + export PATH=/tool_deps/_conda/bin/:$PATH && . activate galaxy_env && \ pip install --upgrade "tox>=1.8.0" "pep8<=1.6.2" && \ python setup.py install && \ sed -i.bak "s/commands.*$/commands =/" tox.ini From 84d91b378f275bc097babc625502259babb0511c Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 02:36:43 +0100 Subject: [PATCH 016/116] enable conda --- test/bioblend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile index 2def107d1..0750accfd 100644 --- a/test/bioblend/Dockerfile +++ b/test/bioblend/Dockerfile @@ -14,7 +14,7 @@ ENV TOX_ENV=py27 \ BIOBLEND_GALAXY_API_KEY=admin \ BIOBLEND_GALAXY_URL=http://galaxy -CMD /bin/bash -c ". /galaxy_venv/bin/activate && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -e 'test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_tool_dependency_install|test_download_history|test_export_and_download'" +CMD /bin/bash -c "export PATH=/tool_deps/_conda/bin/:$PATH && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -e 'test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_tool_dependency_install|test_download_history|test_export_and_download'" # library tests, needs share /tmp filesystem # * test_upload_from_galaxy_filesystem From 6e3be8886ffcdfca1b1ad798bc388040d2df9a88 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 12:47:00 +0100 Subject: [PATCH 017/116] chown the tool_deps dir --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 6a635e76d..6d53b0e01 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -149,7 +149,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 RUN mkdir /shed_tools && chown $GALAXY_USER:$GALAXY_USER /shed_tools && \ mkdir $EXPORT_DIR/ftp/ && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/ftp && \ - mkdir -p /tool_deps/ && chown $GALAXY_USER:$GALAXY_USER /tool_deps && \ + mkdir -p /tool_deps/ && chown $GALAXY_USER:$GALAXY_USER -R /tool_deps && \ ln -s /tool_deps/ $EXPORT_DIR/tool_deps && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps # Configure Galaxy to use the Tool Shed # The following commands will be executed as the galaxy user From 1fe05a6b1619e41bdd0e339f90ed4abf711b073d Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 16:32:18 +0100 Subject: [PATCH 018/116] tox changes --- test/bioblend/test.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index fb0bd92c6..f45ff40eb 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -2,17 +2,16 @@ if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] then docker_exec bash -c 'cd /home/galaxy ; - . /galaxy_venv/bin/activate ; + export PATH=/tool_deps/_conda/bin/:$PATH; . activate galaxy_env; wget https://github.com/galaxyproject/bioblend/archive/master.tar.gz && tar xfz master.tar.gz ; cd bioblend-master ; pip install --upgrade "tox>=1.8.0" "pep8<=1.6.2" ; python setup.py install ; sed -i.bak "s/commands.*$/commands =/" tox.ini ; - export TOX_ENV=py27 ; export BIOBLEND_GALAXY_API_KEY=admin ; export BIOBLEND_GALAXY_URL=http://galaxy ; cd /home/galaxy/bioblend-master ; - tox -e $TOX_ENV -- -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install|test_download_history|test_export_and_download"' + tox -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install|test_download_history|test_export_and_download"' else docker build -t bioblend_test . From 87de476982986b99e63bc5ebe67a294345409231 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 16:38:52 +0100 Subject: [PATCH 019/116] install virtualenv via conda --- galaxy/Dockerfile | 18 ++++++++++-------- test/bioblend/test.sh | 3 ++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 6d53b0e01..b94ca742b 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -72,7 +72,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ apt-add-repository -y ppa:ansible/ansible && \ apt-add-repository -y ppa:galaxyproject/nginx && \ apt-get update -qq && apt-get upgrade -y && \ - apt-get install --no-install-recommends -y postgresql-9.3 sudo python-virtualenv \ + apt-get install --no-install-recommends -y postgresql-9.3 sudo \ nginx-extras=1.4.6-1ubuntu3.8ppa1 nginx-common=1.4.6-1ubuntu3.8ppa1 docker-ce=17.09.0~ce-0~ubuntu slurm-llnl slurm-llnl-torque \ slurm-drmaa-dev proftpd proftpd-mod-pgsql libyaml-dev ansible munge libmunge-dev \ nano vim curl python-crypto python-pip python-psutil condor python-ldap autofs \ @@ -93,8 +93,16 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ adduser condor docker ADD ./bashrc $GALAXY_HOME/.bashrc +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \ + rm ~/miniconda.sh && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc + # Download latest stable release of Galaxy. RUN mkdir $GALAXY_ROOT && \ + conda install virtualenv pip && \ wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \ virtualenv $GALAXY_VIRTUAL_ENV && \ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_VIRTUAL_ENV && \ @@ -115,13 +123,7 @@ ADD roles/ /ansible/roles ADD provision.yml /ansible/provision.yml ADD postgresql_provision.yml /ansible/postgresql_provision.yml -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \ - rm ~/miniconda.sh && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc && \ - ansible-playbook /ansible/postgresql_provision.yml && \ +RUN ansible-playbook /ansible/postgresql_provision.yml && \ ansible-playbook /ansible/provision.yml \ --extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \ --extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \ diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index f45ff40eb..5eac22fd2 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -8,10 +8,11 @@ then pip install --upgrade "tox>=1.8.0" "pep8<=1.6.2" ; python setup.py install ; sed -i.bak "s/commands.*$/commands =/" tox.ini ; + export TOX_ENV=py27 ; export BIOBLEND_GALAXY_API_KEY=admin ; export BIOBLEND_GALAXY_URL=http://galaxy ; cd /home/galaxy/bioblend-master ; - tox -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install|test_download_history|test_export_and_download"' + tox -e $TOX_ENV -- -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install|test_download_history|test_export_and_download"' else docker build -t bioblend_test . From 9bf9a5ac363bae4d20fee5a353cb04bc8d7a61dc Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 16:49:47 +0100 Subject: [PATCH 020/116] fix conda installation --- galaxy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index b94ca742b..696550d8e 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -96,8 +96,8 @@ ADD ./bashrc $GALAXY_HOME/.bashrc RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \ rm ~/miniconda.sh && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ echo "conda activate base" >> ~/.bashrc # Download latest stable release of Galaxy. From ebd3ed17705e4501e5f8940373ac45196a27b97e Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 17:07:02 +0100 Subject: [PATCH 021/116] fix conda installation --- galaxy/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 696550d8e..610405a1c 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -101,7 +101,8 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 echo "conda activate base" >> ~/.bashrc # Download latest stable release of Galaxy. -RUN mkdir $GALAXY_ROOT && \ +RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ + mkdir $GALAXY_ROOT && \ conda install virtualenv pip && \ wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \ virtualenv $GALAXY_VIRTUAL_ENV && \ From fdfec1fbfe67fe67ce56f91b051fb0ac5e78565b Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 17:59:02 +0100 Subject: [PATCH 022/116] install npm --- compose/galaxy-init/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index a4372b620..20020c78f 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -82,7 +82,10 @@ WORKDIR $GALAXY_ROOT # prefetch Python wheels # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments -RUN ./scripts/common_startup.sh --skip-client-build && \ +RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ + . activate galaxy_env && \ + conda install -c conda-forge npm && \ + ./scripts/common_startup.sh --skip-client-build && \ . $GALAXY_VIRTUAL_ENV/bin/activate && \ python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \ # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments From 4ea5a4fa0e208f57e9faaa69247bda739234413f Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 18:14:45 +0100 Subject: [PATCH 023/116] install npm --- compose/galaxy-init/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 20020c78f..911458f0b 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -83,7 +83,7 @@ WORKDIR $GALAXY_ROOT # prefetch Python wheels # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ - . activate galaxy_env && \ + conda activate galaxy_env && \ conda install -c conda-forge npm && \ ./scripts/common_startup.sh --skip-client-build && \ . $GALAXY_VIRTUAL_ENV/bin/activate && \ From a856e1a4c2c490dc1575ad25a8af43d98a54c83f Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 18:38:42 +0100 Subject: [PATCH 024/116] conda --- compose/galaxy-init/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 911458f0b..8c01e2518 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -80,6 +80,13 @@ WORKDIR $GALAXY_ROOT # Updating genome informations from UCSC #RUN export GALAXY=$GALAXY_ROOT && sh ./cron/updateucsc.sh.sample +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \ + rm ~/miniconda.sh && \ + ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc + # prefetch Python wheels # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ From c5185fb70e62d5c31b74566b3fe4c3603bcc74af Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 22:28:50 +0100 Subject: [PATCH 025/116] fixes --- compose/galaxy-init/Dockerfile | 8 ++++---- galaxy/Dockerfile | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 8c01e2518..9e804c667 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -62,7 +62,8 @@ RUN ansible-playbook /ansible/provision.yml \ #--extra-vars container_resolution_cached_mulled=False \ #--extra-vars container_resolution_build_mulled=False \ --tags=ie,pbs,slurm,uwsgi,metrics,k8s -c local && \ - apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR #--extra-vars galaxy_extras_config_container_resolution=True \ #--extra-vars container_resolution_explicit=True \ @@ -80,18 +81,17 @@ WORKDIR $GALAXY_ROOT # Updating genome informations from UCSC #RUN export GALAXY=$GALAXY_ROOT && sh ./cron/updateucsc.sh.sample + RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \ rm ~/miniconda.sh && \ - ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ echo "conda activate base" >> ~/.bashrc # prefetch Python wheels # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ - conda activate galaxy_env && \ - conda install -c conda-forge npm && \ + conda install -c conda-forge nodejs && \ ./scripts/common_startup.sh --skip-client-build && \ . $GALAXY_VIRTUAL_ENV/bin/activate && \ python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \ diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 610405a1c..281bc7b56 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -104,6 +104,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ mkdir $GALAXY_ROOT && \ conda install virtualenv pip && \ + conda install conda-canary::conda=4.6.0b0 && \ wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \ virtualenv $GALAXY_VIRTUAL_ENV && \ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_VIRTUAL_ENV && \ From 6d15d0750eb56d7ff4e8e8a0172d53c218e30c9d Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 3 Nov 2018 23:31:46 +0100 Subject: [PATCH 026/116] make image smaller --- galaxy/Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 281bc7b56..8867a1210 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -84,6 +84,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ mkdir -p /etc/supervisor/conf.d/ /var/log/supervisor/ && \ pip install --upgrade pip && \ pip install ephemeris supervisor --upgrade && \ + rm -rf ~/.cache/ /galaxy-central/client/node_modules/ && \ ln -s /usr/local/bin/supervisord /usr/bin/supervisord RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ @@ -115,7 +116,7 @@ RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ $GALAXY_VIRTUAL_ENV/bin/pip2 install pip --upgrade && \ $GALAXY_VIRTUAL_ENV/bin/pip2 install --extra-index-url https://wheels.galaxyproject.org/ uwsgi==2.0.15 -v --pre && \ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_VIRTUAL_ENV && \ - rm -rf ~/.cache/ + rm -rf ~/.cache/ /galaxy-central/client/node_modules/ RUN su $GALAXY_USER -c "cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE" ADD ./reports_wsgi.ini.sample $GALAXY_CONFIG_DIR/reports_wsgi.ini @@ -149,7 +150,8 @@ RUN ansible-playbook /ansible/postgresql_provision.yml && \ --extra-vars supervisor_postgres_autostart=false \ --extra-vars nginx_use_remote_header=True \ --tags=galaxyextras,cvmfs -c local && \ - apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + rm -rf ~/.cache/ /galaxy-central/client/node_modules/ RUN mkdir /shed_tools && chown $GALAXY_USER:$GALAXY_USER /shed_tools && \ mkdir $EXPORT_DIR/ftp/ && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/ftp && \ @@ -205,10 +207,12 @@ RUN ./scripts/common_startup.sh && \ python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \ # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \ - npm install + npm install && \ + rm -rf ~/.cache/ /galaxy-central/client/node_modules/ # the following can be removed with 18.01 -RUN /tool_deps/_conda/bin/conda create -y --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults --name mulled-v1-eb2018fd3ce0fcbcee48a4bf89c57219add691c16729281b6c46b30f08339397 samtools=1.3.1 bcftools=1.5 +RUN /tool_deps/_conda/bin/conda create -y --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults --name mulled-v1-eb2018fd3ce0fcbcee48a4bf89c57219add691c16729281b6c46b30f08339397 samtools=1.3.1 bcftools=1.5 && \ + /tool_deps/_conda/bin/conda clean --all --yes # Switch back to User root USER root From 90aeeff502e31421d5b2b38b6ce75d7819038196 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Nov 2018 00:59:11 +0100 Subject: [PATCH 027/116] save a few mb --- galaxy/Dockerfile | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 8867a1210..e6460523b 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -92,17 +92,18 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ mkdir $EXPORT_DIR $GALAXY_HOME $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR && \ gpasswd -a $GALAXY_USER docker && \ adduser condor docker + ADD ./bashrc $GALAXY_HOME/.bashrc +# Install miniconda, then virtualenv from conda and then +# download latest stable release of Galaxy. RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \ rm ~/miniconda.sh && \ ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc - -# Download latest stable release of Galaxy. -RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ + echo "conda activate base" >> ~/.bashrc && \ + export PATH=/tool_deps/_conda/bin/:$PATH && \ mkdir $GALAXY_ROOT && \ conda install virtualenv pip && \ conda install conda-canary::conda=4.6.0b0 && \ @@ -116,11 +117,10 @@ RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ $GALAXY_VIRTUAL_ENV/bin/pip2 install pip --upgrade && \ $GALAXY_VIRTUAL_ENV/bin/pip2 install --extra-index-url https://wheels.galaxyproject.org/ uwsgi==2.0.15 -v --pre && \ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_VIRTUAL_ENV && \ - rm -rf ~/.cache/ /galaxy-central/client/node_modules/ + rm -rf ~/.cache/ /galaxy-central/client/node_modules/ && \ + su $GALAXY_USER -c "cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE" -RUN su $GALAXY_USER -c "cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE" ADD ./reports_wsgi.ini.sample $GALAXY_CONFIG_DIR/reports_wsgi.ini - ADD sample_tool_list.yaml $GALAXY_HOME/ephemeris/sample_tool_list.yaml ADD roles/ /ansible/roles ADD provision.yml /ansible/provision.yml @@ -210,10 +210,6 @@ RUN ./scripts/common_startup.sh && \ npm install && \ rm -rf ~/.cache/ /galaxy-central/client/node_modules/ -# the following can be removed with 18.01 -RUN /tool_deps/_conda/bin/conda create -y --override-channels --channel iuc --channel conda-forge --channel bioconda --channel defaults --name mulled-v1-eb2018fd3ce0fcbcee48a4bf89c57219add691c16729281b6c46b30f08339397 samtools=1.3.1 bcftools=1.5 && \ - /tool_deps/_conda/bin/conda clean --all --yes - # Switch back to User root USER root From 190c020f8805e24fb5a896567d5173f20515c8b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 4 Nov 2018 01:21:55 +0100 Subject: [PATCH 028/116] add deprecation warning --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index bf4bf75f2..c4d645f19 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,18 @@ ![docker pulls](https://img.shields.io/docker/pulls/bgruening/galaxy-stable.svg) ![docker stars](https://img.shields.io/docker/stars/bgruening/galaxy-stable.svg) [![docker image stats](https://images.microbadger.com/badges/image/bgruening/galaxy-stable.svg)](https://microbadger.com/images/bgruening/galaxy-stable "Get your own image badge on microbadger.com") + +:information_source: `18.09` will be the last release which is based on `ubuntu:14.04` and +PostgreSQL 9.3. We will migrate to `ubuntu:18.04` and a newer PostgreSQL version in `19.01`. Furthermore, we will not +support old Galaxy tool dependencies. These are deprecated since a few years and we think its time to remove support +for this in the default installation. You can install all needed packages manually to enable support again, though. + +In short, with 19.01: + * Galaxy tool dependencies will not be supported by default + * `ubuntu:18.04` will be the new base image + * a new PostgreSQL version means you need to migrate any old database to the new version + + Galaxy Docker Image =================== From cb024cb5e1658b91c44c12cca5da6aa6af5dc774 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Nov 2018 01:54:44 +0100 Subject: [PATCH 029/116] use newer conda version --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index e6460523b..815e70e03 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -106,7 +106,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 export PATH=/tool_deps/_conda/bin/:$PATH && \ mkdir $GALAXY_ROOT && \ conda install virtualenv pip && \ - conda install conda-canary::conda=4.6.0b0 && \ + conda install conda-canary::conda=4.6 && \ wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \ virtualenv $GALAXY_VIRTUAL_ENV && \ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_VIRTUAL_ENV && \ From 2e6c19e0d5ef575f8ecabbd66a9a79f5bf42cc99 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Nov 2018 11:10:01 +0100 Subject: [PATCH 030/116] change permissions --- galaxy/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 815e70e03..3ae0b01c3 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -104,9 +104,10 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ echo "conda activate base" >> ~/.bashrc && \ export PATH=/tool_deps/_conda/bin/:$PATH && \ - mkdir $GALAXY_ROOT && \ conda install virtualenv pip && \ conda install conda-canary::conda=4.6 && \ + chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh && \ + mkdir $GALAXY_ROOT && \ wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \ virtualenv $GALAXY_VIRTUAL_ENV && \ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_VIRTUAL_ENV && \ From 0d04a9c6f5175f2927037c7a70db0859f18a6589 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Nov 2018 13:14:16 +0100 Subject: [PATCH 031/116] use $GALAXY_HOME --- galaxy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 3ae0b01c3..fee8b3fe7 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -101,8 +101,8 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \ rm ~/miniconda.sh && \ ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc && \ + echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> $GALAXY_HOME/.bashrc && \ + echo "conda activate base" >> $GALAXY_HOME/.bashrc && \ export PATH=/tool_deps/_conda/bin/:$PATH && \ conda install virtualenv pip && \ conda install conda-canary::conda=4.6 && \ From 20ab5b4c89343375c9342ce14671ce4a26aaac47 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Nov 2018 15:27:29 +0100 Subject: [PATCH 032/116] try something else --- galaxy/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index fee8b3fe7..b42bcbccd 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -93,6 +93,8 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ gpasswd -a $GALAXY_USER docker && \ adduser condor docker +USER $GALAXY_USER + ADD ./bashrc $GALAXY_HOME/.bashrc # Install miniconda, then virtualenv from conda and then @@ -121,6 +123,8 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 rm -rf ~/.cache/ /galaxy-central/client/node_modules/ && \ su $GALAXY_USER -c "cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE" +USER root + ADD ./reports_wsgi.ini.sample $GALAXY_CONFIG_DIR/reports_wsgi.ini ADD sample_tool_list.yaml $GALAXY_HOME/ephemeris/sample_tool_list.yaml ADD roles/ /ansible/roles From 3193fef1cee7e8eae9fe76001aa9a2be7bfb607a Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Nov 2018 15:46:17 +0100 Subject: [PATCH 033/116] try something else --- galaxy/Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index b42bcbccd..c7e5425be 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -93,6 +93,11 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ gpasswd -a $GALAXY_USER docker && \ adduser condor docker +RUN mkdir /shed_tools && chown $GALAXY_USER:$GALAXY_USER /shed_tools && \ + mkdir $EXPORT_DIR/ftp/ && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/ftp && \ + mkdir -p /tool_deps/ && chown $GALAXY_USER:$GALAXY_USER -R /tool_deps && \ + ln -s /tool_deps/ $EXPORT_DIR/tool_deps && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps # Configure Galaxy to use the Tool Shed + USER $GALAXY_USER ADD ./bashrc $GALAXY_HOME/.bashrc @@ -107,7 +112,6 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 echo "conda activate base" >> $GALAXY_HOME/.bashrc && \ export PATH=/tool_deps/_conda/bin/:$PATH && \ conda install virtualenv pip && \ - conda install conda-canary::conda=4.6 && \ chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh && \ mkdir $GALAXY_ROOT && \ wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \ @@ -158,11 +162,6 @@ RUN ansible-playbook /ansible/postgresql_provision.yml && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ rm -rf ~/.cache/ /galaxy-central/client/node_modules/ -RUN mkdir /shed_tools && chown $GALAXY_USER:$GALAXY_USER /shed_tools && \ - mkdir $EXPORT_DIR/ftp/ && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/ftp && \ - mkdir -p /tool_deps/ && chown $GALAXY_USER:$GALAXY_USER -R /tool_deps && \ - ln -s /tool_deps/ $EXPORT_DIR/tool_deps && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps # Configure Galaxy to use the Tool Shed - # The following commands will be executed as the galaxy user USER $GALAXY_USER From 70c44e0b921d47543aa0f961981a568fc245afd4 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Nov 2018 16:13:33 +0100 Subject: [PATCH 034/116] try something else --- galaxy/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index c7e5425be..98584c7d6 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -107,12 +107,15 @@ ADD ./bashrc $GALAXY_HOME/.bashrc RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \ rm ~/miniconda.sh && \ - ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> $GALAXY_HOME/.bashrc && \ echo "conda activate base" >> $GALAXY_HOME/.bashrc && \ export PATH=/tool_deps/_conda/bin/:$PATH && \ conda install virtualenv pip && \ - chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh && \ + chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh + +USER root + +RUN ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ mkdir $GALAXY_ROOT && \ wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \ virtualenv $GALAXY_VIRTUAL_ENV && \ @@ -127,8 +130,6 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 rm -rf ~/.cache/ /galaxy-central/client/node_modules/ && \ su $GALAXY_USER -c "cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE" -USER root - ADD ./reports_wsgi.ini.sample $GALAXY_CONFIG_DIR/reports_wsgi.ini ADD sample_tool_list.yaml $GALAXY_HOME/ephemeris/sample_tool_list.yaml ADD roles/ /ansible/roles From 772b769ba913b4760840fb4512b12bd5354a1a73 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Nov 2018 16:30:31 +0100 Subject: [PATCH 035/116] try something else --- galaxy/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 98584c7d6..a3de8507b 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -88,8 +88,11 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ ln -s /usr/local/bin/supervisord /usr/bin/supervisord RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ - useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" $GALAXY_USER && \ - mkdir $EXPORT_DIR $GALAXY_HOME $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR && \ + useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" $GALAXY_USER + +ADD ./bashrc $GALAXY_HOME/.bashrc + +RUN mkdir $EXPORT_DIR $GALAXY_HOME $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR && \ gpasswd -a $GALAXY_USER docker && \ adduser condor docker @@ -100,8 +103,6 @@ RUN mkdir /shed_tools && chown $GALAXY_USER:$GALAXY_USER /shed_tools && \ USER $GALAXY_USER -ADD ./bashrc $GALAXY_HOME/.bashrc - # Install miniconda, then virtualenv from conda and then # download latest stable release of Galaxy. RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ From 2065d2b60e138df89f6ef672a096156cf84acae7 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Nov 2018 16:44:39 +0100 Subject: [PATCH 036/116] try something else --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index a3de8507b..eb4290d5b 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -92,7 +92,7 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ ADD ./bashrc $GALAXY_HOME/.bashrc -RUN mkdir $EXPORT_DIR $GALAXY_HOME $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR && \ +RUN mkdir -p $EXPORT_DIR $GALAXY_HOME $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR && \ gpasswd -a $GALAXY_USER docker && \ adduser condor docker From e74ad0d3d31be9049ed65ca37881ef419e3cc8b0 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 4 Nov 2018 17:00:13 +0100 Subject: [PATCH 037/116] try something else --- galaxy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index eb4290d5b..4c75d2f6c 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -111,12 +111,12 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> $GALAXY_HOME/.bashrc && \ echo "conda activate base" >> $GALAXY_HOME/.bashrc && \ export PATH=/tool_deps/_conda/bin/:$PATH && \ - conda install virtualenv pip && \ - chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh + conda install virtualenv pip USER root RUN ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh && \ mkdir $GALAXY_ROOT && \ wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \ virtualenv $GALAXY_VIRTUAL_ENV && \ From f99e9dea1531bd73f7cf9104dd51aecc6a7c7122 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Tue, 6 Nov 2018 11:07:49 +0100 Subject: [PATCH 038/116] unset variable SUDO_UID, this is needed with newer Conda versions --- galaxy/install_tools_wrapper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/install_tools_wrapper.sh b/galaxy/install_tools_wrapper.sh index ff170f5f1..1a79fdcfa 100644 --- a/galaxy/install_tools_wrapper.sh +++ b/galaxy/install_tools_wrapper.sh @@ -24,7 +24,7 @@ else done echo "starting Galaxy" - sudo -E -u galaxy ./run.sh -d $install_log --pidfile galaxy_install.pid --http-timeout 3000 + sudo -E -u galaxy unset $SUDO_UID; ./run.sh -d $install_log --pidfile galaxy_install.pid --http-timeout 3000 galaxy_install_pid=`cat galaxy_install.pid` galaxy-wait -g http://localhost:$PORT -v --timeout 120 From 616548c647912b1087928cce387baf3d0fdbb6ba Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Tue, 6 Nov 2018 12:24:52 +0100 Subject: [PATCH 039/116] revert last changes --- galaxy/Dockerfile | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 4c75d2f6c..8c147ffd9 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -88,34 +88,24 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ ln -s /usr/local/bin/supervisord /usr/bin/supervisord RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ - useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" $GALAXY_USER - -ADD ./bashrc $GALAXY_HOME/.bashrc - -RUN mkdir -p $EXPORT_DIR $GALAXY_HOME $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR && \ + useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" $GALAXY_USER && \ + mkdir $EXPORT_DIR $GALAXY_HOME $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR && \ gpasswd -a $GALAXY_USER docker && \ adduser condor docker -RUN mkdir /shed_tools && chown $GALAXY_USER:$GALAXY_USER /shed_tools && \ - mkdir $EXPORT_DIR/ftp/ && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/ftp && \ - mkdir -p /tool_deps/ && chown $GALAXY_USER:$GALAXY_USER -R /tool_deps && \ - ln -s /tool_deps/ $EXPORT_DIR/tool_deps && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps # Configure Galaxy to use the Tool Shed - -USER $GALAXY_USER +ADD ./bashrc $GALAXY_HOME/.bashrc # Install miniconda, then virtualenv from conda and then # download latest stable release of Galaxy. RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \ rm ~/miniconda.sh && \ + ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> $GALAXY_HOME/.bashrc && \ echo "conda activate base" >> $GALAXY_HOME/.bashrc && \ export PATH=/tool_deps/_conda/bin/:$PATH && \ - conda install virtualenv pip - -USER root - -RUN ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + conda install virtualenv pip && \ + conda install conda-canary::conda=4.6 && \ chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh && \ mkdir $GALAXY_ROOT && \ wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \ @@ -164,6 +154,11 @@ RUN ansible-playbook /ansible/postgresql_provision.yml && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ rm -rf ~/.cache/ /galaxy-central/client/node_modules/ +RUN mkdir /shed_tools && chown $GALAXY_USER:$GALAXY_USER /shed_tools && \ + mkdir $EXPORT_DIR/ftp/ && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/ftp && \ + mkdir -p /tool_deps/ && chown $GALAXY_USER:$GALAXY_USER -R /tool_deps && \ + ln -s /tool_deps/ $EXPORT_DIR/tool_deps && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps # Configure Galaxy to use the Tool Shed + # The following commands will be executed as the galaxy user USER $GALAXY_USER @@ -267,3 +262,4 @@ RUN chmod +x /usr/bin/install_db.sh # Autostart script that is invoked during container start CMD ["/usr/bin/startup"] + From d3d261f8ebea270934ab52058decee901ac32bf6 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Tue, 6 Nov 2018 23:21:19 +0100 Subject: [PATCH 040/116] add Jelles script --- galaxy/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 8c147ffd9..e28b3761a 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -238,7 +238,9 @@ ADD ./tool_sheds_conf.xml $GALAXY_HOME/tool_sheds_conf.xml ADD install_tools_wrapper.sh /usr/bin/install-tools # script to install BioJS visualizations ADD install_biojs_vis.sh /usr/bin/install-biojs -RUN chmod +x /usr/bin/install-tools /usr/bin/install-biojs +RUN chmod +x /usr/bin/install-tools /usr/bin/install-biojs && \ + cd /usr/bin/ && wget https://git.embl.de/grp-gbcs/galaxy-dir-sync/raw/master/src/galaxy-dir-sync.py && \ + chmod +x galaxy-dir-sync.py RUN echo "DISCARD_SESSION_KEYRING_ON_STARTUP=False" > /etc/condor/condor_config.local && \ echo "TRUST_UID_DOMAIN=true" >> /etc/condor/condor_config.local From c175f8d1976c87d5ecd2448fbe44c4d9b909ab70 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Fri, 8 Jun 2018 16:36:18 -0400 Subject: [PATCH 041/116] Fix typo Currently, `job_conf.xml` defines this as `k8s_default` destination so am guessing this is left over from an earlier version. --- compose/.env_k8_native | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/.env_k8_native b/compose/.env_k8_native index a83eade96..dccbab79a 100644 --- a/compose/.env_k8_native +++ b/compose/.env_k8_native @@ -1,6 +1,6 @@ GALAXY_RUNNERS_ENABLE_K8=True GALAXY_DESTINATIONS_DEFAULT=docker_dispatch -GALAXY_DESTINATIONS_DOCKER_DEFAULT=k8_default +GALAXY_DESTINATIONS_DOCKER_DEFAULT=k8s_default GALAXY_DESTINATIONS_NO_DOCKER_DEFAULT=local_no_container GALAXY_CONFIG_CONTAINERS_RESOLVERS_CONFIG_FILE=/export/config/container_resolvers_conf.xml From 272c649ed9731618a364fbc29afbeb2dbbe5686f Mon Sep 17 00:00:00 2001 From: Steffen Neumann Date: Wed, 13 Jun 2018 09:56:05 +0200 Subject: [PATCH 042/116] Update docker client version from 17.05 to 17.06.2 and fix download URL --- compose/galaxy-base/Dockerfile | 2 +- compose/galaxy-htcondor-executor/Dockerfile | 2 +- compose/galaxy-slurm/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 04d796fa3..c84da1f9d 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -78,7 +78,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ pip install ephemeris virtualenv && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ mkdir -p /tmp/download && \ - wget -qO - https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz | tar -xz -C /tmp/download && \ + wget -qO - https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz | tar -xz -C /tmp/download && \ mv /tmp/download/docker/docker /usr/bin/ && \ rm -rf /tmp/download && \ rm -rf ~/.cache/ diff --git a/compose/galaxy-htcondor-executor/Dockerfile b/compose/galaxy-htcondor-executor/Dockerfile index 890ce338c..46f4a6e27 100644 --- a/compose/galaxy-htcondor-executor/Dockerfile +++ b/compose/galaxy-htcondor-executor/Dockerfile @@ -12,7 +12,7 @@ LANG=en_US.UTF-8 ADD startup.sh /usr/bin/startup.sh RUN mkdir -p /tmp/download && \ - wget --no-check-certificate -qO - https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz | tar -xz -C /tmp/download && \ + wget --no-check-certificate -qO - https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz | tar -xz -C /tmp/download && \ mv /tmp/download/docker/docker /usr/bin/ && \ rm -rf /tmp/download && \ rm -rf ~/.cache/ && \ diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 9b064cec8..43064c59a 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -31,7 +31,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ mkdir /var/log/supervisor && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ mkdir -p /tmp/download && \ - wget -qO - https://get.docker.com/builds/Linux/x86_64/docker-17.05.0-ce.tgz | tar -xz -C /tmp/download && \ + wget -qO - https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz | tar -xz -C /tmp/download && \ mv /tmp/download/docker/docker /usr/bin/ && \ rm -rf /tmp/download && \ rm -rf ~/.cache/ From c221a4fdf7d8cf1fde7050b80a3a76fa06d43469 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Tue, 10 Jul 2018 11:11:13 +0100 Subject: [PATCH 043/116] Adds script for building containers for k8s for 18.05 --- compose/build-community-images-for-k8s.sh | 245 ++++++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100755 compose/build-community-images-for-k8s.sh diff --git a/compose/build-community-images-for-k8s.sh b/compose/build-community-images-for-k8s.sh new file mode 100755 index 000000000..7ee40f1eb --- /dev/null +++ b/compose/build-community-images-for-k8s.sh @@ -0,0 +1,245 @@ +#!/bin/bash +# vi: fdm=marker + +set -e + +### Constants ### + +SCRIPT_NAME="${0##*/}" +TRUE='true' +FALSE='false' +DFT_DOCKER_PUSH_ENABLED=$TRUE + +### Functions ### + +function log() { + echo -e "$@" >&2 +} + +function error { + + local msg=$1 + local code=$2 + [[ -z $code ]] && code=1 + + log "[ERROR] $msg" + + exit $code +} + + +function debug { + + local msg=$1 + local level=$2 + [[ -n $level ]] || level=1 + + [[ $DEBUG -lt $level ]] || log "[DEBUG] $msg" +} + +function warning { + + local msg=$1 + + log "[WARNING] ##### $msg #####" +} + +function print_help { + ( + echo "Usage: $SCRIPT_NAME [options]" + echo + echo "Build Galaxy docker images for use with k8s." + echo + echo "Options:" + echo " -g, --debug Debug mode. [false]" + echo " -h, --help Print this help message." + echo " -p, --push Push docker images. You can also set environment variable DOCKER_PUSH_ENABLED to \"true\". [true]" + echo " +p, --no-push Do not push dockers. You can also set environment variable DOCKER_PUSH_ENABLED to \"false\"." + echo " --push-intermediate Also push intermediate images [false]" + echo " --no-cache Tell Docker not to use cached images. [false]" + echo " --init-tag Set the tag for the Galaxy Init Flavour container image." + echo " --postgres-tag Set the tag for the Galaxy Postgres container image." + echo " --proftpd-tag Set the tag for the Galaxy Proftpd container image." + echo " --web-tag Set the tag for the Galaxy Web k8s container image." + echo " -u, --container-user Set the container user. You can also set the environment variable CONTAINER_USER. [pcm32]" + echo " -r, --container-registry Set the container registry. You can also set the environment variable CONTAINER_REGISTRY." + echo + ) >&2 +} + +function read_args { + + local args="$*" # save arguments for debugging purpose + + # Read options + while [[ $# > 0 ]] ; do + shift_count=1 + case $1 in + -g|--debug) DEBUG=$((DEBUG + 1)) ;; + -h|--help) print_help ; exit 0 ;; + -p|--push) DOCKER_PUSH_ENABLED=$TRUE ;; + +p|--no-push) DOCKER_PUSH_ENABLED=$FALSE ;; + --push-intermediate) PUSH_INTERMEDIATE_IMAGES=$TRUE ;; + --no-cache) NO_CACHE="--no-cache" ;; + --postgres-tag) OVERRIDE_POSTGRES_TAG="$2" ; shift_count=2 ;; + --proftpd-tag) OVERRIDE_PROFTPD_TAG="$2" ; shift_count=2 ;; + --init-tag) OVERRIDE_GALAXY_INIT_PHENO_FLAVOURED_TAG="$2" ; shift_count=2 ;; + --web-tag) OVERRIDE_GALAXY_WEB_K8S_TAG="$2" ; shift_count=2 ;; + -u|--container-user) CONTAINER_USER="$2" ; shift_count=2 ;; + -r|--container-registry) CONTAINER_REGISTRY="$2" ; shift_count=2 ;; + *) error "Illegal option $1." 98 ;; + esac + shift $shift_count + done + + # Debug messages + debug "Command line arguments: $args" + debug "Argument DEBUG=$DEBUG" + debug "Argument DOCKER_PUSH_ENABLED=$DOCKER_PUSH_ENABLED" + debug "Argument DOCKER_USER=$DOCKER_USER" + debug "Argument OVERRIDE_GALAXY_INIT_PHENO_FLAVOURED_TAG=$OVERRIDE_GALAXY_INIT_PHENO_FLAVOURED_TAG" + debug "Argument OVERRIDE_GALAXY_WEB_K8S_TAG=$OVERRIDE_GALAXY_WEB_K8S_TAG" + debug "Argument OVERRIDE_POSTGRES_TAG=$OVERRIDE_POSTGRES_TAG" + debug "Argument OVERRIDE_PROFTPD_TAG=$OVERRIDE_PROFTPD_TAG" + debug "shift_count=$shift_count" +} + +### MAIN ### + +read_args "$@" + +DOCKER_PUSH_ENABLED=${DOCKER_PUSH_ENABLED:-$DFT_DOCKER_PUSH_ENABLED} + +DOCKER_REPO=${CONTAINER_REGISTRY:-} +DOCKER_USER=${CONTAINER_USER:-pcm32} + +ANSIBLE_REPO=${ANSIBLE_REPO:-galaxyproject/ansible-galaxy-extras} +ANSIBLE_RELEASE=${ANSIBLE_RELEASE:-master} + +GALAXY_VERSION=18.05 + +GALAXY_BASE_FROM_TO_REPLACE=${GALAXY_BASE_FROM_TO_REPLACE:-quay.io/bgruening/galaxy-base:$GALAXY_VERSION} + +GALAXY_RELEASE=${GALAXY_RELEASE:-release_$GALAXY_VERSION} +GALAXY_REPO=${GALAXY_REPO:-galaxyproject/galaxy} + +GALAXY_VER_FOR_POSTGRES=$GALAXY_VERSION +# Uncomment to push intermediate images, otherwise only the images needed for the helm chart are pushed. +#PUSH_INTERMEDIATE_IMAGES=yes + +# Set tags +TAG=v$GALAXY_VERSION + +if [[ -n ${CONTAINER_TAG_PREFIX:-} && -n ${BUILD_NUMBER:-} ]]; then + TAG=${CONTAINER_TAG_PREFIX}.${BUILD_NUMBER} +fi + +GALAXY_BASE_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-base:$TAG +GALAXY_INIT_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-init:$TAG +GALAXY_WEB_K8S_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-web-k8s:$TAG + +if [[ -n "${DOCKER_REPO}" ]]; then + # Append slash, avoiding double slash + DOCKER_REPO="${DOCKER_REPO%/}/" +fi + +if [[ -n "${OVERRIDE_GALAXY_WEB_K8S_TAG:-}" ]]; then + GALAXY_WEB_K8S_TAG="${OVERRIDE_GALAXY_WEB_K8S_TAG}" +else + GALAXY_WEB_K8S_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-web-k8s:$TAG +fi + +if [[ -n "${OVERRIDE_POSTGRES_TAG:-}" ]]; then + POSTGRES_TAG="${OVERRIDE_POSTGRES_TAG}" +else + PG_Dockerfile="galaxy-postgres/Dockerfile" + + [[ -f "${PG_Dockerfile}" ]] || error "The galaxy-postgres Dockerfile is missing under galaxy-Postgres." 99 + POSTGRES_VERSION=$(grep FROM "${PG_Dockerfile}" | awk -F":" '{ print $2 }') + + POSTGRES_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-postgres:$POSTGRES_VERSION"_for_"$GALAXY_VER_FOR_POSTGRES +fi + +if [[ -n "${OVERRIDE_PROFTPD_TAG:-}" ]]; then + PROFTPD_TAG="${OVERRIDE_PROFTPD_TAG}" +else + PROFTPD_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-proftpd:for_galaxy_v$GALAXY_VER_FOR_POSTGRES +fi + +### do work + +if [ -n $ANSIBLE_REPO ] + then + log "Making custom galaxy-base:$TAG from $ANSIBLE_REPO at $ANSIBLE_RELEASE" + docker build $NO_CACHE --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t $GALAXY_BASE_TAG galaxy-base/ + if [[ ! -z ${PUSH_INTERMEDIATE_IMAGES+x} ]]; + then + log "Pushing intermediate image $DOCKER_REPO$DOCKER_USER/galaxy-base:$TAG" + docker push $GALAXY_BASE_TAG + fi +fi + + +if [ -n $GALAXY_REPO ] + then + log "Making custom galaxy-init:$TAG from $GALAXY_REPO at $GALAXY_RELEASE" + DOCKERFILE_INIT_1=Dockerfile + if [ -n $ANSIBLE_REPO ] + then + sed s+$GALAXY_BASE_FROM_TO_REPLACE+$GALAXY_BASE_TAG+ galaxy-init/Dockerfile > galaxy-init/Dockerfile_init + FROM=`grep ^FROM galaxy-init/Dockerfile_init | awk '{ print $2 }'` + log "Using FROM $FROM for galaxy init" + DOCKERFILE_INIT_1=Dockerfile_init + fi + docker build $NO_CACHE --build-arg GALAXY_REPO=$GALAXY_REPO --build-arg GALAXY_RELEASE=$GALAXY_RELEASE -t $GALAXY_INIT_TAG -f galaxy-init/$DOCKERFILE_INIT_1 galaxy-init/ + if [[ "${DOCKER_PUSH_ENABLED:-}" = "true" ]]; then + log "Pushing image $GALAXY_INIT_TAG" + docker push $GALAXY_INIT_TAG + fi +fi + +# GALAXY_INIT_FLAVOURED_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-init-flavoured:$TAG +# +# DOCKERFILE_INIT_FLAVOUR=Dockerfile_init +# if [ -n $GALAXY_REPO ] +# then +# echo "Making custom galaxy-init-pheno-flavoured:$TAG starting from galaxy-init-pheno:$TAG" +# sed s+pcm32/galaxy-init$+$GALAXY_INIT_PHENO_TAG+ Dockerfile_init > Dockerfile_init_tagged +# DOCKERFILE_INIT_FLAVOUR=Dockerfile_init_tagged +# fi +# docker build $NO_CACHE -t $GALAXY_INIT_FLAVOURED_TAG -f $DOCKERFILE_INIT_FLAVOUR . +# docker push $GALAXY_INIT_PHENO_FLAVOURED_TAG + + + +DOCKERFILE_WEB=Dockerfile +if [ -n $GALAXY_REPO ] +then + echo "Making custom galaxy-web-k8s:$TAG from $GALAXY_REPO at $GALAXY_RELEASE" + sed s+$GALAXY_BASE_FROM_TO_REPLACE+$GALAXY_BASE_TAG+ galaxy-web/Dockerfile > galaxy-web/Dockerfile_web + DOCKERFILE_WEB=Dockerfile_web +fi +docker build $NO_CACHE --build-arg GALAXY_ANSIBLE_TAGS=supervisor,startup,scripts,nginx,k8,k8s -t $GALAXY_WEB_K8S_TAG -f galaxy-web/$DOCKERFILE_WEB galaxy-web/ +if [[ "${DOCKER_PUSH_ENABLED:-}" = "true" ]]; then + docker push $GALAXY_WEB_K8S_TAG +fi + +# Build postgres +docker build -t $POSTGRES_TAG -f galaxy-postgres/Dockerfile galaxy-postgres/ +if [[ "${DOCKER_PUSH_ENABLED:-}" = "true" ]]; then + docker push $POSTGRES_TAG +fi + +# Build proftpd +docker build -t $PROFTPD_TAG -f galaxy-proftpd/Dockerfile galaxy-proftpd/ +if [[ "${DOCKER_PUSH_ENABLED:-}" = "true" ]]; then + docker push $PROFTPD_TAG +fi + +log "Relevant containers:" +log "Web: $GALAXY_WEB_K8S_TAG" +log "Init: $GALAXY_INIT_TAG" +log "Postgres: $POSTGRES_TAG" +log "Proftpd: $PROFTPD_TAG" + +log "Now build your Galaxy own galaxy init container starting from $GALAXY_INIT_TAG" From df63c1aee6f5726d4d9df83e69339c4e5efacd92 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Fri, 27 Jul 2018 15:29:48 +0100 Subject: [PATCH 044/116] Makes script more general, includes condor and graphana, makes k8s optional. --- ...r-k8s.sh => build-orchestration-images.sh} | 113 +++++++++++------- 1 file changed, 72 insertions(+), 41 deletions(-) rename compose/{build-community-images-for-k8s.sh => build-orchestration-images.sh} (68%) diff --git a/compose/build-community-images-for-k8s.sh b/compose/build-orchestration-images.sh similarity index 68% rename from compose/build-community-images-for-k8s.sh rename to compose/build-orchestration-images.sh index 7ee40f1eb..5a73844ec 100755 --- a/compose/build-community-images-for-k8s.sh +++ b/compose/build-orchestration-images.sh @@ -17,11 +17,9 @@ function log() { } function error { - local msg=$1 local code=$2 [[ -z $code ]] && code=1 - log "[ERROR] $msg" exit $code @@ -29,7 +27,6 @@ function error { function debug { - local msg=$1 local level=$2 [[ -n $level ]] || level=1 @@ -38,7 +35,6 @@ function debug { } function warning { - local msg=$1 log "[WARNING] ##### $msg #####" @@ -48,7 +44,7 @@ function print_help { ( echo "Usage: $SCRIPT_NAME [options]" echo - echo "Build Galaxy docker images for use with k8s." + echo "Build Galaxy docker compose images for different purposes." echo echo "Options:" echo " -g, --debug Debug mode. [false]" @@ -61,7 +57,10 @@ function print_help { echo " --postgres-tag Set the tag for the Galaxy Postgres container image." echo " --proftpd-tag Set the tag for the Galaxy Proftpd container image." echo " --web-tag Set the tag for the Galaxy Web k8s container image." - echo " -u, --container-user Set the container user. You can also set the environment variable CONTAINER_USER. [pcm32]" + echo " --k8s Set settings for Kubernetes usage." + echo " --graphana Build graphana container." + echo " --condor Build condor containers." + echo " -u, --container-user Set the container user. You can also set the environment variable CONTAINER_USER." echo " -r, --container-registry Set the container registry. You can also set the environment variable CONTAINER_REGISTRY." echo ) >&2 @@ -79,12 +78,15 @@ function read_args { -h|--help) print_help ; exit 0 ;; -p|--push) DOCKER_PUSH_ENABLED=$TRUE ;; +p|--no-push) DOCKER_PUSH_ENABLED=$FALSE ;; + --k8s) BUILD_FOR_K8S=$TRUE ;; + --condor) BUILD_FOR_CONDOR=$TRUE ;; + --graphana) BUILD_FOR_GRAPHANA=$TRUE ;; --push-intermediate) PUSH_INTERMEDIATE_IMAGES=$TRUE ;; --no-cache) NO_CACHE="--no-cache" ;; --postgres-tag) OVERRIDE_POSTGRES_TAG="$2" ; shift_count=2 ;; --proftpd-tag) OVERRIDE_PROFTPD_TAG="$2" ; shift_count=2 ;; --init-tag) OVERRIDE_GALAXY_INIT_PHENO_FLAVOURED_TAG="$2" ; shift_count=2 ;; - --web-tag) OVERRIDE_GALAXY_WEB_K8S_TAG="$2" ; shift_count=2 ;; + --web-tag) OVERRIDE_GALAXY_WEB_TAG="$2" ; shift_count=2 ;; -u|--container-user) CONTAINER_USER="$2" ; shift_count=2 ;; -r|--container-registry) CONTAINER_REGISTRY="$2" ; shift_count=2 ;; *) error "Illegal option $1." 98 ;; @@ -97,14 +99,16 @@ function read_args { debug "Argument DEBUG=$DEBUG" debug "Argument DOCKER_PUSH_ENABLED=$DOCKER_PUSH_ENABLED" debug "Argument DOCKER_USER=$DOCKER_USER" - debug "Argument OVERRIDE_GALAXY_INIT_PHENO_FLAVOURED_TAG=$OVERRIDE_GALAXY_INIT_PHENO_FLAVOURED_TAG" - debug "Argument OVERRIDE_GALAXY_WEB_K8S_TAG=$OVERRIDE_GALAXY_WEB_K8S_TAG" + debug "Argument OVERRIDE_GALAXY_WEB_TAG=$OVERRIDE_GALAXY_WEB_TAG" debug "Argument OVERRIDE_POSTGRES_TAG=$OVERRIDE_POSTGRES_TAG" debug "Argument OVERRIDE_PROFTPD_TAG=$OVERRIDE_PROFTPD_TAG" debug "shift_count=$shift_count" } ### MAIN ### +BUILD_FOR_K8S=$FALSE +BUILD_FOR_CONDOR=$FALSE +BUILD_FOR_GRAPHANA=$FALSE read_args "$@" @@ -116,9 +120,10 @@ DOCKER_USER=${CONTAINER_USER:-pcm32} ANSIBLE_REPO=${ANSIBLE_REPO:-galaxyproject/ansible-galaxy-extras} ANSIBLE_RELEASE=${ANSIBLE_RELEASE:-master} -GALAXY_VERSION=18.05 +GALAXY_VERSION=${GALAXY_VERSION:-18.05} GALAXY_BASE_FROM_TO_REPLACE=${GALAXY_BASE_FROM_TO_REPLACE:-quay.io/bgruening/galaxy-base:$GALAXY_VERSION} +CONDOR_BASE_FROM_TO_REPLACE=quay.io/bgruening/galaxy-htcondor-base:$GALAXY_VERSION GALAXY_RELEASE=${GALAXY_RELEASE:-release_$GALAXY_VERSION} GALAXY_REPO=${GALAXY_REPO:-galaxyproject/galaxy} @@ -128,26 +133,21 @@ GALAXY_VER_FOR_POSTGRES=$GALAXY_VERSION #PUSH_INTERMEDIATE_IMAGES=yes # Set tags -TAG=v$GALAXY_VERSION +TAG=${GALAXY_TAG:-v$GALAXY_VERSION} +# TODO This is PhenoMeNal Jenkins specific, should be removed at some point. if [[ -n ${CONTAINER_TAG_PREFIX:-} && -n ${BUILD_NUMBER:-} ]]; then TAG=${CONTAINER_TAG_PREFIX}.${BUILD_NUMBER} fi -GALAXY_BASE_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-base:$TAG -GALAXY_INIT_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-init:$TAG -GALAXY_WEB_K8S_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-web-k8s:$TAG - if [[ -n "${DOCKER_REPO}" ]]; then # Append slash, avoiding double slash DOCKER_REPO="${DOCKER_REPO%/}/" fi -if [[ -n "${OVERRIDE_GALAXY_WEB_K8S_TAG:-}" ]]; then - GALAXY_WEB_K8S_TAG="${OVERRIDE_GALAXY_WEB_K8S_TAG}" -else - GALAXY_WEB_K8S_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-web-k8s:$TAG -fi +GALAXY_BASE_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-base:$TAG +GALAXY_INIT_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-init:$TAG +GALAXY_WEB_TAG=${OVERRIDE_GALAXY_WEB_TAG:-$DOCKER_REPO$DOCKER_USER/galaxy-web:$TAG} if [[ -n "${OVERRIDE_POSTGRES_TAG:-}" ]]; then POSTGRES_TAG="${OVERRIDE_POSTGRES_TAG}" @@ -166,6 +166,10 @@ else PROFTPD_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-proftpd:for_galaxy_v$GALAXY_VER_FOR_POSTGRES fi +CONDOR_BASE_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor-base:$TAG +CONDOR_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor:$TAG +CONDOR_EXEC_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor-executor:$TAG +GRAPHANA_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-graphana:$TAG ### do work if [ -n $ANSIBLE_REPO ] @@ -198,48 +202,75 @@ if [ -n $GALAXY_REPO ] fi fi -# GALAXY_INIT_FLAVOURED_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-init-flavoured:$TAG -# -# DOCKERFILE_INIT_FLAVOUR=Dockerfile_init -# if [ -n $GALAXY_REPO ] -# then -# echo "Making custom galaxy-init-pheno-flavoured:$TAG starting from galaxy-init-pheno:$TAG" -# sed s+pcm32/galaxy-init$+$GALAXY_INIT_PHENO_TAG+ Dockerfile_init > Dockerfile_init_tagged -# DOCKERFILE_INIT_FLAVOUR=Dockerfile_init_tagged -# fi -# docker build $NO_CACHE -t $GALAXY_INIT_FLAVOURED_TAG -f $DOCKERFILE_INIT_FLAVOUR . -# docker push $GALAXY_INIT_PHENO_FLAVOURED_TAG - - DOCKERFILE_WEB=Dockerfile if [ -n $GALAXY_REPO ] then - echo "Making custom galaxy-web-k8s:$TAG from $GALAXY_REPO at $GALAXY_RELEASE" + echo "Making custom galaxy-web:$TAG from $GALAXY_REPO at $GALAXY_RELEASE" sed s+$GALAXY_BASE_FROM_TO_REPLACE+$GALAXY_BASE_TAG+ galaxy-web/Dockerfile > galaxy-web/Dockerfile_web DOCKERFILE_WEB=Dockerfile_web fi -docker build $NO_CACHE --build-arg GALAXY_ANSIBLE_TAGS=supervisor,startup,scripts,nginx,k8,k8s -t $GALAXY_WEB_K8S_TAG -f galaxy-web/$DOCKERFILE_WEB galaxy-web/ -if [[ "${DOCKER_PUSH_ENABLED:-}" = "true" ]]; then - docker push $GALAXY_WEB_K8S_TAG +K8S_ANSIBLE_TAGS="" +if $BUILD_FOR_K8S; + K8S_ANSIBLE_TAGS=,k8,k8s +fi +docker build $NO_CACHE --build-arg GALAXY_ANSIBLE_TAGS=supervisor,startup,scripts,nginx$K8S_ANSIBLE_TAGS -t $GALAXY_WEB_TAG -f galaxy-web/$DOCKERFILE_WEB galaxy-web/ +if $DOCKER_PUSH_ENABLED; then + docker push $GALAXY_WEB_TAG fi # Build postgres docker build -t $POSTGRES_TAG -f galaxy-postgres/Dockerfile galaxy-postgres/ -if [[ "${DOCKER_PUSH_ENABLED:-}" = "true" ]]; then +if $DOCKER_PUSH_ENABLED; then docker push $POSTGRES_TAG fi # Build proftpd docker build -t $PROFTPD_TAG -f galaxy-proftpd/Dockerfile galaxy-proftpd/ -if [[ "${DOCKER_PUSH_ENABLED:-}" = "true" ]]; then +if $DOCKER_PUSH_ENABLED; then docker push $PROFTPD_TAG fi +# Build condor +if $BUILD_FOR_CONDOR; then + docker build $NO_CACHE -t $CONDOR_BASE_TAG galaxy-htcondor-base + sed s+$CONDOR_BASE_FROM_TO_REPLACE+$CONDOR_BASE_TAG+ galaxy-htcondor/Dockerfile > galaxy-htcondor/Dockerfile_condor + FROM=`grep ^FROM galaxy-htcondor/Dockerfile_condor | awk '{ print $2 }'` + log "Using FROM $FROM for condor" + docker build $NO_CACHE -t $CONDOR_TAG -f galaxy-htcondor/Docker_condor galaxy-htcondor/ + sed s+$CONDOR_BASE_FROM_TO_REPLACE+$CONDOR_BASE_TAG+ galaxy-htcondor-executor/Dockerfile > galaxy-htcondor-executor/Dockerfile_condor + FROM=`grep ^FROM galaxy-htcondor-executor/Dockerfile_condor | awk '{ print $2 }'` + log "Using FROM $FROM for condor-executor" + docker build $NO_CACHE -t $CONDOR_EXEC_TAG -f galaxy-htcondor-executor/Docker_condor galaxy-htcondor-executor/ + if $DOCKER_PUSH_ENABLED; then + docker push $CONDOR_TAG + docker push $CONDOR_EXEC_TAG + fi +fi + +# Build for graphana +if $BUILD_FOR_GRAPHANA; then + docker build -t $GRAPHANA_TAG ./galaxy-grafana +fi + log "Relevant containers:" -log "Web: $GALAXY_WEB_K8S_TAG" +log "Web: $GALAXY_WEB_TAG" log "Init: $GALAXY_INIT_TAG" log "Postgres: $POSTGRES_TAG" log "Proftpd: $PROFTPD_TAG" +if $BUILD_FOR_CONDOR; then + log "Condor: $CONDOR_TAG" + log "Condor-exec: $CONDOR_EXEC_TAG" +fi +if $BUILD_FOR_GRAPHANA; then + log "Graphana: $GRAPHANA_TAG" +fi -log "Now build your Galaxy own galaxy init container starting from $GALAXY_INIT_TAG" +log "Now build your own Galaxy init container starting FROM $GALAXY_INIT_TAG to add you own flavour, tools, workflows, etc." +if $BUILD_FOR_K8S; + log "" + log "For k8s: Once you have built your own init container use it within the galaxy-stable Helm chart at https://github.com/galaxyproject/galaxy-kubernetes together with:" + log " - Web: $GALAXY_WEB_TAG" + log " - Postgres: $POSTGRES_TAG" + log " - Proftpd: $PROFTPD_TAG" +fi From 6692595d107eac2b49fa28b01448ddb64aa33851 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 08:49:21 +0100 Subject: [PATCH 045/116] Adjusts travis and docker compose to work with new building script --- .travis.yml | 15 ++++++++++----- compose/docker-compose.yml | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index b263b1a4e..6da011628 100644 --- a/.travis.yml +++ b/.travis.yml @@ -120,7 +120,12 @@ before_install: INSTALL_REPO_ARG="--galaxy-url http://localhost:80" SAMPLE_TOOLS=/export/config/sample_tool_list.yaml cd "$WORKING_DIR" - ./buildlocal.sh + # For build script + export CONTAINER_REGISTRY=quay.io/ + export CONTAINER_USER=bgruening + ./build-orchestration-images.sh --no-push > image-build.log + export TAG_POSTGRES=$(cat image-build.log | grep '^Postgres:' | awk -F':' '{ print $3 }' ) + export TAG_PROFTPD=$(cat image-build.log | grep '^Proftpd:' | awk -F':' '{ print $3 }' ) export COMPOSE_PROJECT_NAME=galaxy_compose docker-compose up -d @@ -138,7 +143,7 @@ before_install: docker-compose stop galaxy-slurm sleep 30 fi - + if [ "${COMPOSE_SLURM}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] then # turn down the htcondor services @@ -150,7 +155,7 @@ before_install: if [ "${COMPOSE_SLURM_SINGULARITY}" ] then # docker-compose is already started and has pre-populated the /export dir - # we now turn it down again and copy in an example tool with tool_conf.xml and + # we now turn it down again and copy in an example tool with tool_conf.xml and # a test singularity image. If we copy this from the beginning, the magic Docker Galax startup # script will not work as it detects something in /export/ docker-compose down @@ -174,7 +179,7 @@ before_install: parsec datasets show_dataset $OUTPUT_ID | jq .misc_info | grep singularity fi - + docker-compose logs --tail 50 # Define start functions docker_exec() { @@ -255,7 +260,7 @@ script: - date > time.txt && curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true # Test FTP Server get - curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD - + # Test CVMFS - docker_exec bash -c "service autofs start" - docker_exec bash -c "cvmfs_config chksetup" diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 8e32dd950..fa458df04 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -3,7 +3,7 @@ services: # proftpd container galaxy-proftpd: # build: galaxy-proftpd - image: quay.io/bgruening/galaxy-proftpd:${TAG:-latest} + image: quay.io/bgruening/galaxy-proftpd:${TAG_PROFTPD:-latest} env_file: .env environment: - proftpd_db_connection=galaxy@galaxy-postgres @@ -43,7 +43,7 @@ services: # If you are using the official postgres image, it needs to be populated by calling # docker-compose run galaxy install_db.sh # on first run - image: quay.io/bgruening/galaxy-postgres:${TAG:-latest} + image: quay.io/bgruening/galaxy-postgres:${TAG_POSTGRES:-latest} container_name: galaxy-postgres hostname: galaxy-postgres env_file: .env From 68fd56225322b86504a67e1feea2784d61151fe5 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 09:38:28 +0100 Subject: [PATCH 046/116] Improves handling of tags --- .travis.yml | 3 ++- compose/build-orchestration-images.sh | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6da011628..93b950c53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -123,9 +123,10 @@ before_install: # For build script export CONTAINER_REGISTRY=quay.io/ export CONTAINER_USER=bgruening - ./build-orchestration-images.sh --no-push > image-build.log + ./build-orchestration-images.sh --no-push 2> image-build.log export TAG_POSTGRES=$(cat image-build.log | grep '^Postgres:' | awk -F':' '{ print $3 }' ) export TAG_PROFTPD=$(cat image-build.log | grep '^Proftpd:' | awk -F':' '{ print $3 }' ) + export TAG=$(cat image-build.log | grep '^Web:' | awk -F':' '{ print $3 }' ) export COMPOSE_PROJECT_NAME=galaxy_compose docker-compose up -d diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index 5a73844ec..759a429e5 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -211,7 +211,7 @@ then DOCKERFILE_WEB=Dockerfile_web fi K8S_ANSIBLE_TAGS="" -if $BUILD_FOR_K8S; +if $BUILD_FOR_K8S; then K8S_ANSIBLE_TAGS=,k8,k8s fi docker build $NO_CACHE --build-arg GALAXY_ANSIBLE_TAGS=supervisor,startup,scripts,nginx$K8S_ANSIBLE_TAGS -t $GALAXY_WEB_TAG -f galaxy-web/$DOCKERFILE_WEB galaxy-web/ @@ -267,7 +267,7 @@ if $BUILD_FOR_GRAPHANA; then fi log "Now build your own Galaxy init container starting FROM $GALAXY_INIT_TAG to add you own flavour, tools, workflows, etc." -if $BUILD_FOR_K8S; +if $BUILD_FOR_K8S; then log "" log "For k8s: Once you have built your own init container use it within the galaxy-stable Helm chart at https://github.com/galaxyproject/galaxy-kubernetes together with:" log " - Web: $GALAXY_WEB_TAG" From c3d9e6e1e5fc10b003a1affbe51d2755af2fb99c Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 10:25:28 +0100 Subject: [PATCH 047/116] Slurm support --- compose/build-orchestration-images.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index 759a429e5..80e584314 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -60,6 +60,7 @@ function print_help { echo " --k8s Set settings for Kubernetes usage." echo " --graphana Build graphana container." echo " --condor Build condor containers." + echo " --slurm Build slurm containers." echo " -u, --container-user Set the container user. You can also set the environment variable CONTAINER_USER." echo " -r, --container-registry Set the container registry. You can also set the environment variable CONTAINER_REGISTRY." echo @@ -80,6 +81,7 @@ function read_args { +p|--no-push) DOCKER_PUSH_ENABLED=$FALSE ;; --k8s) BUILD_FOR_K8S=$TRUE ;; --condor) BUILD_FOR_CONDOR=$TRUE ;; + --slurm) BUILD_FOR_SLURM=$TRUE ;; --graphana) BUILD_FOR_GRAPHANA=$TRUE ;; --push-intermediate) PUSH_INTERMEDIATE_IMAGES=$TRUE ;; --no-cache) NO_CACHE="--no-cache" ;; @@ -109,6 +111,7 @@ function read_args { BUILD_FOR_K8S=$FALSE BUILD_FOR_CONDOR=$FALSE BUILD_FOR_GRAPHANA=$FALSE +BUILD_FOR_SLURM=$FALSE read_args "$@" @@ -170,6 +173,7 @@ CONDOR_BASE_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor-base:$TAG CONDOR_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor:$TAG CONDOR_EXEC_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor-executor:$TAG GRAPHANA_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-graphana:$TAG +SLURM_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-slurm:$TAG ### do work if [ -n $ANSIBLE_REPO ] @@ -248,6 +252,11 @@ if $BUILD_FOR_CONDOR; then fi fi +# Build for slurm +if $BUILD_FOR_SLURM; then + docker build -t $SLURM_TAG ./galaxy-slurm +fi + # Build for graphana if $BUILD_FOR_GRAPHANA; then docker build -t $GRAPHANA_TAG ./galaxy-grafana @@ -262,6 +271,8 @@ if $BUILD_FOR_CONDOR; then log "Condor: $CONDOR_TAG" log "Condor-exec: $CONDOR_EXEC_TAG" fi +if $BUILD_FOR_SLURM; then + log "Slurm: $SLURM_TAG" if $BUILD_FOR_GRAPHANA; then log "Graphana: $GRAPHANA_TAG" fi From c462849daa377636dac26bbba1b040decb1e89b1 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 10:25:54 +0100 Subject: [PATCH 048/116] Deprecation notice on buildlocal.sh --- compose/buildlocal.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh index ddfceb35f..0c6a47c82 100755 --- a/compose/buildlocal.sh +++ b/compose/buildlocal.sh @@ -1,6 +1,10 @@ #!/bin/bash set -x -e +echo "***************** Notice **************************************************" +echo "This script is deprecated, please use build-orchestration-images.sh instead." +echo "***************** Notice **************************************************" + ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras ANSIBLE_RELEASE=18.09 @@ -35,3 +39,7 @@ docker build -t quay.io/bgruening/galaxy-htcondor$TAG ./galaxy-htcondor docker build -t quay.io/bgruening/galaxy-htcondor-executor$TAG ./galaxy-htcondor-executor docker build -t quay.io/bgruening/galaxy-grafana$TAG ./galaxy-grafana + +echo "***************** Notice **************************************************" +echo "This script is deprecated, please use build-orchestration-images.sh instead." +echo "***************** Notice **************************************************" From cce96a1c149a5cce42bcaae467fbde98f09ddd53 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 10:26:31 +0100 Subject: [PATCH 049/116] builds condor, k8s, slurm and graphana on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 93b950c53..fb4186c3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -123,7 +123,7 @@ before_install: # For build script export CONTAINER_REGISTRY=quay.io/ export CONTAINER_USER=bgruening - ./build-orchestration-images.sh --no-push 2> image-build.log + ./build-orchestration-images.sh --no-push --condor --graphana --slurm --k8s 2> image-build.log export TAG_POSTGRES=$(cat image-build.log | grep '^Postgres:' | awk -F':' '{ print $3 }' ) export TAG_PROFTPD=$(cat image-build.log | grep '^Proftpd:' | awk -F':' '{ print $3 }' ) export TAG=$(cat image-build.log | grep '^Web:' | awk -F':' '{ print $3 }' ) From 9459bece483a3641926f8b14af944ee023742cc4 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 11:10:53 +0100 Subject: [PATCH 050/116] shows errors of container building phase --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fb4186c3f..0a303b837 100644 --- a/.travis.yml +++ b/.travis.yml @@ -124,6 +124,7 @@ before_install: export CONTAINER_REGISTRY=quay.io/ export CONTAINER_USER=bgruening ./build-orchestration-images.sh --no-push --condor --graphana --slurm --k8s 2> image-build.log + cat image-build.log export TAG_POSTGRES=$(cat image-build.log | grep '^Postgres:' | awk -F':' '{ print $3 }' ) export TAG_PROFTPD=$(cat image-build.log | grep '^Proftpd:' | awk -F':' '{ print $3 }' ) export TAG=$(cat image-build.log | grep '^Web:' | awk -F':' '{ print $3 }' ) From 8f850a624b403a840ce96e343ba10402b6a77566 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 11:20:21 +0100 Subject: [PATCH 051/116] Changes tag passing strategy --- .travis.yml | 7 ++----- compose/build-orchestration-images.sh | 6 ++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0a303b837..cc28f08fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -123,11 +123,8 @@ before_install: # For build script export CONTAINER_REGISTRY=quay.io/ export CONTAINER_USER=bgruening - ./build-orchestration-images.sh --no-push --condor --graphana --slurm --k8s 2> image-build.log - cat image-build.log - export TAG_POSTGRES=$(cat image-build.log | grep '^Postgres:' | awk -F':' '{ print $3 }' ) - export TAG_PROFTPD=$(cat image-build.log | grep '^Proftpd:' | awk -F':' '{ print $3 }' ) - export TAG=$(cat image-build.log | grep '^Web:' | awk -F':' '{ print $3 }' ) + ./build-orchestration-images.sh --no-push --condor --graphana --slurm --k8s + source ./tags-for-compose-to-source.sh export COMPOSE_PROJECT_NAME=galaxy_compose docker-compose up -d diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index 80e584314..67d227b49 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -285,3 +285,9 @@ if $BUILD_FOR_K8S; then log " - Postgres: $POSTGRES_TAG" log " - Proftpd: $PROFTPD_TAG" fi + +cat > ./tags-for-compose-to-source.sh < Date: Sat, 28 Jul 2018 11:35:54 +0100 Subject: [PATCH 052/116] Fixes Dockerfile redirection typo --- compose/build-orchestration-images.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index 67d227b49..594bb9741 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -241,11 +241,11 @@ if $BUILD_FOR_CONDOR; then sed s+$CONDOR_BASE_FROM_TO_REPLACE+$CONDOR_BASE_TAG+ galaxy-htcondor/Dockerfile > galaxy-htcondor/Dockerfile_condor FROM=`grep ^FROM galaxy-htcondor/Dockerfile_condor | awk '{ print $2 }'` log "Using FROM $FROM for condor" - docker build $NO_CACHE -t $CONDOR_TAG -f galaxy-htcondor/Docker_condor galaxy-htcondor/ + docker build $NO_CACHE -t $CONDOR_TAG -f galaxy-htcondor/Dockerfile_condor galaxy-htcondor/ sed s+$CONDOR_BASE_FROM_TO_REPLACE+$CONDOR_BASE_TAG+ galaxy-htcondor-executor/Dockerfile > galaxy-htcondor-executor/Dockerfile_condor FROM=`grep ^FROM galaxy-htcondor-executor/Dockerfile_condor | awk '{ print $2 }'` log "Using FROM $FROM for condor-executor" - docker build $NO_CACHE -t $CONDOR_EXEC_TAG -f galaxy-htcondor-executor/Docker_condor galaxy-htcondor-executor/ + docker build $NO_CACHE -t $CONDOR_EXEC_TAG -f galaxy-htcondor-executor/Dockerfile_condor galaxy-htcondor-executor/ if $DOCKER_PUSH_ENABLED; then docker push $CONDOR_TAG docker push $CONDOR_EXEC_TAG From a02b62b39ae9cef3869bb2930d178073fafe37b1 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 11:55:05 +0100 Subject: [PATCH 053/116] Removes additional double quote --- compose/build-orchestration-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index 594bb9741..f29ef25ad 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -289,5 +289,5 @@ fi cat > ./tags-for-compose-to-source.sh < Date: Sat, 28 Jul 2018 19:18:06 +0100 Subject: [PATCH 054/116] Changes how delim is used with cat --- compose/build-orchestration-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index f29ef25ad..279c8f54b 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -286,7 +286,7 @@ if $BUILD_FOR_K8S; then log " - Proftpd: $PROFTPD_TAG" fi -cat > ./tags-for-compose-to-source.sh < ./tags-for-compose-to-source.sh export TAG=$GALAXY_WEB_TAG export TAG_POSTGRES=$POSTGRES_TAG export TAG_PROFTPD=$PROFTPD_TAG From 813752951ea72bf52614a3ba541bee682d45b6a4 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 19:35:10 +0100 Subject: [PATCH 055/116] Avoids delims and simple echo to file for sourcing --- compose/build-orchestration-images.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index 279c8f54b..c09f922ef 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -286,8 +286,6 @@ if $BUILD_FOR_K8S; then log " - Proftpd: $PROFTPD_TAG" fi -cat << DELIM > ./tags-for-compose-to-source.sh -export TAG=$GALAXY_WEB_TAG -export TAG_POSTGRES=$POSTGRES_TAG -export TAG_PROFTPD=$PROFTPD_TAG -DELIM +echo "export TAG=$GALAXY_WEB_TAG" > tags-for-compose-to-source.sh +echo "export TAG_POSTGRES=$POSTGRES_TAG" >> tags-for-compose-to-source.sh +echo "export TAG_PROFTPD=$PROFTPD_TAG" >> tags-for-compose-to-source.sh From 524a8310a38287c9087ac4a7e4d16bc632e28604 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 19:52:19 +0100 Subject: [PATCH 056/116] Problem was missing if closure. --- compose/build-orchestration-images.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index c09f922ef..f6adbca5e 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -273,6 +273,7 @@ if $BUILD_FOR_CONDOR; then fi if $BUILD_FOR_SLURM; then log "Slurm: $SLURM_TAG" +fi if $BUILD_FOR_GRAPHANA; then log "Graphana: $GRAPHANA_TAG" fi From c56dfac060e3bf67179ccfcf3ceca40c8d9f840c Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 23:25:17 +0100 Subject: [PATCH 057/116] Only last tags should get exported, not entire URIs for travis. --- compose/build-orchestration-images.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index f6adbca5e..f9fa2344c 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -287,6 +287,6 @@ if $BUILD_FOR_K8S; then log " - Proftpd: $PROFTPD_TAG" fi -echo "export TAG=$GALAXY_WEB_TAG" > tags-for-compose-to-source.sh -echo "export TAG_POSTGRES=$POSTGRES_TAG" >> tags-for-compose-to-source.sh -echo "export TAG_PROFTPD=$PROFTPD_TAG" >> tags-for-compose-to-source.sh +echo "export TAG="$(echo $GALAXY_WEB_TAG | awk -F':' '{print $2}') > tags-for-compose-to-source.sh +echo "export TAG_POSTGRES="$(echo $POSTGRES_TAG | awk -F':' '{print $2}') >> tags-for-compose-to-source.sh +echo "export TAG_PROFTPD="$(echo $PROFTPD_TAG | awk -F':' '{print $2}') >> tags-for-compose-to-source.sh From fae3f44de9fccaf8d71496e6535ea886e065ebc8 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Sat, 28 Jul 2018 23:48:45 +0100 Subject: [PATCH 058/116] Typos for grafana --- .travis.yml | 2 +- compose/build-orchestration-images.sh | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc28f08fa..139c83b77 100644 --- a/.travis.yml +++ b/.travis.yml @@ -123,7 +123,7 @@ before_install: # For build script export CONTAINER_REGISTRY=quay.io/ export CONTAINER_USER=bgruening - ./build-orchestration-images.sh --no-push --condor --graphana --slurm --k8s + ./build-orchestration-images.sh --no-push --condor --grafana --slurm --k8s source ./tags-for-compose-to-source.sh export COMPOSE_PROJECT_NAME=galaxy_compose docker-compose up -d diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index f9fa2344c..ce4cf4cbf 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -58,7 +58,7 @@ function print_help { echo " --proftpd-tag Set the tag for the Galaxy Proftpd container image." echo " --web-tag Set the tag for the Galaxy Web k8s container image." echo " --k8s Set settings for Kubernetes usage." - echo " --graphana Build graphana container." + echo " --grafana Build grafana container." echo " --condor Build condor containers." echo " --slurm Build slurm containers." echo " -u, --container-user Set the container user. You can also set the environment variable CONTAINER_USER." @@ -82,7 +82,7 @@ function read_args { --k8s) BUILD_FOR_K8S=$TRUE ;; --condor) BUILD_FOR_CONDOR=$TRUE ;; --slurm) BUILD_FOR_SLURM=$TRUE ;; - --graphana) BUILD_FOR_GRAPHANA=$TRUE ;; + --grafana) BUILD_FOR_GRAFANA=$TRUE ;; --push-intermediate) PUSH_INTERMEDIATE_IMAGES=$TRUE ;; --no-cache) NO_CACHE="--no-cache" ;; --postgres-tag) OVERRIDE_POSTGRES_TAG="$2" ; shift_count=2 ;; @@ -110,7 +110,7 @@ function read_args { ### MAIN ### BUILD_FOR_K8S=$FALSE BUILD_FOR_CONDOR=$FALSE -BUILD_FOR_GRAPHANA=$FALSE +BUILD_FOR_GRAFANA=$FALSE BUILD_FOR_SLURM=$FALSE read_args "$@" @@ -172,7 +172,7 @@ fi CONDOR_BASE_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor-base:$TAG CONDOR_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor:$TAG CONDOR_EXEC_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor-executor:$TAG -GRAPHANA_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-graphana:$TAG +GRAFANA_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-grafana:$TAG SLURM_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-slurm:$TAG ### do work @@ -257,9 +257,9 @@ if $BUILD_FOR_SLURM; then docker build -t $SLURM_TAG ./galaxy-slurm fi -# Build for graphana -if $BUILD_FOR_GRAPHANA; then - docker build -t $GRAPHANA_TAG ./galaxy-grafana +# Build for grafana +if $BUILD_FOR_GRAFANA; then + docker build -t $GRAFANA_TAG ./galaxy-grafana fi log "Relevant containers:" @@ -274,8 +274,8 @@ fi if $BUILD_FOR_SLURM; then log "Slurm: $SLURM_TAG" fi -if $BUILD_FOR_GRAPHANA; then - log "Graphana: $GRAPHANA_TAG" +if $BUILD_FOR_GRAFANA; then + log "Grafana: $GRAFANA_TAG" fi log "Now build your own Galaxy init container starting FROM $GALAXY_INIT_TAG to add you own flavour, tools, workflows, etc." From 283aff29c0debe840616217d5e0d61eb60d925e5 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Mon, 29 Oct 2018 17:18:59 +0000 Subject: [PATCH 059/116] Try compose logs to check status of container before curl call --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 139c83b77..20b3dbb39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -247,6 +247,11 @@ script: cd $TRAVIS_BUILD_DIR/test/gridengine/ && bash test.sh && cd $WORKING_DIR fi # Test Web api + - | + if [ "${COMPOSE_CONDOR_DOCKER}" ] + then + docker-compose logs --tail 50 + fi - curl -v --fail $BIOBLEND_GALAXY_URL/api/version # Test self-signed HTTPS From e834ce4ef702533fc44bbf9589eb5f910c9f7174 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 7 Nov 2018 15:01:06 +0000 Subject: [PATCH 060/116] [compose] Pin nodejs to 9.11.1 on init --- compose/galaxy-init/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 9e804c667..89bc9b89b 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -91,7 +91,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 # prefetch Python wheels # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ - conda install -c conda-forge nodejs && \ + conda install -c conda-forge nodejs=9.11.1 && \ ./scripts/common_startup.sh --skip-client-build && \ . $GALAXY_VIRTUAL_ENV/bin/activate && \ python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \ From 2e948c33ecd4fb3f8a051245f47892afb512ec79 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 7 Nov 2018 15:58:18 +0000 Subject: [PATCH 061/116] [compose] Removes old adjustments for ubuntu 14.04 --- compose/galaxy-web/Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 390f8f9d5..b35cdd433 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -20,14 +20,15 @@ GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost \ # Set the number of Galaxy handlers GALAXY_HANDLER_NUMPROCS=2 -RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ - apt-get -qq update && apt-get install --no-install-recommends -y apt-transport-https wget && \ - echo "deb [arch=amd64] http://ppa.launchpad.net/galaxyproject/nginx/ubuntu trusty main" > /etc/apt/sources.list.d/nginx.list && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 9735427B && \ - apt-get update -qq && apt-get upgrade -y && \ +RUN apt-get -qq update && apt-get install --no-install-recommends -y apt-transport-https wget && \ + # echo "deb [arch=amd64] http://ppa.launchpad.net/galaxyproject/nginx/ubuntu trusty main" > /etc/apt/sources.list.d/nginx.list && \ + # apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 9735427B && \ + # apt-get update -qq && apt-get upgrade -y && \ + # apt-get update -qq && \ apt-get install --no-install-recommends -y \ - slurm-drmaa-dev nginx-extras=1.4.6-1ubuntu3.8ppa1 nginx-common=1.4.6-1ubuntu3.8ppa1 supervisor \ - slurm-llnl autofs && \ + # slurm-drmaa-dev nginx-extras=1.4.6-1ubuntu3.8ppa1 nginx-common=1.4.6-1ubuntu3.8ppa1 supervisor \ + # slurm-drmaa-dev slurm-llnl \ + nginx-extras nginx-common supervisor autofs && \ # install and remove supervisor, so get the service config file apt-get -qq remove -y supervisor && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ From 9c03f9937951b08a9a14bd95617819f98bc36041 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 7 Nov 2018 15:59:28 +0000 Subject: [PATCH 062/116] Don't upgrade pip, 9.0 should be sufficient (should it?) To avoid upgrading issues (main problem) that break build. --- compose/galaxy-web/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index b35cdd433..4204ba2b6 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get -qq update && apt-get install --no-install-recommends -y apt-transpo # this can be optimised I think, we should get the SLURM client only and update it gpasswd -a $GALAXY_USER docker && \ mkdir -p /etc/supervisor/conf.d/ /var/log/supervisor/ && \ - pip install --upgrade pip && \ + #pip install --upgrade pip && \ pip install ephemeris supervisor --upgrade && \ ln -s /usr/local/bin/supervisord /usr/bin/supervisord From cf04ecf02c351995a808a98b176098a74461f17b Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 7 Nov 2018 16:00:11 +0000 Subject: [PATCH 063/116] Improves build script from retrievals fro substitutions --- compose/build-orchestration-images.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index ce4cf4cbf..a56c501d5 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -123,9 +123,10 @@ DOCKER_USER=${CONTAINER_USER:-pcm32} ANSIBLE_REPO=${ANSIBLE_REPO:-galaxyproject/ansible-galaxy-extras} ANSIBLE_RELEASE=${ANSIBLE_RELEASE:-master} -GALAXY_VERSION=${GALAXY_VERSION:-18.05} +GALAXY_VERSION=${GALAXY_VERSION:-18.09} -GALAXY_BASE_FROM_TO_REPLACE=${GALAXY_BASE_FROM_TO_REPLACE:-quay.io/bgruening/galaxy-base:$GALAXY_VERSION} +# GALAXY_BASE_FROM_TO_REPLACE=${GALAXY_BASE_FROM_TO_REPLACE:-quay.io/bgruening/galaxy-base:$GALAXY_VERSION} +GALAXY_BASE_FROM_TO_REPLACE=$(grep ^FROM galaxy-init/Dockerfile | awk '{ print $2 }') # init starts from base, so we get it from there. CONDOR_BASE_FROM_TO_REPLACE=quay.io/bgruening/galaxy-htcondor-base:$GALAXY_VERSION GALAXY_RELEASE=${GALAXY_RELEASE:-release_$GALAXY_VERSION} @@ -210,8 +211,11 @@ fi DOCKERFILE_WEB=Dockerfile if [ -n $GALAXY_REPO ] then - echo "Making custom galaxy-web:$TAG from $GALAXY_REPO at $GALAXY_RELEASE" + log "Making custom galaxy-web:$TAG from $GALAXY_REPO at $GALAXY_RELEASE" + GALAXY_BASE_FROM_TO_REPLACE=$(grep ^FROM galaxy-web/Dockerfile | awk '{ print $2 }') sed s+$GALAXY_BASE_FROM_TO_REPLACE+$GALAXY_BASE_TAG+ galaxy-web/Dockerfile > galaxy-web/Dockerfile_web + FROM=$(grep ^FROM galaxy-web/Dockerfile_web | awk '{ print $2 }') + log "Using FROM $FROM for galaxy web" DOCKERFILE_WEB=Dockerfile_web fi K8S_ANSIBLE_TAGS="" From c7be226917fe21906a0893af6cd5073102c71c90 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 7 Nov 2018 17:25:21 +0000 Subject: [PATCH 064/116] [compose] Avoid nginx uploading as suggested in gitter To avoid nginx not starting due to upload module not available for ubuntu 18.04 --- compose/galaxy-web/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 4204ba2b6..f52c68499 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -55,6 +55,7 @@ RUN rm -f /usr/bin/startup && \ --extra-vars galaxy_extras_config_proftpd=False \ --extra-vars galaxy_extras_config_cvmfs=True \ --extra-vars galaxy_tool_data_table_config_file=/etc/galaxy/tool_data_table_conf.xml \ + --extra-vars galaxy_extras_config_nginx_upload=False \ --extra-vars nginx_upload_store_path=$GALAXY_CONFIG_NGINX_UPLOAD_STORE \ --extra-vars nginx_welcome_location=$NGINX_WELCOME_LOCATION \ --extra-vars nginx_welcome_path=$NGINX_WELCOME_PATH \ From 5915886395adc6c3aef97e254983f1c8f3256955 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 7 Nov 2018 21:52:40 +0000 Subject: [PATCH 065/116] [compose] Build client on init --- compose/galaxy-init/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 89bc9b89b..2baa011cd 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -91,13 +91,14 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 # prefetch Python wheels # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ - conda install -c conda-forge nodejs=9.11.1 && \ + conda install -c conda-forge nodejs=9.11.1 yarn=1.12.1 git && \ ./scripts/common_startup.sh --skip-client-build && \ . $GALAXY_VIRTUAL_ENV/bin/activate && \ python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \ # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \ npm install && \ + cd $GALAXY_ROOT/client && yarn install --network-timeout 120000 --check-files && yarn run build-production-maps && \ rm -rf /home/galaxy/.cache/ # Switch back to User root @@ -106,6 +107,7 @@ USER root # Create symlinks from $GALAXY_ROOT/* to /export/* RUN mv $GALAXY_ROOT/config /galaxy-export/config && ln -s -f /export/config $GALAXY_ROOT/config && \ mv $GALAXY_ROOT/tools /galaxy-export/tools && ln -s -f /export/tools $GALAXY_ROOT/tools && \ + mv $GALAXY_ROOT/client /galaxy-export/client && ln -s -f /export/client $GALAXY_ROOT/client && \ mv $GALAXY_ROOT/tool-data /galaxy-export/tool-data && ln -s -f /export/tool-data $GALAXY_ROOT/tool-data && \ mv $GALAXY_ROOT/display_applications /galaxy-export/display_applications && ln -s -f /export/display_applications $GALAXY_ROOT/display_applications && \ mv $GALAXY_ROOT/database /galaxy-export/database && ln -s -f /export/database $GALAXY_ROOT/database && \ From ef83c0f093cd3b08d87e2c4e17925e60c6e001e8 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Wed, 7 Nov 2018 22:57:19 +0100 Subject: [PATCH 066/116] use 18.09 version --- compose/galaxy-init/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 89bc9b89b..0d50cd720 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM quay.io/bgruening/galaxy-base:dev +FROM quay.io/bgruening/galaxy-base:18.09 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com From ac649f22ce18d0cb8e5d24f61163e758f9630777 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 7 Nov 2018 23:07:52 +0000 Subject: [PATCH 067/116] Remove unnecessary copying of client folder. --- compose/galaxy-init/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 0d5be74e9..b1b2eab7c 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -107,7 +107,6 @@ USER root # Create symlinks from $GALAXY_ROOT/* to /export/* RUN mv $GALAXY_ROOT/config /galaxy-export/config && ln -s -f /export/config $GALAXY_ROOT/config && \ mv $GALAXY_ROOT/tools /galaxy-export/tools && ln -s -f /export/tools $GALAXY_ROOT/tools && \ - mv $GALAXY_ROOT/client /galaxy-export/client && ln -s -f /export/client $GALAXY_ROOT/client && \ mv $GALAXY_ROOT/tool-data /galaxy-export/tool-data && ln -s -f /export/tool-data $GALAXY_ROOT/tool-data && \ mv $GALAXY_ROOT/display_applications /galaxy-export/display_applications && ln -s -f /export/display_applications $GALAXY_ROOT/display_applications && \ mv $GALAXY_ROOT/database /galaxy-export/database && ln -s -f /export/database $GALAXY_ROOT/database && \ From 6d96599dcb77fab7e171b7436590e6976d84da30 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Thu, 8 Nov 2018 17:12:43 +0000 Subject: [PATCH 068/116] Update schema for 18.09 --- compose/galaxy-postgres/init-galaxy-db.sql.in | 193 +++++++++++++++++- 1 file changed, 190 insertions(+), 3 deletions(-) diff --git a/compose/galaxy-postgres/init-galaxy-db.sql.in b/compose/galaxy-postgres/init-galaxy-db.sql.in index 04099ffd5..b23916dec 100644 --- a/compose/galaxy-postgres/init-galaxy-db.sql.in +++ b/compose/galaxy-postgres/init-galaxy-db.sql.in @@ -372,6 +372,73 @@ CREATE SEQUENCE cleanup_event_metadata_file_association_id_seq ALTER SEQUENCE cleanup_event_metadata_file_association_id_seq OWNED BY cleanup_event_metadata_file_association.id; +-- +-- Name: cleanup_event_user_association; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE cleanup_event_user_association ( + id integer NOT NULL, + create_time timestamp without time zone, + cleanup_event_id integer, + user_id integer +); + + +-- +-- Name: cleanup_event_user_association_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE cleanup_event_user_association_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: cleanup_event_user_association_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE cleanup_event_user_association_id_seq OWNED BY cleanup_event_user_association.id; + + +-- +-- Name: cloudauthz; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE cloudauthz ( + id integer NOT NULL, + user_id integer, + provider character varying(255), + config bytea, + authn_id integer, + tokens bytea, + last_update timestamp without time zone, + last_activity timestamp without time zone, + description text +); + + +-- +-- Name: cloudauthz_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE cloudauthz_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: cloudauthz_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE cloudauthz_id_seq OWNED BY cloudauthz.id; + + -- -- Name: data_manager_history_association; Type: TABLE; Schema: public; Owner: - -- @@ -2090,7 +2157,7 @@ CREATE TABLE job_metric_numeric ( job_id integer, plugin character varying(255), metric_name character varying(255), - metric_value numeric(22,7) + metric_value numeric(26,7) ); @@ -4283,7 +4350,7 @@ CREATE TABLE task_metric_numeric ( task_id integer, plugin character varying(255), metric_name character varying(255), - metric_value numeric(22,7) + metric_value numeric(26,7) ); @@ -5692,6 +5759,20 @@ ALTER TABLE ONLY cleanup_event_library_folder_association ALTER COLUMN id SET DE ALTER TABLE ONLY cleanup_event_metadata_file_association ALTER COLUMN id SET DEFAULT nextval('cleanup_event_metadata_file_association_id_seq'::regclass); +-- +-- Name: cleanup_event_user_association id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY cleanup_event_user_association ALTER COLUMN id SET DEFAULT nextval('cleanup_event_user_association_id_seq'::regclass); + + +-- +-- Name: cloudauthz id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY cloudauthz ALTER COLUMN id SET DEFAULT nextval('cloudauthz_id_seq'::regclass); + + -- -- Name: data_manager_history_association id; Type: DEFAULT; Schema: public; Owner: - -- @@ -6942,6 +7023,36 @@ COPY cleanup_event_metadata_file_association (id, create_time, cleanup_event_id, SELECT pg_catalog.setval('cleanup_event_metadata_file_association_id_seq', 1, false); +-- +-- Data for Name: cleanup_event_user_association; Type: TABLE DATA; Schema: public; Owner: - +-- + +COPY cleanup_event_user_association (id, create_time, cleanup_event_id, user_id) FROM stdin; +\. + + +-- +-- Name: cleanup_event_user_association_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('cleanup_event_user_association_id_seq', 1, false); + + +-- +-- Data for Name: cloudauthz; Type: TABLE DATA; Schema: public; Owner: - +-- + +COPY cloudauthz (id, user_id, provider, config, authn_id, tokens, last_update, last_activity, description) FROM stdin; +\. + + +-- +-- Name: cloudauthz_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - +-- + +SELECT pg_catalog.setval('cloudauthz_id_seq', 1, false); + + -- -- Data for Name: data_manager_history_association; Type: TABLE DATA; Schema: public; Owner: - -- @@ -8126,7 +8237,7 @@ GalaxyTools lib/tool_shed/galaxy_install/migrate 1 -- COPY migrate_version (repository_id, repository_path, version) FROM stdin; -Galaxy lib/galaxy/model/migrate 141 +Galaxy lib/galaxy/model/migrate 144 \. @@ -9381,6 +9492,22 @@ ALTER TABLE ONLY cleanup_event ADD CONSTRAINT cleanup_event_pkey PRIMARY KEY (id); +-- +-- Name: cleanup_event_user_association cleanup_event_user_association_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY cleanup_event_user_association + ADD CONSTRAINT cleanup_event_user_association_pkey PRIMARY KEY (id); + + +-- +-- Name: cloudauthz cloudauthz_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY cloudauthz + ADD CONSTRAINT cloudauthz_pkey PRIMARY KEY (id); + + -- -- Name: data_manager_history_association data_manager_history_association_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -10792,6 +10919,34 @@ CREATE INDEX ix_cleanup_event_metadata_file_association_cleanup_event_id ON clea CREATE INDEX ix_cleanup_event_metadata_file_association_metadata_file_id ON cleanup_event_metadata_file_association USING btree (metadata_file_id); +-- +-- Name: ix_cleanup_event_user_association_cleanup_event_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_cleanup_event_user_association_cleanup_event_id ON cleanup_event_user_association USING btree (cleanup_event_id); + + +-- +-- Name: ix_cleanup_event_user_association_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_cleanup_event_user_association_user_id ON cleanup_event_user_association USING btree (user_id); + + +-- +-- Name: ix_cloudauthz_authn_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_cloudauthz_authn_id ON cloudauthz USING btree (authn_id); + + +-- +-- Name: ix_cloudauthz_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_cloudauthz_user_id ON cloudauthz USING btree (user_id); + + -- -- Name: ix_data_manager_history_association_history_id; Type: INDEX; Schema: public; Owner: - -- @@ -14045,6 +14200,38 @@ ALTER TABLE ONLY cleanup_event_metadata_file_association ADD CONSTRAINT cleanup_event_metadata_file_association_metadata_file_id_fkey FOREIGN KEY (metadata_file_id) REFERENCES metadata_file(id); +-- +-- Name: cleanup_event_user_association cleanup_event_user_association_cleanup_event_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY cleanup_event_user_association + ADD CONSTRAINT cleanup_event_user_association_cleanup_event_id_fkey FOREIGN KEY (cleanup_event_id) REFERENCES cleanup_event(id); + + +-- +-- Name: cleanup_event_user_association cleanup_event_user_association_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY cleanup_event_user_association + ADD CONSTRAINT cleanup_event_user_association_user_id_fkey FOREIGN KEY (user_id) REFERENCES galaxy_user(id); + + +-- +-- Name: cloudauthz cloudauthz_authn_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY cloudauthz + ADD CONSTRAINT cloudauthz_authn_id_fkey FOREIGN KEY (authn_id) REFERENCES oidc_user_authnz_tokens(id); + + +-- +-- Name: cloudauthz cloudauthz_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY cloudauthz + ADD CONSTRAINT cloudauthz_user_id_fkey FOREIGN KEY (user_id) REFERENCES galaxy_user(id); + + -- -- Name: data_manager_history_association data_manager_history_association_history_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- From f7531ad1f89187ff658de354f6b868688b17dad3 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Thu, 8 Nov 2018 22:54:17 +0100 Subject: [PATCH 069/116] install new slurm --- compose/galaxy-base/Dockerfile | 4 ++-- compose/galaxy-web/Dockerfile | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index c84da1f9d..23af2a20b 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -73,8 +73,8 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ apt-get install --no-install-recommends -y sudo \ supervisor linux-image-extra-virtual munge \ ansible nano python-pip wget htcondor \ - unattended-upgrades \ - gridengine-drmaa1.0 && \ + unattended-upgrades && \ +## gridengine-drmaa1.0 && \ # conflicting with new slurm pip install ephemeris virtualenv && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ mkdir -p /tmp/download && \ diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index f52c68499..1ba7b3635 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -28,9 +28,15 @@ RUN apt-get -qq update && apt-get install --no-install-recommends -y apt-transpo apt-get install --no-install-recommends -y \ # slurm-drmaa-dev nginx-extras=1.4.6-1ubuntu3.8ppa1 nginx-common=1.4.6-1ubuntu3.8ppa1 supervisor \ # slurm-drmaa-dev slurm-llnl \ - nginx-extras nginx-common supervisor autofs && \ + nginx-extras nginx-common supervisor autofs libslurm32 libslurmdb32 + +RUN wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ + wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ + dpkg -i slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ + dpkg -i slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb + # install and remove supervisor, so get the service config file - apt-get -qq remove -y supervisor && \ +RUN apt-get -qq remove -y supervisor && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ groupadd --gid 999 docker && \ # this can be optimised I think, we should get the SLURM client only and update it From 4c015bcd045f0ee6d78854b401f9b5bf7f0abba3 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Fri, 9 Nov 2018 13:35:37 +0000 Subject: [PATCH 070/116] Use API based uploads instead of nginx --- compose/galaxy-base/Dockerfile | 2 -- compose/galaxy-web/Dockerfile | 1 - 2 files changed, 3 deletions(-) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index c84da1f9d..8c1e1beb3 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -36,8 +36,6 @@ GALAXY_CONFIG_FTP_UPLOAD_SITE=galaxy.docker.org \ GALAXY_CONFIG_USE_PBKDF2=False \ GALAXY_CONFIG_NGINX_X_ACCEL_REDIRECT_BASE=/_x_accel_redirect \ GALAXY_CONFIG_NGINX_X_ARCHIVE_FILES_BASE=/_x_accel_redirect \ -GALAXY_CONFIG_NGINX_UPLOAD_STORE=/export/nginx_upload_store \ -GALAXY_CONFIG_NGINX_UPLOAD_PATH=/_upload \ GALAXY_CONFIG_DYNAMIC_PROXY_MANAGE=False \ GALAXY_CONFIG_VISUALIZATION_PLUGINS_DIRECTORY=config/plugins/visualizations \ GALAXY_CONFIG_TRUST_IPYTHON_NOTEBOOK_CONVERSION=True \ diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index f52c68499..e8b5025a4 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -56,7 +56,6 @@ RUN rm -f /usr/bin/startup && \ --extra-vars galaxy_extras_config_cvmfs=True \ --extra-vars galaxy_tool_data_table_config_file=/etc/galaxy/tool_data_table_conf.xml \ --extra-vars galaxy_extras_config_nginx_upload=False \ - --extra-vars nginx_upload_store_path=$GALAXY_CONFIG_NGINX_UPLOAD_STORE \ --extra-vars nginx_welcome_location=$NGINX_WELCOME_LOCATION \ --extra-vars nginx_welcome_path=$NGINX_WELCOME_PATH \ --extra-vars startup_sleeplock=True \ From 234cf74461740416d052f8c64701b9573af19cd9 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 9 Nov 2018 15:29:52 +0100 Subject: [PATCH 071/116] enable SGE again --- compose/galaxy-base/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 23af2a20b..c84da1f9d 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -73,8 +73,8 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ apt-get install --no-install-recommends -y sudo \ supervisor linux-image-extra-virtual munge \ ansible nano python-pip wget htcondor \ - unattended-upgrades && \ -## gridengine-drmaa1.0 && \ # conflicting with new slurm + unattended-upgrades \ + gridengine-drmaa1.0 && \ pip install ephemeris virtualenv && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ mkdir -p /tmp/download && \ From 3f40125e74da22131c133688ab0edc9cb862840b Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 9 Nov 2018 15:32:32 +0100 Subject: [PATCH 072/116] merge layers --- compose/galaxy-web/Dockerfile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 28c2a82fc..46b25b820 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -28,15 +28,13 @@ RUN apt-get -qq update && apt-get install --no-install-recommends -y apt-transpo apt-get install --no-install-recommends -y \ # slurm-drmaa-dev nginx-extras=1.4.6-1ubuntu3.8ppa1 nginx-common=1.4.6-1ubuntu3.8ppa1 supervisor \ # slurm-drmaa-dev slurm-llnl \ - nginx-extras nginx-common supervisor autofs libslurm32 libslurmdb32 - -RUN wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ + nginx-extras nginx-common supervisor autofs libslurm32 libslurmdb32 && \ + wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ dpkg -i slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ - dpkg -i slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb - + dpkg -i slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ # install and remove supervisor, so get the service config file -RUN apt-get -qq remove -y supervisor && \ + apt-get -qq remove -y supervisor && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ groupadd --gid 999 docker && \ # this can be optimised I think, we should get the SLURM client only and update it From 8e244b60abd9da8acd7ea2446cc5ffc7d6d6b42d Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 9 Nov 2018 21:31:51 +0100 Subject: [PATCH 073/116] remove cruft --- compose/galaxy-init/Dockerfile | 2 +- compose/galaxy-web/Dockerfile | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index b1b2eab7c..7ccd3d1d3 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -98,7 +98,7 @@ RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \ npm install && \ - cd $GALAXY_ROOT/client && yarn install --network-timeout 120000 --check-files && yarn run build-production-maps && \ + ##cd $GALAXY_ROOT/client && yarn install --network-timeout 120000 --check-files && yarn run build-production-maps && \ rm -rf /home/galaxy/.cache/ # Switch back to User root diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 46b25b820..8a207fec0 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -20,14 +20,13 @@ GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost \ # Set the number of Galaxy handlers GALAXY_HANDLER_NUMPROCS=2 -RUN apt-get -qq update && apt-get install --no-install-recommends -y apt-transport-https wget && \ +RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ + apt-get -qq update && apt-get install --no-install-recommends -y apt-transport-https wget && \ # echo "deb [arch=amd64] http://ppa.launchpad.net/galaxyproject/nginx/ubuntu trusty main" > /etc/apt/sources.list.d/nginx.list && \ # apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 9735427B && \ # apt-get update -qq && apt-get upgrade -y && \ # apt-get update -qq && \ apt-get install --no-install-recommends -y \ - # slurm-drmaa-dev nginx-extras=1.4.6-1ubuntu3.8ppa1 nginx-common=1.4.6-1ubuntu3.8ppa1 supervisor \ - # slurm-drmaa-dev slurm-llnl \ nginx-extras nginx-common supervisor autofs libslurm32 libslurmdb32 && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ From b6722e4892f73a84aa115b965c1bf81d2e0c012a Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 10 Nov 2018 09:17:39 +0100 Subject: [PATCH 074/116] install new slurm --- compose/galaxy-slurm/Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 43064c59a..e3f6aa492 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -13,12 +13,15 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ groupadd -r $GALAXY_USER -g $GALAXY_GID && \ useradd -u $GALAXY_UID -m -r -g $GALAXY_USER -c "Galaxy user" $GALAXY_USER && \ apt-get update -qq && apt-get install -y --no-install-recommends apt-transport-https \ - slurm-llnl slurm-llnl-torque slurm-drmaa-dev unattended-upgrades \ - python-pip python-psutil supervisor wget build-essential && \ + unattended-upgrades python-pip python-psutil supervisor wget build-essential && \ + libslurm32 libslurmdb32 && \ + wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ + wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ + dpkg -i slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ + dpkg -i slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ /usr/sbin/create-munge-key && \ touch /var/log/slurm-llnl/slurmctld.log /var/log/slurm-llnl/slurmd.log && \ mkdir /tmp/slurm && pip install --upgrade supervisor virtualenv && \ - wget https://github.com/singularityware/singularity/releases/download/$SINGULARITY_VERSION/singularity-$SINGULARITY_VERSION.tar.gz && \ tar xvf singularity-$SINGULARITY_VERSION.tar.gz && \ cd singularity-$SINGULARITY_VERSION && \ @@ -26,7 +29,6 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ make && \ make install && \ rm -rf singularity-$SINGULARITY_VERSION singularity-$SINGULARITY_VERSION.tar.gz && \ - apt-get remove -y supervisor build-essential && \ mkdir /var/log/supervisor && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ From cc29851ea388971c4d160420282faac4bd3617b3 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 10 Nov 2018 09:39:29 +0100 Subject: [PATCH 075/116] fix syntax --- compose/galaxy-slurm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index e3f6aa492..fb780ef11 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -13,7 +13,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ groupadd -r $GALAXY_USER -g $GALAXY_GID && \ useradd -u $GALAXY_UID -m -r -g $GALAXY_USER -c "Galaxy user" $GALAXY_USER && \ apt-get update -qq && apt-get install -y --no-install-recommends apt-transport-https \ - unattended-upgrades python-pip python-psutil supervisor wget build-essential && \ + unattended-upgrades python-pip python-psutil supervisor wget build-essential \ libslurm32 libslurmdb32 && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ From 7d834ec35a5b719c1d90dc67cba29b8fa9aa4ed2 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 11 Dec 2018 13:42:27 +0100 Subject: [PATCH 076/116] update galaxyextras 18.09 --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index e6612923a..67081dc46 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "galaxy/roles/galaxyproject.galaxyextras"] path = galaxy/roles/galaxyprojectdotorg.galaxyextras url = https://github.com/galaxyproject/ansible-galaxy-extras - branch = 18.01 + branch = 18.09 [submodule "galaxy/roles/galaxy-postgresql"] path = galaxy/roles/galaxy-postgresql url = https://github.com/galaxyproject/ansible-postgresql From 9998785680cfde5f8b2c60f19e5ff5e3cae1e180 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 11 Dec 2018 14:24:39 +0100 Subject: [PATCH 077/116] need munge --- compose/galaxy-slurm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index fb780ef11..ebbe73e75 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -14,7 +14,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ useradd -u $GALAXY_UID -m -r -g $GALAXY_USER -c "Galaxy user" $GALAXY_USER && \ apt-get update -qq && apt-get install -y --no-install-recommends apt-transport-https \ unattended-upgrades python-pip python-psutil supervisor wget build-essential \ - libslurm32 libslurmdb32 && \ + libslurm32 libslurmdb32 munge && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ dpkg -i slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ From beb8440b0c898a3e4ed1f5d2cd3c86802ae9940a Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 11 Dec 2018 14:40:30 +0100 Subject: [PATCH 078/116] fix slurm install --- compose/galaxy-slurm/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index ebbe73e75..0f660395f 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -13,13 +13,14 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ groupadd -r $GALAXY_USER -g $GALAXY_GID && \ useradd -u $GALAXY_UID -m -r -g $GALAXY_USER -c "Galaxy user" $GALAXY_USER && \ apt-get update -qq && apt-get install -y --no-install-recommends apt-transport-https \ - unattended-upgrades python-pip python-psutil supervisor wget build-essential \ - libslurm32 libslurmdb32 munge && \ + unattended-upgrades python-pip python-psutil python-setuptools supervisor wget \ + build-essential libslurm32 libslurmdb32 munge locales && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ dpkg -i slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ dpkg -i slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ /usr/sbin/create-munge-key && \ + mkdir /var/log/slurm-llnl && \ touch /var/log/slurm-llnl/slurmctld.log /var/log/slurm-llnl/slurmd.log && \ mkdir /tmp/slurm && pip install --upgrade supervisor virtualenv && \ wget https://github.com/singularityware/singularity/releases/download/$SINGULARITY_VERSION/singularity-$SINGULARITY_VERSION.tar.gz && \ From 1be3274c25c9c9be248180c96110f96ab0bba193 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 11 Dec 2018 14:50:25 +0100 Subject: [PATCH 079/116] fix requirements install --- compose/galaxy-slurm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 0f660395f..7b182ab54 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -70,7 +70,7 @@ RUN chown -R $SLURM_UID:$SLURM_GID "$GALAXY_VENV" && \ . "$GALAXY_VENV"/bin/activate && \ pip install --upgrade pip && \ pip install galaxy-lib && \ - pip install -r "$GALAXY_VENV"/requirements.txt --index-url https://wheels.galaxyproject.org/simple && \ + pip install -r "$GALAXY_VENV"/requirements.txt --index-url https://wheels.galaxyproject.org/simple --extra-index-url https://pypi.python.org/simple && \ rm -rf ~/.cache/ From 2efd5251921b18e9ea39e0c5ab240abb389cc419 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 11 Dec 2018 16:39:36 +0100 Subject: [PATCH 080/116] try hack for ansible rule --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 20b3dbb39..a0fc5b388 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,6 +110,7 @@ before_install: # start building this repo - git submodule update --init --recursive + - sed -i 's|/galaxy.ini|/galaxy.yml|g' galaxy/roles/galaxyprojectdotorg.galaxyextras/defaults/main.yml - sudo chown 1450 /tmp && sudo chmod a=rwx /tmp - | if [ "${COMPOSE_SLURM}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${KUBE}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] From c194c51f89eff91abc6d6d8e2a7ebf70112ccbcb Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 11 Dec 2018 17:03:13 +0100 Subject: [PATCH 081/116] fix for the hack --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a0fc5b388..9baa311dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,7 +110,7 @@ before_install: # start building this repo - git submodule update --init --recursive - - sed -i 's|/galaxy.ini|/galaxy.yml|g' galaxy/roles/galaxyprojectdotorg.galaxyextras/defaults/main.yml + - sed -i 's|/galaxy.ini|/galaxy.yml|g' ../galaxy/roles/galaxyprojectdotorg.galaxyextras/defaults/main.yml - sudo chown 1450 /tmp && sudo chmod a=rwx /tmp - | if [ "${COMPOSE_SLURM}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${KUBE}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] From 14814930b46b43a99d8e978c306ebb5016e443d2 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 11 Dec 2018 17:08:29 +0100 Subject: [PATCH 082/116] hack for the hack --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9baa311dc..0e2b1fed2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,7 +110,7 @@ before_install: # start building this repo - git submodule update --init --recursive - - sed -i 's|/galaxy.ini|/galaxy.yml|g' ../galaxy/roles/galaxyprojectdotorg.galaxyextras/defaults/main.yml + - sed -i 's|/galaxy.ini|/galaxy.yml|g' /home/travis/build/abretaud/docker-galaxy-stable/galaxy/roles/galaxyprojectdotorg.galaxyextras/defaults/main.yml - sudo chown 1450 /tmp && sudo chmod a=rwx /tmp - | if [ "${COMPOSE_SLURM}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${KUBE}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] From 5bd1ba5c1e408765f8376de0f3e00dc675829071 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 12 Dec 2018 09:34:26 +0100 Subject: [PATCH 083/116] supervisor stuff --- compose/galaxy-slurm/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 7b182ab54..2f505af80 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -22,7 +22,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ /usr/sbin/create-munge-key && \ mkdir /var/log/slurm-llnl && \ touch /var/log/slurm-llnl/slurmctld.log /var/log/slurm-llnl/slurmd.log && \ - mkdir /tmp/slurm && pip install --upgrade supervisor virtualenv && \ + mkdir /tmp/slurm && pip install --upgrade virtualenv && \ wget https://github.com/singularityware/singularity/releases/download/$SINGULARITY_VERSION/singularity-$SINGULARITY_VERSION.tar.gz && \ tar xvf singularity-$SINGULARITY_VERSION.tar.gz && \ cd singularity-$SINGULARITY_VERSION && \ @@ -30,7 +30,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ make && \ make install && \ rm -rf singularity-$SINGULARITY_VERSION singularity-$SINGULARITY_VERSION.tar.gz && \ - apt-get remove -y supervisor build-essential && \ + apt-get remove -y build-essential && \ mkdir /var/log/supervisor && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ mkdir -p /tmp/download && \ From 9efc04cda4829eaccaf4f715940ac102411d2ede Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 12 Dec 2018 09:52:23 +0100 Subject: [PATCH 084/116] no need to mkdir --- compose/galaxy-slurm/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 2f505af80..178a6084f 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -31,7 +31,6 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ make install && \ rm -rf singularity-$SINGULARITY_VERSION singularity-$SINGULARITY_VERSION.tar.gz && \ apt-get remove -y build-essential && \ - mkdir /var/log/supervisor && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ mkdir -p /tmp/download && \ wget -qO - https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz | tar -xz -C /tmp/download && \ From 2312f4f6d26c2fa7b56495ed1026e09e15a87f52 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 12 Dec 2018 10:30:16 +0100 Subject: [PATCH 085/116] slurm startup fix --- compose/galaxy-slurm/startup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-slurm/startup.sh b/compose/galaxy-slurm/startup.sh index e6ebf8994..799796c6c 100644 --- a/compose/galaxy-slurm/startup.sh +++ b/compose/galaxy-slurm/startup.sh @@ -16,6 +16,7 @@ fi if [ ! -f "$SLURM_CONF_PATH" ] then + mkdir -p /etc/slurm-llnl/ python /usr/local/bin/configure_slurm.py cp /etc/slurm-llnl/slurm.conf "$SLURM_CONF_PATH" fi @@ -23,4 +24,4 @@ mkdir -p /tmp/slurm chown $SLURM_USER_NAME /tmp/slurm ln -sf "$GALAXY_DIR" "$SYMLINK_TARGET" ln -sf "$SLURM_CONF_PATH" /etc/slurm-llnl/slurm.conf -exec /usr/local/bin/supervisord -n -c /etc/supervisor/supervisord.conf +exec /usr/bin/supervisord -n -c /etc/supervisor/supervisord.conf From b533b207d733d70a9082e7581dcd76f4c9539b14 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 12 Dec 2018 11:57:51 +0100 Subject: [PATCH 086/116] For 18.09 uses migration 142, not 144 --- compose/galaxy-postgres/init-galaxy-db.sql.in | 190 +----------------- 1 file changed, 1 insertion(+), 189 deletions(-) diff --git a/compose/galaxy-postgres/init-galaxy-db.sql.in b/compose/galaxy-postgres/init-galaxy-db.sql.in index b23916dec..5fa025111 100644 --- a/compose/galaxy-postgres/init-galaxy-db.sql.in +++ b/compose/galaxy-postgres/init-galaxy-db.sql.in @@ -372,73 +372,6 @@ CREATE SEQUENCE cleanup_event_metadata_file_association_id_seq ALTER SEQUENCE cleanup_event_metadata_file_association_id_seq OWNED BY cleanup_event_metadata_file_association.id; --- --- Name: cleanup_event_user_association; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE cleanup_event_user_association ( - id integer NOT NULL, - create_time timestamp without time zone, - cleanup_event_id integer, - user_id integer -); - - --- --- Name: cleanup_event_user_association_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE cleanup_event_user_association_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: cleanup_event_user_association_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE cleanup_event_user_association_id_seq OWNED BY cleanup_event_user_association.id; - - --- --- Name: cloudauthz; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE cloudauthz ( - id integer NOT NULL, - user_id integer, - provider character varying(255), - config bytea, - authn_id integer, - tokens bytea, - last_update timestamp without time zone, - last_activity timestamp without time zone, - description text -); - - --- --- Name: cloudauthz_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE cloudauthz_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: cloudauthz_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE cloudauthz_id_seq OWNED BY cloudauthz.id; - - -- -- Name: data_manager_history_association; Type: TABLE; Schema: public; Owner: - -- @@ -5759,20 +5692,6 @@ ALTER TABLE ONLY cleanup_event_library_folder_association ALTER COLUMN id SET DE ALTER TABLE ONLY cleanup_event_metadata_file_association ALTER COLUMN id SET DEFAULT nextval('cleanup_event_metadata_file_association_id_seq'::regclass); --- --- Name: cleanup_event_user_association id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY cleanup_event_user_association ALTER COLUMN id SET DEFAULT nextval('cleanup_event_user_association_id_seq'::regclass); - - --- --- Name: cloudauthz id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY cloudauthz ALTER COLUMN id SET DEFAULT nextval('cloudauthz_id_seq'::regclass); - - -- -- Name: data_manager_history_association id; Type: DEFAULT; Schema: public; Owner: - -- @@ -7023,36 +6942,6 @@ COPY cleanup_event_metadata_file_association (id, create_time, cleanup_event_id, SELECT pg_catalog.setval('cleanup_event_metadata_file_association_id_seq', 1, false); --- --- Data for Name: cleanup_event_user_association; Type: TABLE DATA; Schema: public; Owner: - --- - -COPY cleanup_event_user_association (id, create_time, cleanup_event_id, user_id) FROM stdin; -\. - - --- --- Name: cleanup_event_user_association_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - --- - -SELECT pg_catalog.setval('cleanup_event_user_association_id_seq', 1, false); - - --- --- Data for Name: cloudauthz; Type: TABLE DATA; Schema: public; Owner: - --- - -COPY cloudauthz (id, user_id, provider, config, authn_id, tokens, last_update, last_activity, description) FROM stdin; -\. - - --- --- Name: cloudauthz_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - --- - -SELECT pg_catalog.setval('cloudauthz_id_seq', 1, false); - - -- -- Data for Name: data_manager_history_association; Type: TABLE DATA; Schema: public; Owner: - -- @@ -8237,7 +8126,7 @@ GalaxyTools lib/tool_shed/galaxy_install/migrate 1 -- COPY migrate_version (repository_id, repository_path, version) FROM stdin; -Galaxy lib/galaxy/model/migrate 144 +Galaxy lib/galaxy/model/migrate 142 \. @@ -9492,22 +9381,6 @@ ALTER TABLE ONLY cleanup_event ADD CONSTRAINT cleanup_event_pkey PRIMARY KEY (id); --- --- Name: cleanup_event_user_association cleanup_event_user_association_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY cleanup_event_user_association - ADD CONSTRAINT cleanup_event_user_association_pkey PRIMARY KEY (id); - - --- --- Name: cloudauthz cloudauthz_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY cloudauthz - ADD CONSTRAINT cloudauthz_pkey PRIMARY KEY (id); - - -- -- Name: data_manager_history_association data_manager_history_association_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -10919,34 +10792,6 @@ CREATE INDEX ix_cleanup_event_metadata_file_association_cleanup_event_id ON clea CREATE INDEX ix_cleanup_event_metadata_file_association_metadata_file_id ON cleanup_event_metadata_file_association USING btree (metadata_file_id); --- --- Name: ix_cleanup_event_user_association_cleanup_event_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX ix_cleanup_event_user_association_cleanup_event_id ON cleanup_event_user_association USING btree (cleanup_event_id); - - --- --- Name: ix_cleanup_event_user_association_user_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX ix_cleanup_event_user_association_user_id ON cleanup_event_user_association USING btree (user_id); - - --- --- Name: ix_cloudauthz_authn_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX ix_cloudauthz_authn_id ON cloudauthz USING btree (authn_id); - - --- --- Name: ix_cloudauthz_user_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX ix_cloudauthz_user_id ON cloudauthz USING btree (user_id); - - -- -- Name: ix_data_manager_history_association_history_id; Type: INDEX; Schema: public; Owner: - -- @@ -14200,38 +14045,6 @@ ALTER TABLE ONLY cleanup_event_metadata_file_association ADD CONSTRAINT cleanup_event_metadata_file_association_metadata_file_id_fkey FOREIGN KEY (metadata_file_id) REFERENCES metadata_file(id); --- --- Name: cleanup_event_user_association cleanup_event_user_association_cleanup_event_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY cleanup_event_user_association - ADD CONSTRAINT cleanup_event_user_association_cleanup_event_id_fkey FOREIGN KEY (cleanup_event_id) REFERENCES cleanup_event(id); - - --- --- Name: cleanup_event_user_association cleanup_event_user_association_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY cleanup_event_user_association - ADD CONSTRAINT cleanup_event_user_association_user_id_fkey FOREIGN KEY (user_id) REFERENCES galaxy_user(id); - - --- --- Name: cloudauthz cloudauthz_authn_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY cloudauthz - ADD CONSTRAINT cloudauthz_authn_id_fkey FOREIGN KEY (authn_id) REFERENCES oidc_user_authnz_tokens(id); - - --- --- Name: cloudauthz cloudauthz_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY cloudauthz - ADD CONSTRAINT cloudauthz_user_id_fkey FOREIGN KEY (user_id) REFERENCES galaxy_user(id); - - -- -- Name: data_manager_history_association data_manager_history_association_history_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -16651,4 +16464,3 @@ ALTER TABLE ONLY workflow_tag_association -- -- PostgreSQL database dump complete -- - From d456e53d3d41683abccdf05ae1cee48ece736467 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 12 Dec 2018 15:23:22 +0100 Subject: [PATCH 087/116] condor fixes --- compose/galaxy-htcondor-executor/Dockerfile | 4 ++++ compose/galaxy-htcondor-executor/startup.sh | 2 -- compose/galaxy-htcondor/Dockerfile | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/compose/galaxy-htcondor-executor/Dockerfile b/compose/galaxy-htcondor-executor/Dockerfile index 46f4a6e27..d4daf3e78 100644 --- a/compose/galaxy-htcondor-executor/Dockerfile +++ b/compose/galaxy-htcondor-executor/Dockerfile @@ -27,6 +27,10 @@ RUN mkdir -p /tmp/download && \ #chmod +rx /root && \ chmod +x /usr/bin/startup.sh +RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \ + mkdir -p /var/run/condor/ /var/lock/condor/ && \ + chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/ + ENV CONDOR_CPUS=1 \ CONDOR_MEMORY=1024 diff --git a/compose/galaxy-htcondor-executor/startup.sh b/compose/galaxy-htcondor-executor/startup.sh index cf7f728ab..147a1d417 100644 --- a/compose/galaxy-htcondor-executor/startup.sh +++ b/compose/galaxy-htcondor-executor/startup.sh @@ -30,8 +30,6 @@ SCHED_NAME = $CONDOR_HOST " > /etc/condor/condor_config.local /usr/bin/telegraf --config /etc/telegraf/telegraf.conf & -sudo -u condor touch /var/log/condor/StartLog -sudo -u condor touch /var/log/condor/StarterLog tail -f -n 1000 /var/log/condor/StartLog /var/log/condor/StarterLog & # Mysterious bug? Docker doesn't output its version as condor user if this is not executed before diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index 4c26d3265..59e79cbc7 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -3,4 +3,8 @@ FROM quay.io/bgruening/galaxy-htcondor-base:18.09 ADD condor_config.local /etc/condor/condor_config.local ADD supervisord.conf /etc/supervisord.conf +RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \ + mkdir -p /var/run/condor/ /var/lock/condor/ && \ + chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/ + CMD ["/usr/bin/supervisord"] From 716b859066ebd8ea8d036dd37bfbb5fac39cab02 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 12 Dec 2018 15:23:48 +0100 Subject: [PATCH 088/116] avoid harmless error message --- compose/galaxy-init/startup.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/compose/galaxy-init/startup.sh b/compose/galaxy-init/startup.sh index ef5fd69b1..88e7537d8 100644 --- a/compose/galaxy-init/startup.sh +++ b/compose/galaxy-init/startup.sh @@ -30,13 +30,16 @@ done # Optional, might not work { - if [ ! -d "/export/var/lib/docker" ] + if [ -d "/var/lib/docker" ] then - echo "Moving to /export/var/lib/docker" - mkdir -p /export/var/lib/ - mv /var/lib/docker /export/var/lib/docker - chown -R $GALAXY_UID:$GALAXY_GID /export/var/lib/docker - fi + if [ ! -d "/export/var/lib/docker" ] + then + echo "Moving to /export/var/lib/docker" + mkdir -p /export/var/lib/ + mv /var/lib/docker /export/var/lib/docker + chown -R $GALAXY_UID:$GALAXY_GID /export/var/lib/docker + fi + fi } || echo "Moving docker lib failed, this is not a fatal error" echo "Initialization complete" From 8f42276150398a41d6ceaf77cbc5b3db33126e72 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 12 Dec 2018 15:24:02 +0100 Subject: [PATCH 089/116] slurm install --- compose/galaxy-slurm/Dockerfile | 6 ++---- compose/galaxy-web/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 178a6084f..e640f6951 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -14,15 +14,13 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ useradd -u $GALAXY_UID -m -r -g $GALAXY_USER -c "Galaxy user" $GALAXY_USER && \ apt-get update -qq && apt-get install -y --no-install-recommends apt-transport-https \ unattended-upgrades python-pip python-psutil python-setuptools supervisor wget \ - build-essential libslurm32 libslurmdb32 munge locales && \ + build-essential munge locales slurm-wlm slurm-wlm-torque && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ dpkg -i slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ dpkg -i slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ /usr/sbin/create-munge-key && \ - mkdir /var/log/slurm-llnl && \ - touch /var/log/slurm-llnl/slurmctld.log /var/log/slurm-llnl/slurmd.log && \ - mkdir /tmp/slurm && pip install --upgrade virtualenv && \ + pip install --upgrade virtualenv && \ wget https://github.com/singularityware/singularity/releases/download/$SINGULARITY_VERSION/singularity-$SINGULARITY_VERSION.tar.gz && \ tar xvf singularity-$SINGULARITY_VERSION.tar.gz && \ cd singularity-$SINGULARITY_VERSION && \ diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 8a207fec0..33748be1a 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -27,7 +27,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ # apt-get update -qq && apt-get upgrade -y && \ # apt-get update -qq && \ apt-get install --no-install-recommends -y \ - nginx-extras nginx-common supervisor autofs libslurm32 libslurmdb32 && \ + nginx-extras nginx-common supervisor autofs slurm-wlm slurm-wlm-torque && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ dpkg -i slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ From 57dcde9678961efc433b2896f4f389d481a9d5ff Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 12 Dec 2018 16:01:36 +0100 Subject: [PATCH 090/116] fix missing ansible tag --- compose/build-orchestration-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index a56c501d5..95c2db152 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -222,7 +222,7 @@ K8S_ANSIBLE_TAGS="" if $BUILD_FOR_K8S; then K8S_ANSIBLE_TAGS=,k8,k8s fi -docker build $NO_CACHE --build-arg GALAXY_ANSIBLE_TAGS=supervisor,startup,scripts,nginx$K8S_ANSIBLE_TAGS -t $GALAXY_WEB_TAG -f galaxy-web/$DOCKERFILE_WEB galaxy-web/ +docker build $NO_CACHE --build-arg GALAXY_ANSIBLE_TAGS=supervisor,startup,scripts,nginx,cvmfs$K8S_ANSIBLE_TAGS -t $GALAXY_WEB_TAG -f galaxy-web/$DOCKERFILE_WEB galaxy-web/ if $DOCKER_PUSH_ENABLED; then docker push $GALAXY_WEB_TAG fi From 933b13a1d0a1ea9fc9f83579e5551779e94526cf Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 12 Dec 2018 16:03:46 +0100 Subject: [PATCH 091/116] kube fix? --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0e2b1fed2..65ea591af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,6 +103,10 @@ before_install: - | if [ "${KUBE}" ] then + # The compose file recognises ENV vars to change the defaul behavior + cd ${COMPOSE_DIR} + ln -sf .env_k8_native .env + curl -L https://github.com/kubernetes-incubator/kompose/releases/download/v0.5.0/kompose-linux-amd64 -o kompose chmod +x kompose sudo mv ./kompose /usr/bin/kompose From 05be0b7e93f5ae9128fc05aa59c7caba039ab2a4 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 12 Dec 2018 17:12:52 +0100 Subject: [PATCH 092/116] not needed --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 65ea591af..71e24511f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -114,7 +114,6 @@ before_install: # start building this repo - git submodule update --init --recursive - - sed -i 's|/galaxy.ini|/galaxy.yml|g' /home/travis/build/abretaud/docker-galaxy-stable/galaxy/roles/galaxyprojectdotorg.galaxyextras/defaults/main.yml - sudo chown 1450 /tmp && sudo chmod a=rwx /tmp - | if [ "${COMPOSE_SLURM}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${KUBE}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] From 209b057d878747d97cd5e3a4af98adf2fb658471 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Thu, 13 Dec 2018 09:40:34 +0100 Subject: [PATCH 093/116] move to htcondor-base --- compose/galaxy-htcondor-base/Dockerfile | 5 +++++ compose/galaxy-htcondor-executor/Dockerfile | 4 ---- compose/galaxy-htcondor/Dockerfile | 4 ---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/compose/galaxy-htcondor-base/Dockerfile b/compose/galaxy-htcondor-base/Dockerfile index 3087fa2d2..afe67f87e 100644 --- a/compose/galaxy-htcondor-base/Dockerfile +++ b/compose/galaxy-htcondor-base/Dockerfile @@ -18,3 +18,8 @@ RUN wget https://dl.influxdata.com/telegraf/releases/telegraf-1.5.0_linux_amd64. rm telegraf-1.5.0_linux_amd64.tar.gz ADD telegraf.conf /etc/telegraf/telegraf.conf + + +RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \ + mkdir -p /var/run/condor/ /var/lock/condor/ && \ + chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/ diff --git a/compose/galaxy-htcondor-executor/Dockerfile b/compose/galaxy-htcondor-executor/Dockerfile index d4daf3e78..46f4a6e27 100644 --- a/compose/galaxy-htcondor-executor/Dockerfile +++ b/compose/galaxy-htcondor-executor/Dockerfile @@ -27,10 +27,6 @@ RUN mkdir -p /tmp/download && \ #chmod +rx /root && \ chmod +x /usr/bin/startup.sh -RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \ - mkdir -p /var/run/condor/ /var/lock/condor/ && \ - chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/ - ENV CONDOR_CPUS=1 \ CONDOR_MEMORY=1024 diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index 59e79cbc7..4c26d3265 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -3,8 +3,4 @@ FROM quay.io/bgruening/galaxy-htcondor-base:18.09 ADD condor_config.local /etc/condor/condor_config.local ADD supervisord.conf /etc/supervisord.conf -RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \ - mkdir -p /var/run/condor/ /var/lock/condor/ && \ - chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/ - CMD ["/usr/bin/supervisord"] From edbef97c5fa8f4d9d0f8e54db72b85bccba6d51e Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Thu, 13 Dec 2018 09:50:11 +0100 Subject: [PATCH 094/116] need 18.09 branch of ansible-galaxy-extras --- compose/build-orchestration-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh index 95c2db152..1002f4fab 100755 --- a/compose/build-orchestration-images.sh +++ b/compose/build-orchestration-images.sh @@ -121,7 +121,7 @@ DOCKER_REPO=${CONTAINER_REGISTRY:-} DOCKER_USER=${CONTAINER_USER:-pcm32} ANSIBLE_REPO=${ANSIBLE_REPO:-galaxyproject/ansible-galaxy-extras} -ANSIBLE_RELEASE=${ANSIBLE_RELEASE:-master} +ANSIBLE_RELEASE=${ANSIBLE_RELEASE:-18.09} GALAXY_VERSION=${GALAXY_VERSION:-18.09} From b50f1d1c122f5b1f35905e0e1ed7a53ff05158e5 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Thu, 13 Dec 2018 15:55:11 +0100 Subject: [PATCH 095/116] need nodejs installed (but can't use the ansible task as the galaxy code is not available yet at build time) --- compose/galaxy-web/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile index 33748be1a..5394f628b 100644 --- a/compose/galaxy-web/Dockerfile +++ b/compose/galaxy-web/Dockerfile @@ -22,11 +22,10 @@ GALAXY_HANDLER_NUMPROCS=2 RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ apt-get -qq update && apt-get install --no-install-recommends -y apt-transport-https wget && \ - # echo "deb [arch=amd64] http://ppa.launchpad.net/galaxyproject/nginx/ubuntu trusty main" > /etc/apt/sources.list.d/nginx.list && \ - # apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 9735427B && \ - # apt-get update -qq && apt-get upgrade -y && \ - # apt-get update -qq && \ - apt-get install --no-install-recommends -y \ + echo "deb https://deb.nodesource.com/node_9.x $(lsb_release -sc) main" > /etc/apt/sources.list.d/nodejs.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 68576280 && \ + apt-get update -qq && \ + apt-get install --no-install-recommends -y nodejs \ nginx-extras nginx-common supervisor autofs slurm-wlm slurm-wlm-torque && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \ From 95a5d7cb71de1c8adae51bd5684ef027ad70afa6 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 14 Dec 2018 11:05:38 +0100 Subject: [PATCH 096/116] trigger rebuild From 33d03bca6d341911ba6bf33c24dc3701ea6782d5 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 14 Dec 2018 15:01:49 +0100 Subject: [PATCH 097/116] fix error at startup: this line mounted a not-yet existing file (created later by galaxy-init). It was mounted as an empty dir, so galaxy refused to run. The /usr/bin/startup script will copy the good file from $GALAXY_ROOT a bit later during galaxy-web startup, after galaxy-init has created it (https://github.com/galaxyproject/ansible-galaxy-extras/blob/18.09/templates/startup.sh.j2#L42). --- compose/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index fa458df04..8517a3dd4 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -324,7 +324,6 @@ services: volumes: # This is the directory where all your files from Galaxy will be stored # on your host system - - ${EXPORT_DIR:-/export}/galaxy-central/config/galaxy.yml.sample:/etc/galaxy/galaxy.yml - ${EXPORT_DIR:-/export}/:/export/ - /var/run/docker.sock:/var/run/docker.sock depends_on: From 52f92d42d2c67489ffd8cab77ed512620b703ed9 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 14 Dec 2018 15:33:53 +0100 Subject: [PATCH 098/116] update bioblend testing --- test/bioblend/Dockerfile | 2 +- test/bioblend/test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile index 0750accfd..6e44daf75 100644 --- a/test/bioblend/Dockerfile +++ b/test/bioblend/Dockerfile @@ -14,7 +14,7 @@ ENV TOX_ENV=py27 \ BIOBLEND_GALAXY_API_KEY=admin \ BIOBLEND_GALAXY_URL=http://galaxy -CMD /bin/bash -c "export PATH=/tool_deps/_conda/bin/:$PATH && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -e 'test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_tool_dependency_install|test_download_history|test_export_and_download'" +CMD /bin/bash -c "export PATH=/tool_deps/_conda/bin/:$PATH && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -k 'not test_download_dataset and not test_upload_from_galaxy_filesystem and not test_get_datasets and not test_datasets_from_fs and not test_tool_dependency_install and not test_download_history and not test_export_and_download'" # library tests, needs share /tmp filesystem # * test_upload_from_galaxy_filesystem diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index 5eac22fd2..a031967f3 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -12,7 +12,7 @@ then export BIOBLEND_GALAXY_API_KEY=admin ; export BIOBLEND_GALAXY_URL=http://galaxy ; cd /home/galaxy/bioblend-master ; - tox -e $TOX_ENV -- -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install|test_download_history|test_export_and_download"' + tox -e $TOX_ENV -- -k "not test_download_dataset and not test_upload_from_galaxy_filesystem and not test_get_datasets and not test_datasets_from_fs and not test_existing_history and not test_new_history and not test_params and not test_tool_dependency_install and not test_download_history and not test_export_and_download"' else docker build -t bioblend_test . From e938b860f2d0ea2edbc90fe05c8bcc1235283124 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 14 Dec 2018 17:08:46 +0100 Subject: [PATCH 099/116] fix bioblend test env --- test/bioblend/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index a031967f3..381590641 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -2,7 +2,7 @@ if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] then docker_exec bash -c 'cd /home/galaxy ; - export PATH=/tool_deps/_conda/bin/:$PATH; . activate galaxy_env; + . /galaxy_venv/bin/activate ; wget https://github.com/galaxyproject/bioblend/archive/master.tar.gz && tar xfz master.tar.gz ; cd bioblend-master ; pip install --upgrade "tox>=1.8.0" "pep8<=1.6.2" ; From 7e53dc0edc72b2529593b6ba889e8d63b94d879b Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 11:48:08 +0100 Subject: [PATCH 100/116] fix image tag --- compose/dumpsql.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/dumpsql.sh b/compose/dumpsql.sh index efe6c488d..e18fba5f1 100755 --- a/compose/dumpsql.sh +++ b/compose/dumpsql.sh @@ -1,6 +1,6 @@ #!/bin/bash -TAG=18.09 +TAG=v18.09 # Sets the image of postgres to use POSTGRES=postgres:9.6.5 From 5516786d8d7afe7d5cdafcb94ef2cadbd44c503f Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 12:26:02 +0100 Subject: [PATCH 101/116] dump fix --- compose/galaxy-postgres/init-galaxy-db.sql.in | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/galaxy-postgres/init-galaxy-db.sql.in b/compose/galaxy-postgres/init-galaxy-db.sql.in index 5fa025111..eb19cbb2b 100644 --- a/compose/galaxy-postgres/init-galaxy-db.sql.in +++ b/compose/galaxy-postgres/init-galaxy-db.sql.in @@ -16464,3 +16464,4 @@ ALTER TABLE ONLY workflow_tag_association -- -- PostgreSQL database dump complete -- + From 72866eddaa6ccdd8ca08c9fcf4ae1c87b5f08211 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 14:49:51 +0100 Subject: [PATCH 102/116] reenable client building --- compose/galaxy-init/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index 7ccd3d1d3..b1b2eab7c 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -98,7 +98,7 @@ RUN export PATH=/tool_deps/_conda/bin/:$PATH && \ # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \ npm install && \ - ##cd $GALAXY_ROOT/client && yarn install --network-timeout 120000 --check-files && yarn run build-production-maps && \ + cd $GALAXY_ROOT/client && yarn install --network-timeout 120000 --check-files && yarn run build-production-maps && \ rm -rf /home/galaxy/.cache/ # Switch back to User root From c21cfccff70dae64d834953f61d995857ddc5b6e Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 14:50:12 +0100 Subject: [PATCH 103/116] fix slurm_singularity test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 71e24511f..e86646da4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -179,7 +179,7 @@ before_install: sleep 15 # we are lucky that the out test image has a small bug and always prints a warning. We use this to grep if the tool was running with SINGULARITY # WARNING Bind file destination does not exist in container /usr/local/var/singularity/mnt/overlay/final/etc/... - parsec datasets show_dataset $OUTPUT_ID | jq .misc_info | grep singularity + parsec jobs show_job --full_details $OUTPUT_ID | jq .stderr | grep singularity fi From 7ec499efa27a36312000b8f708a9edc6612f3100 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 15:32:36 +0100 Subject: [PATCH 104/116] Need to delay setting of GALAXY_CONFIG_TOOL_CONFIG_FILE --- .travis.yml | 3 +++ compose/.env_slurm_singularity | 2 +- compose/.env_slurm_singularity2 | 23 +++++++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 compose/.env_slurm_singularity2 diff --git a/.travis.yml b/.travis.yml index e86646da4..3aced9433 100644 --- a/.travis.yml +++ b/.travis.yml @@ -169,6 +169,9 @@ before_install: sudo curl -L -o /export/cat_tool_conf.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat_tool_conf.xml sudo curl -L -o /export/cat.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat.xml + rm .env + ln -sf .env_slurm_singularity2 .env + docker-compose up -d sleep 60 diff --git a/compose/.env_slurm_singularity b/compose/.env_slurm_singularity index 0dcd70a00..b3481110e 100644 --- a/compose/.env_slurm_singularity +++ b/compose/.env_slurm_singularity @@ -15,7 +15,7 @@ GALAXY_RUNNERS_ENABLE_SLURM=True # With this Galaxy will search for mulled based images (in /export/database/container_images/singularity/mulled/) GALAXY_CONFIG_ENABLE_BETA_MULLED_CONTAINERS=True -GALAXY_CONFIG_TOOL_CONFIG_FILE=config/tool_conf.xml.sample,config/shed_tool_conf.xml,/export/cat_tool_conf.xml +GALAXY_CONFIG_TOOL_CONFIG_FILE=config/tool_conf.xml.sample,config/shed_tool_conf.xml GALAXY_SINGULARITY_VOLUMES=$tool_directory:rw,$working_directory:rw,$default_file_path:rw diff --git a/compose/.env_slurm_singularity2 b/compose/.env_slurm_singularity2 new file mode 100644 index 000000000..0dcd70a00 --- /dev/null +++ b/compose/.env_slurm_singularity2 @@ -0,0 +1,23 @@ +# This file is setting up a composed Galaxy instance with support +# for submitting SLURM. +# Docker compose will use parameters specified in an `.env` file +# next to the docker-compose.yml file. +# We recommend to symlink this file and play around with different +# Galaxy deployments. +# ln -sf .env_slurm .env + +TAG=18.01 + +# default destination is our SLURM cluster with Singularity enabled +GALAXY_DESTINATIONS_DEFAULT=slurm_cluster_singularity +GALAXY_RUNNERS_ENABLE_SLURM=True + +# With this Galaxy will search for mulled based images (in /export/database/container_images/singularity/mulled/) +GALAXY_CONFIG_ENABLE_BETA_MULLED_CONTAINERS=True + +GALAXY_CONFIG_TOOL_CONFIG_FILE=config/tool_conf.xml.sample,config/shed_tool_conf.xml,/export/cat_tool_conf.xml + +GALAXY_SINGULARITY_VOLUMES=$tool_directory:rw,$working_directory:rw,$default_file_path:rw + +GALAXY_CONFIG_CLEANUP_JOB=onsuccess +GALAXY_PORT=8080 From 27ee4ab0fce3bc4c66366ef075b5816b6e36ceea Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 17:07:45 +0100 Subject: [PATCH 105/116] create needed dirs/files for condor --- compose/galaxy-base/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile index 8c1e1beb3..51ade8aa6 100644 --- a/compose/galaxy-base/Dockerfile +++ b/compose/galaxy-base/Dockerfile @@ -81,6 +81,10 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \ rm -rf /tmp/download && \ rm -rf ~/.cache/ +RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \ + mkdir -p /var/run/condor/ /var/lock/condor/ && \ + chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/ + RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \ useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" $GALAXY_USER && \ mkdir $EXPORT_DIR $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $GALAXY_LOGS_DIR $EXPORT_DIR From 2c0be5203b4559586c6476eafe75feaea817866c Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 17:42:26 +0100 Subject: [PATCH 106/116] update tag --- compose/.env_htcondor_docker | 2 +- compose/.env_k8_native | 2 +- compose/.env_slurm | 2 +- compose/.env_slurm_singularity | 2 +- compose/.env_slurm_singularity2 | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compose/.env_htcondor_docker b/compose/.env_htcondor_docker index 12182a726..e082eb769 100644 --- a/compose/.env_htcondor_docker +++ b/compose/.env_htcondor_docker @@ -6,7 +6,7 @@ # Galaxy deployments. # ln -sf .env_htcondor_docker .env -TAG=18.01 +TAG=18.09 GALAXY_DESTINATIONS_DEFAULT=condor_cluster diff --git a/compose/.env_k8_native b/compose/.env_k8_native index dccbab79a..f1c4f7c63 100644 --- a/compose/.env_k8_native +++ b/compose/.env_k8_native @@ -4,4 +4,4 @@ GALAXY_DESTINATIONS_DOCKER_DEFAULT=k8s_default GALAXY_DESTINATIONS_NO_DOCKER_DEFAULT=local_no_container GALAXY_CONFIG_CONTAINERS_RESOLVERS_CONFIG_FILE=/export/config/container_resolvers_conf.xml -TAG=18.01 +TAG=18.09 diff --git a/compose/.env_slurm b/compose/.env_slurm index 7cd741493..11d8abf70 100644 --- a/compose/.env_slurm +++ b/compose/.env_slurm @@ -6,7 +6,7 @@ # Galaxy deployments. # ln -sf .env_slurm .env -TAG=18.01 +TAG=18.09 # default destination is our SLURM cluster GALAXY_DESTINATIONS_DEFAULT=slurm_cluster diff --git a/compose/.env_slurm_singularity b/compose/.env_slurm_singularity index b3481110e..926b6c534 100644 --- a/compose/.env_slurm_singularity +++ b/compose/.env_slurm_singularity @@ -6,7 +6,7 @@ # Galaxy deployments. # ln -sf .env_slurm .env -TAG=18.01 +TAG=18.09 # default destination is our SLURM cluster with Singularity enabled GALAXY_DESTINATIONS_DEFAULT=slurm_cluster_singularity diff --git a/compose/.env_slurm_singularity2 b/compose/.env_slurm_singularity2 index 0dcd70a00..83c6d3134 100644 --- a/compose/.env_slurm_singularity2 +++ b/compose/.env_slurm_singularity2 @@ -6,7 +6,7 @@ # Galaxy deployments. # ln -sf .env_slurm .env -TAG=18.01 +TAG=18.09 # default destination is our SLURM cluster with Singularity enabled GALAXY_DESTINATIONS_DEFAULT=slurm_cluster_singularity From 40b74acd413bcaae64f527bb980310b9dcac91e6 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 17:43:50 +0100 Subject: [PATCH 107/116] latest kubernetes needs go >=1.11.1 --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3aced9433..a86a92182 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,10 +45,10 @@ before_install: if [ "${KUBE}" ] then # setup k8s, we will do this before building Galaxy because it takes some time and hopefully we can do both in prallel - gimme 1.8 - source ~/.gimme/envs/go1.8.env - sudo ln -s /home/travis/.gimme/versions/go1.7.linux.amd64/bin/gofmt /usr/bin/gofmt - sudo ln -s /home/travis/.gimme/versions/go1.7.linux.amd64/bin/go /usr/bin/go + gimme 1.11.1 + source ~/.gimme/envs/go1.11.1.env + sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/gofmt /usr/bin/gofmt + sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/go /usr/bin/go go version mkdir ../kubernetes wget -q -O - https://github.com/kubernetes/kubernetes/archive/master.tar.gz | tar xzf - --strip-components=1 -C ../kubernetes From 7beba07a3f3dbccdd80af95eed372a95d793971e Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 17:44:55 +0100 Subject: [PATCH 108/116] fix path problem (not sure if sudo is really needed) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a86a92182..22f1923b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ before_install: sudo ln -s `pwd`/third_party/etcd/etcd /usr/bin/etcd sudo ln -s `pwd`/third_party/etcd/etcdctl /usr/bin/etcdctl # this needs to run in backgroud later, for now try to see the output - sudo ./hack/local-up-cluster.sh & + ./hack/local-up-cluster.sh & cd ../docker-galaxy-stable fi From bb3083d95c27f6975d7ae02ae899bfaf418f2202 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 18:53:41 +0100 Subject: [PATCH 109/116] longer timeout --- test/bioblend/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index 381590641..aeefa876d 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -11,6 +11,7 @@ then export TOX_ENV=py27 ; export BIOBLEND_GALAXY_API_KEY=admin ; export BIOBLEND_GALAXY_URL=http://galaxy ; + export BIOBLEND_TEST_JOB_TIMEOUT="120"; cd /home/galaxy/bioblend-master ; tox -e $TOX_ENV -- -k "not test_download_dataset and not test_upload_from_galaxy_filesystem and not test_get_datasets and not test_datasets_from_fs and not test_existing_history and not test_new_history and not test_params and not test_tool_dependency_install and not test_download_history and not test_export_and_download"' From 83e2f2458b2ded8dab50bc5e7d272c56085f086d Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Tue, 18 Dec 2018 18:54:04 +0100 Subject: [PATCH 110/116] Longer timeout --- test/bioblend/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile index 6e44daf75..2374df727 100644 --- a/test/bioblend/Dockerfile +++ b/test/bioblend/Dockerfile @@ -12,7 +12,8 @@ RUN wget https://github.com/galaxyproject/bioblend/archive/master.tar.gz && tar ENV TOX_ENV=py27 \ BIOBLEND_GALAXY_API_KEY=admin \ - BIOBLEND_GALAXY_URL=http://galaxy + BIOBLEND_GALAXY_URL=http://galaxy \ + BIOBLEND_TEST_JOB_TIMEOUT="120" CMD /bin/bash -c "export PATH=/tool_deps/_conda/bin/:$PATH && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -k 'not test_download_dataset and not test_upload_from_galaxy_filesystem and not test_get_datasets and not test_datasets_from_fs and not test_tool_dependency_install and not test_download_history and not test_export_and_download'" From 1cf32151926fa670f8b6e9f8e61e01b68f22c810 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 19 Dec 2018 10:01:47 +0100 Subject: [PATCH 111/116] fix ignored BIOBLEND_TEST_JOB_TIMEOUT --- test/bioblend/Dockerfile | 3 ++- test/bioblend/test.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile index 2374df727..fb353d660 100644 --- a/test/bioblend/Dockerfile +++ b/test/bioblend/Dockerfile @@ -8,7 +8,8 @@ RUN wget https://github.com/galaxyproject/bioblend/archive/master.tar.gz && tar export PATH=/tool_deps/_conda/bin/:$PATH && . activate galaxy_env && \ pip install --upgrade "tox>=1.8.0" "pep8<=1.6.2" && \ python setup.py install && \ - sed -i.bak "s/commands.*$/commands =/" tox.ini + sed -i.bak "s/commands.*$/commands =/" tox.ini && \ + sed -i.bak2 "s/GALAXY_VERSION/GALAXY_VERSION BIOBLEND_TEST_JOB_TIMEOUT/" tox.ini ENV TOX_ENV=py27 \ BIOBLEND_GALAXY_API_KEY=admin \ diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index aeefa876d..fa683b8b3 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -8,6 +8,7 @@ then pip install --upgrade "tox>=1.8.0" "pep8<=1.6.2" ; python setup.py install ; sed -i.bak "s/commands.*$/commands =/" tox.ini ; + sed -i.bak2 "s/GALAXY_VERSION/GALAXY_VERSION BIOBLEND_TEST_JOB_TIMEOUT/" tox.ini ; export TOX_ENV=py27 ; export BIOBLEND_GALAXY_API_KEY=admin ; export BIOBLEND_GALAXY_URL=http://galaxy ; From 76185aa6f96b0cc29399c981bbb693b31e5e9402 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 19 Dec 2018 11:06:53 +0100 Subject: [PATCH 112/116] wait more!? --- test/bioblend/Dockerfile | 2 +- test/bioblend/test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile index fb353d660..2dccfcd76 100644 --- a/test/bioblend/Dockerfile +++ b/test/bioblend/Dockerfile @@ -14,7 +14,7 @@ RUN wget https://github.com/galaxyproject/bioblend/archive/master.tar.gz && tar ENV TOX_ENV=py27 \ BIOBLEND_GALAXY_API_KEY=admin \ BIOBLEND_GALAXY_URL=http://galaxy \ - BIOBLEND_TEST_JOB_TIMEOUT="120" + BIOBLEND_TEST_JOB_TIMEOUT="240" CMD /bin/bash -c "export PATH=/tool_deps/_conda/bin/:$PATH && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -k 'not test_download_dataset and not test_upload_from_galaxy_filesystem and not test_get_datasets and not test_datasets_from_fs and not test_tool_dependency_install and not test_download_history and not test_export_and_download'" diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index fa683b8b3..1f84d7add 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -12,7 +12,7 @@ then export TOX_ENV=py27 ; export BIOBLEND_GALAXY_API_KEY=admin ; export BIOBLEND_GALAXY_URL=http://galaxy ; - export BIOBLEND_TEST_JOB_TIMEOUT="120"; + export BIOBLEND_TEST_JOB_TIMEOUT="240"; cd /home/galaxy/bioblend-master ; tox -e $TOX_ENV -- -k "not test_download_dataset and not test_upload_from_galaxy_filesystem and not test_get_datasets and not test_datasets_from_fs and not test_existing_history and not test_new_history and not test_params and not test_tool_dependency_install and not test_download_history and not test_export_and_download"' From e5b586febeb1b1f878053b1243d89ddd82670981 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 19 Dec 2018 14:00:44 +0100 Subject: [PATCH 113/116] update kompose --- .travis.yml | 2 +- compose/galaxy-init/Dockerfile | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 22f1923b5..27f98763c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -107,7 +107,7 @@ before_install: cd ${COMPOSE_DIR} ln -sf .env_k8_native .env - curl -L https://github.com/kubernetes-incubator/kompose/releases/download/v0.5.0/kompose-linux-amd64 -o kompose + curl -L https://github.com/kubernetes-incubator/kompose/releases/download/v1.17.1/kompose-linux-amd64 -o kompose chmod +x kompose sudo mv ./kompose /usr/bin/kompose fi diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile index b1b2eab7c..975212a12 100644 --- a/compose/galaxy-init/Dockerfile +++ b/compose/galaxy-init/Dockerfile @@ -65,11 +65,6 @@ RUN ansible-playbook /ansible/provision.yml \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR -#--extra-vars galaxy_extras_config_container_resolution=True \ -#--extra-vars container_resolution_explicit=True \ -#--extra-vars container_resolution_cached_mulled=False \ -#--extra-vars container_resolution_build_mulled=False \ - # The following commands will be executed as User galaxy USER galaxy From e7b0a306cd61c5fd1e47a82469a0f4d63ec01832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Fri, 21 Dec 2018 19:51:55 +0100 Subject: [PATCH 114/116] shift deprecation warning to 19.05 as I failed to release this version in time --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c4d645f19..fb9025966 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ [![docker image stats](https://images.microbadger.com/badges/image/bgruening/galaxy-stable.svg)](https://microbadger.com/images/bgruening/galaxy-stable "Get your own image badge on microbadger.com") -:information_source: `18.09` will be the last release which is based on `ubuntu:14.04` and -PostgreSQL 9.3. We will migrate to `ubuntu:18.04` and a newer PostgreSQL version in `19.01`. Furthermore, we will not +:information_source: `19.01` will be the last release which is based on `ubuntu:14.04` and +PostgreSQL 9.3. We will migrate to `ubuntu:18.04` and a newer PostgreSQL version in `19.05`. Furthermore, we will not support old Galaxy tool dependencies. These are deprecated since a few years and we think its time to remove support for this in the default installation. You can install all needed packages manually to enable support again, though. -In short, with 19.01: +In short, with 19.05: * Galaxy tool dependencies will not be supported by default * `ubuntu:18.04` will be the new base image * a new PostgreSQL version means you need to migrate any old database to the new version From 392d463056706995cd12a4ea029b6400baeb60db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Fri, 21 Dec 2018 20:06:07 +0100 Subject: [PATCH 115/116] I don't feel good to install beta versions in stable releases, even if it is 10times faster --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index e28b3761a..f6995e17c 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -105,7 +105,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86 echo "conda activate base" >> $GALAXY_HOME/.bashrc && \ export PATH=/tool_deps/_conda/bin/:$PATH && \ conda install virtualenv pip && \ - conda install conda-canary::conda=4.6 && \ + # conda install conda-canary::conda=4.6 && \ chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh && \ mkdir $GALAXY_ROOT && \ wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \ From 4e63fdf227f0dc0e4a6110259a90a544e356fdaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Fri, 21 Dec 2018 20:12:55 +0100 Subject: [PATCH 116/116] add news entry --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index fb9025966..59923ce6b 100644 --- a/README.md +++ b/README.md @@ -826,6 +826,9 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro - Singularity support and tests (compose only) - more work on K8s support and testing (@jmchilton) - using .env files to configure the compose setup for SLURM, Condor, K8s, SLURM-Singularity, Condor-Docker +- 18.09: + - new and more powerful orchestration build script (build-orchestration-images.sh) by @pcm32 + - a lot of bug-fixes to the compose setup by @abretaud # Support & Bug Reports [[toc]](#toc)