Merge pull request #464 from abretaud/1804

18.09 update

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "galaxy/roles/galaxyproject.galaxyextras"]
 	path = galaxy/roles/galaxyprojectdotorg.galaxyextras
 	url = https://github.com/galaxyproject/ansible-galaxy-extras
-        branch = 18.01
+        branch = 18.09
 [submodule "galaxy/roles/galaxy-postgresql"]
 	path = galaxy/roles/galaxy-postgresql
 	url = https://github.com/galaxyproject/ansible-postgresql

.travis.yml

@@ -45,10 +45,10 @@ before_install:
     if [ "${KUBE}" ]
     then
       # setup k8s, we will do this before building Galaxy because it takes some time and hopefully we can do both in prallel
-      gimme 1.8
-      source ~/.gimme/envs/go1.8.env
-      sudo ln -s /home/travis/.gimme/versions/go1.7.linux.amd64/bin/gofmt /usr/bin/gofmt
-      sudo ln -s /home/travis/.gimme/versions/go1.7.linux.amd64/bin/go /usr/bin/go
+      gimme 1.11.1
+      source ~/.gimme/envs/go1.11.1.env
+      sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/gofmt /usr/bin/gofmt
+      sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/go /usr/bin/go
       go version
       mkdir ../kubernetes
       wget -q -O - https://github.com/kubernetes/kubernetes/archive/master.tar.gz | tar xzf - --strip-components=1 -C ../kubernetes
@@ -59,7 +59,7 @@ before_install:
       sudo ln -s `pwd`/third_party/etcd/etcd /usr/bin/etcd
       sudo ln -s `pwd`/third_party/etcd/etcdctl /usr/bin/etcdctl
       # this needs to run in backgroud later, for now try to see the output
-      sudo ./hack/local-up-cluster.sh &
+      ./hack/local-up-cluster.sh &
       cd ../docker-galaxy-stable
     fi
 
@@ -103,7 +103,11 @@ before_install:
   - |
     if [ "${KUBE}" ]
     then
-        curl -L https://github.com/kubernetes-incubator/kompose/releases/download/v0.5.0/kompose-linux-amd64 -o kompose
+        # The compose file recognises ENV vars to change the defaul behavior
+        cd ${COMPOSE_DIR}
+        ln -sf .env_k8_native .env
+
+        curl -L https://github.com/kubernetes-incubator/kompose/releases/download/v1.17.1/kompose-linux-amd64 -o kompose
         chmod +x kompose
         sudo mv ./kompose /usr/bin/kompose
     fi
@@ -165,6 +169,9 @@ before_install:
             sudo curl -L -o /export/cat_tool_conf.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat_tool_conf.xml
             sudo curl -L -o /export/cat.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat.xml
 
+            rm .env
+            ln -sf .env_slurm_singularity2 .env
+
             docker-compose up -d
             sleep 60
 
@@ -175,7 +182,7 @@ before_install:
             sleep 15
             # we are lucky that the out test image has a small bug and always prints a warning. We use this to grep if the tool was running with SINGULARITY
             # WARNING Bind file destination does not exist in container /usr/local/var/singularity/mnt/overlay/final/etc/...
-            parsec datasets show_dataset $OUTPUT_ID | jq .misc_info | grep singularity
+            parsec jobs show_job --full_details $OUTPUT_ID | jq .stderr | grep singularity
 
         fi
 

compose/.env_htcondor_docker

@@ -6,7 +6,7 @@
 # Galaxy deployments.
 # ln -sf .env_htcondor_docker .env
 
-TAG=18.01
+TAG=18.09
 
 GALAXY_DESTINATIONS_DEFAULT=condor_cluster
 

compose/.env_k8_native

@@ -4,4 +4,4 @@ GALAXY_DESTINATIONS_DOCKER_DEFAULT=k8s_default
 GALAXY_DESTINATIONS_NO_DOCKER_DEFAULT=local_no_container
 GALAXY_CONFIG_CONTAINERS_RESOLVERS_CONFIG_FILE=/export/config/container_resolvers_conf.xml
 
-TAG=18.01
+TAG=18.09

compose/.env_slurm

@@ -6,7 +6,7 @@
 # Galaxy deployments.
 # ln -sf .env_slurm .env
 
-TAG=18.01
+TAG=18.09
 
 # default destination is our SLURM cluster
 GALAXY_DESTINATIONS_DEFAULT=slurm_cluster

compose/.env_slurm_singularity

@@ -6,7 +6,7 @@
 # Galaxy deployments.
 # ln -sf .env_slurm .env
 
-TAG=18.01
+TAG=18.09
 
 # default destination is our SLURM cluster with Singularity enabled
 GALAXY_DESTINATIONS_DEFAULT=slurm_cluster_singularity
@@ -15,7 +15,7 @@ GALAXY_RUNNERS_ENABLE_SLURM=True
 # With this Galaxy will search for mulled based images (in /export/database/container_images/singularity/mulled/)
 GALAXY_CONFIG_ENABLE_BETA_MULLED_CONTAINERS=True
 
-GALAXY_CONFIG_TOOL_CONFIG_FILE=config/tool_conf.xml.sample,config/shed_tool_conf.xml,/export/cat_tool_conf.xml
+GALAXY_CONFIG_TOOL_CONFIG_FILE=config/tool_conf.xml.sample,config/shed_tool_conf.xml
 
 GALAXY_SINGULARITY_VOLUMES=$tool_directory:rw,$working_directory:rw,$default_file_path:rw
 

compose/.env_slurm_singularity2

@@ -0,0 +1,23 @@
+# This file is setting up a composed Galaxy instance with support
+# for submitting SLURM.
+# Docker compose will use parameters specified in an `.env` file
+# next to the docker-compose.yml file.
+# We recommend to symlink this file and play around with different
+# Galaxy deployments.
+# ln -sf .env_slurm .env
+
+TAG=18.09
+
+# default destination is our SLURM cluster with Singularity enabled
+GALAXY_DESTINATIONS_DEFAULT=slurm_cluster_singularity
+GALAXY_RUNNERS_ENABLE_SLURM=True
+
+# With this Galaxy will search for mulled based images (in /export/database/container_images/singularity/mulled/)
+GALAXY_CONFIG_ENABLE_BETA_MULLED_CONTAINERS=True
+
+GALAXY_CONFIG_TOOL_CONFIG_FILE=config/tool_conf.xml.sample,config/shed_tool_conf.xml,/export/cat_tool_conf.xml
+
+GALAXY_SINGULARITY_VOLUMES=$tool_directory:rw,$working_directory:rw,$default_file_path:rw
+
+GALAXY_CONFIG_CLEANUP_JOB=onsuccess
+GALAXY_PORT=8080

compose/build-orchestration-images.sh

@@ -121,7 +121,7 @@ DOCKER_REPO=${CONTAINER_REGISTRY:-}
 DOCKER_USER=${CONTAINER_USER:-pcm32}
 
 ANSIBLE_REPO=${ANSIBLE_REPO:-galaxyproject/ansible-galaxy-extras}
-ANSIBLE_RELEASE=${ANSIBLE_RELEASE:-master}
+ANSIBLE_RELEASE=${ANSIBLE_RELEASE:-18.09}
 
 GALAXY_VERSION=${GALAXY_VERSION:-18.09}
 
@@ -222,7 +222,7 @@ K8S_ANSIBLE_TAGS=""
 if $BUILD_FOR_K8S; then
   K8S_ANSIBLE_TAGS=,k8,k8s
 fi
-docker build $NO_CACHE --build-arg GALAXY_ANSIBLE_TAGS=supervisor,startup,scripts,nginx$K8S_ANSIBLE_TAGS -t $GALAXY_WEB_TAG -f galaxy-web/$DOCKERFILE_WEB galaxy-web/
+docker build $NO_CACHE --build-arg GALAXY_ANSIBLE_TAGS=supervisor,startup,scripts,nginx,cvmfs$K8S_ANSIBLE_TAGS -t $GALAXY_WEB_TAG -f galaxy-web/$DOCKERFILE_WEB galaxy-web/
 if $DOCKER_PUSH_ENABLED; then
   docker push $GALAXY_WEB_TAG
 fi

compose/docker-compose.yml

@@ -324,7 +324,6 @@ services:
     volumes:
         # This is the directory where all your files from Galaxy will be stored
         # on your host system
-        - ${EXPORT_DIR:-/export}/galaxy-central/config/galaxy.yml.sample:/etc/galaxy/galaxy.yml
         - ${EXPORT_DIR:-/export}/:/export/
         - /var/run/docker.sock:/var/run/docker.sock
     depends_on:

compose/dumpsql.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-TAG=18.09
+TAG=v18.09
 
 # Sets the image of postgres to use
 POSTGRES=postgres:9.6.5

compose/galaxy-base/Dockerfile

@@ -81,6 +81,10 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
     rm -rf /tmp/download && \
     rm -rf ~/.cache/
 
+RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \
+    mkdir -p /var/run/condor/ /var/lock/condor/ && \
+    chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/
+
 RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \
     useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" $GALAXY_USER && \
     mkdir $EXPORT_DIR $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $GALAXY_LOGS_DIR $EXPORT_DIR

compose/galaxy-htcondor-base/Dockerfile

@@ -18,3 +18,8 @@ RUN wget https://dl.influxdata.com/telegraf/releases/telegraf-1.5.0_linux_amd64.
     rm telegraf-1.5.0_linux_amd64.tar.gz
 
 ADD telegraf.conf /etc/telegraf/telegraf.conf
+
+
+RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \
+    mkdir -p /var/run/condor/ /var/lock/condor/ && \
+    chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/

compose/galaxy-htcondor-executor/startup.sh

@@ -30,8 +30,6 @@ SCHED_NAME = $CONDOR_HOST
 " > /etc/condor/condor_config.local
 
 /usr/bin/telegraf --config /etc/telegraf/telegraf.conf &
-sudo -u condor touch /var/log/condor/StartLog
-sudo -u condor touch /var/log/condor/StarterLog
 tail -f -n 1000 /var/log/condor/StartLog /var/log/condor/StarterLog &
 
 # Mysterious bug? Docker doesn't output its version as condor user if this is not executed before

compose/galaxy-init/Dockerfile

@@ -65,11 +65,6 @@ RUN ansible-playbook /ansible/provision.yml \
     apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
     chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR
 
-#--extra-vars galaxy_extras_config_container_resolution=True \
-#--extra-vars container_resolution_explicit=True \
-#--extra-vars container_resolution_cached_mulled=False \
-#--extra-vars container_resolution_build_mulled=False \
-
 
 # The following commands will be executed as User galaxy
 USER galaxy
@@ -98,7 +93,7 @@ RUN export PATH=/tool_deps/_conda/bin/:$PATH && \
     # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments
     cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \
     npm install && \
-    ##cd $GALAXY_ROOT/client && yarn install --network-timeout 120000 --check-files && yarn run build-production-maps && \
+    cd $GALAXY_ROOT/client && yarn install --network-timeout 120000 --check-files && yarn run build-production-maps && \
     rm -rf /home/galaxy/.cache/
 
 # Switch back to User root

compose/galaxy-init/startup.sh

@@ -30,13 +30,16 @@ done
 
 # Optional, might not work
 {
-  if [ ! -d "/export/var/lib/docker" ]
+  if [ -d "/var/lib/docker" ]
   then
-    echo "Moving to /export/var/lib/docker"
-    mkdir -p /export/var/lib/
-    mv /var/lib/docker /export/var/lib/docker
-    chown -R $GALAXY_UID:$GALAXY_GID /export/var/lib/docker
-  fi
+      if [ ! -d "/export/var/lib/docker" ]
+      then
+        echo "Moving to /export/var/lib/docker"
+        mkdir -p /export/var/lib/
+        mv /var/lib/docker /export/var/lib/docker
+        chown -R $GALAXY_UID:$GALAXY_GID /export/var/lib/docker
+      fi
+    fi
 } || echo "Moving docker lib failed, this is not a fatal error"
 
 echo "Initialization complete"