diff --git a/.gitmodules b/.gitmodules
index e6612923a..67081dc46 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,7 +1,7 @@
[submodule "galaxy/roles/galaxyproject.galaxyextras"]
path = galaxy/roles/galaxyprojectdotorg.galaxyextras
url = https://github.com/galaxyproject/ansible-galaxy-extras
- branch = 18.01
+ branch = 18.09
[submodule "galaxy/roles/galaxy-postgresql"]
path = galaxy/roles/galaxy-postgresql
url = https://github.com/galaxyproject/ansible-postgresql
diff --git a/.travis.yml b/.travis.yml
index b263b1a4e..27f98763c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -45,10 +45,10 @@ before_install:
if [ "${KUBE}" ]
then
# setup k8s, we will do this before building Galaxy because it takes some time and hopefully we can do both in prallel
- gimme 1.8
- source ~/.gimme/envs/go1.8.env
- sudo ln -s /home/travis/.gimme/versions/go1.7.linux.amd64/bin/gofmt /usr/bin/gofmt
- sudo ln -s /home/travis/.gimme/versions/go1.7.linux.amd64/bin/go /usr/bin/go
+ gimme 1.11.1
+ source ~/.gimme/envs/go1.11.1.env
+ sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/gofmt /usr/bin/gofmt
+ sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/go /usr/bin/go
go version
mkdir ../kubernetes
wget -q -O - https://github.com/kubernetes/kubernetes/archive/master.tar.gz | tar xzf - --strip-components=1 -C ../kubernetes
@@ -59,7 +59,7 @@ before_install:
sudo ln -s `pwd`/third_party/etcd/etcd /usr/bin/etcd
sudo ln -s `pwd`/third_party/etcd/etcdctl /usr/bin/etcdctl
# this needs to run in backgroud later, for now try to see the output
- sudo ./hack/local-up-cluster.sh &
+ ./hack/local-up-cluster.sh &
cd ../docker-galaxy-stable
fi
@@ -103,7 +103,11 @@ before_install:
- |
if [ "${KUBE}" ]
then
- curl -L https://github.com/kubernetes-incubator/kompose/releases/download/v0.5.0/kompose-linux-amd64 -o kompose
+ # The compose file recognises ENV vars to change the defaul behavior
+ cd ${COMPOSE_DIR}
+ ln -sf .env_k8_native .env
+
+ curl -L https://github.com/kubernetes-incubator/kompose/releases/download/v1.17.1/kompose-linux-amd64 -o kompose
chmod +x kompose
sudo mv ./kompose /usr/bin/kompose
fi
@@ -120,7 +124,11 @@ before_install:
INSTALL_REPO_ARG="--galaxy-url http://localhost:80"
SAMPLE_TOOLS=/export/config/sample_tool_list.yaml
cd "$WORKING_DIR"
- ./buildlocal.sh
+ # For build script
+ export CONTAINER_REGISTRY=quay.io/
+ export CONTAINER_USER=bgruening
+ ./build-orchestration-images.sh --no-push --condor --grafana --slurm --k8s
+ source ./tags-for-compose-to-source.sh
export COMPOSE_PROJECT_NAME=galaxy_compose
docker-compose up -d
@@ -138,7 +146,7 @@ before_install:
docker-compose stop galaxy-slurm
sleep 30
fi
-
+
if [ "${COMPOSE_SLURM}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ]
then
# turn down the htcondor services
@@ -150,7 +158,7 @@ before_install:
if [ "${COMPOSE_SLURM_SINGULARITY}" ]
then
# docker-compose is already started and has pre-populated the /export dir
- # we now turn it down again and copy in an example tool with tool_conf.xml and
+ # we now turn it down again and copy in an example tool with tool_conf.xml and
# a test singularity image. If we copy this from the beginning, the magic Docker Galax startup
# script will not work as it detects something in /export/
docker-compose down
@@ -161,6 +169,9 @@ before_install:
sudo curl -L -o /export/cat_tool_conf.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat_tool_conf.xml
sudo curl -L -o /export/cat.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat.xml
+ rm .env
+ ln -sf .env_slurm_singularity2 .env
+
docker-compose up -d
sleep 60
@@ -171,10 +182,10 @@ before_install:
sleep 15
# we are lucky that the out test image has a small bug and always prints a warning. We use this to grep if the tool was running with SINGULARITY
# WARNING Bind file destination does not exist in container /usr/local/var/singularity/mnt/overlay/final/etc/...
- parsec datasets show_dataset $OUTPUT_ID | jq .misc_info | grep singularity
+ parsec jobs show_job --full_details $OUTPUT_ID | jq .stderr | grep singularity
fi
-
+
docker-compose logs --tail 50
# Define start functions
docker_exec() {
@@ -243,6 +254,11 @@ script:
cd $TRAVIS_BUILD_DIR/test/gridengine/ && bash test.sh && cd $WORKING_DIR
fi
# Test Web api
+ - |
+ if [ "${COMPOSE_CONDOR_DOCKER}" ]
+ then
+ docker-compose logs --tail 50
+ fi
- curl -v --fail $BIOBLEND_GALAXY_URL/api/version
# Test self-signed HTTPS
@@ -255,7 +271,7 @@ script:
- date > time.txt && curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true
# Test FTP Server get
- curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD
-
+
# Test CVMFS
- docker_exec bash -c "service autofs start"
- docker_exec bash -c "cvmfs_config chksetup"
diff --git a/README.md b/README.md
index 64961185b..9d65468d6 100644
--- a/README.md
+++ b/README.md
@@ -5,6 +5,18 @@
 
[](https://microbadger.com/images/bgruening/galaxy-stable "Get your own image badge on microbadger.com")
+
+:information_source: `19.01` will be the last release which is based on `ubuntu:14.04` and
+PostgreSQL 9.3. We will migrate to `ubuntu:18.04` and a newer PostgreSQL version in `19.05`. Furthermore, we will not
+support old Galaxy tool dependencies. These are deprecated since a few years and we think its time to remove support
+for this in the default installation. You can install all needed packages manually to enable support again, though.
+
+In short, with 19.05:
+ * Galaxy tool dependencies will not be supported by default
+ * `ubuntu:18.04` will be the new base image
+ * a new PostgreSQL version means you need to migrate any old database to the new version
+
+
Galaxy Docker Image
===================
@@ -815,6 +827,9 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro
- Singularity support and tests (compose only)
- more work on K8s support and testing (@jmchilton)
- using .env files to configure the compose setup for SLURM, Condor, K8s, SLURM-Singularity, Condor-Docker
+- 18.09:
+ - new and more powerful orchestration build script (build-orchestration-images.sh) by @pcm32
+ - a lot of bug-fixes to the compose setup by @abretaud
# Support & Bug Reports [[toc]](#toc)
diff --git a/compose/.env_htcondor_docker b/compose/.env_htcondor_docker
index 12182a726..e082eb769 100644
--- a/compose/.env_htcondor_docker
+++ b/compose/.env_htcondor_docker
@@ -6,7 +6,7 @@
# Galaxy deployments.
# ln -sf .env_htcondor_docker .env
-TAG=18.01
+TAG=18.09
GALAXY_DESTINATIONS_DEFAULT=condor_cluster
diff --git a/compose/.env_k8_native b/compose/.env_k8_native
index dccbab79a..f1c4f7c63 100644
--- a/compose/.env_k8_native
+++ b/compose/.env_k8_native
@@ -4,4 +4,4 @@ GALAXY_DESTINATIONS_DOCKER_DEFAULT=k8s_default
GALAXY_DESTINATIONS_NO_DOCKER_DEFAULT=local_no_container
GALAXY_CONFIG_CONTAINERS_RESOLVERS_CONFIG_FILE=/export/config/container_resolvers_conf.xml
-TAG=18.01
+TAG=18.09
diff --git a/compose/.env_slurm b/compose/.env_slurm
index 7cd741493..11d8abf70 100644
--- a/compose/.env_slurm
+++ b/compose/.env_slurm
@@ -6,7 +6,7 @@
# Galaxy deployments.
# ln -sf .env_slurm .env
-TAG=18.01
+TAG=18.09
# default destination is our SLURM cluster
GALAXY_DESTINATIONS_DEFAULT=slurm_cluster
diff --git a/compose/.env_slurm_singularity b/compose/.env_slurm_singularity
index 0dcd70a00..926b6c534 100644
--- a/compose/.env_slurm_singularity
+++ b/compose/.env_slurm_singularity
@@ -6,7 +6,7 @@
# Galaxy deployments.
# ln -sf .env_slurm .env
-TAG=18.01
+TAG=18.09
# default destination is our SLURM cluster with Singularity enabled
GALAXY_DESTINATIONS_DEFAULT=slurm_cluster_singularity
@@ -15,7 +15,7 @@ GALAXY_RUNNERS_ENABLE_SLURM=True
# With this Galaxy will search for mulled based images (in /export/database/container_images/singularity/mulled/)
GALAXY_CONFIG_ENABLE_BETA_MULLED_CONTAINERS=True
-GALAXY_CONFIG_TOOL_CONFIG_FILE=config/tool_conf.xml.sample,config/shed_tool_conf.xml,/export/cat_tool_conf.xml
+GALAXY_CONFIG_TOOL_CONFIG_FILE=config/tool_conf.xml.sample,config/shed_tool_conf.xml
GALAXY_SINGULARITY_VOLUMES=$tool_directory:rw,$working_directory:rw,$default_file_path:rw
diff --git a/compose/.env_slurm_singularity2 b/compose/.env_slurm_singularity2
new file mode 100644
index 000000000..83c6d3134
--- /dev/null
+++ b/compose/.env_slurm_singularity2
@@ -0,0 +1,23 @@
+# This file is setting up a composed Galaxy instance with support
+# for submitting SLURM.
+# Docker compose will use parameters specified in an `.env` file
+# next to the docker-compose.yml file.
+# We recommend to symlink this file and play around with different
+# Galaxy deployments.
+# ln -sf .env_slurm .env
+
+TAG=18.09
+
+# default destination is our SLURM cluster with Singularity enabled
+GALAXY_DESTINATIONS_DEFAULT=slurm_cluster_singularity
+GALAXY_RUNNERS_ENABLE_SLURM=True
+
+# With this Galaxy will search for mulled based images (in /export/database/container_images/singularity/mulled/)
+GALAXY_CONFIG_ENABLE_BETA_MULLED_CONTAINERS=True
+
+GALAXY_CONFIG_TOOL_CONFIG_FILE=config/tool_conf.xml.sample,config/shed_tool_conf.xml,/export/cat_tool_conf.xml
+
+GALAXY_SINGULARITY_VOLUMES=$tool_directory:rw,$working_directory:rw,$default_file_path:rw
+
+GALAXY_CONFIG_CLEANUP_JOB=onsuccess
+GALAXY_PORT=8080
diff --git a/compose/build-orchestration-images.sh b/compose/build-orchestration-images.sh
new file mode 100755
index 000000000..1002f4fab
--- /dev/null
+++ b/compose/build-orchestration-images.sh
@@ -0,0 +1,296 @@
+#!/bin/bash
+# vi: fdm=marker
+
+set -e
+
+### Constants ###
+
+SCRIPT_NAME="${0##*/}"
+TRUE='true'
+FALSE='false'
+DFT_DOCKER_PUSH_ENABLED=$TRUE
+
+### Functions ###
+
+function log() {
+ echo -e "$@" >&2
+}
+
+function error {
+ local msg=$1
+ local code=$2
+ [[ -z $code ]] && code=1
+ log "[ERROR] $msg"
+
+ exit $code
+}
+
+
+function debug {
+ local msg=$1
+ local level=$2
+ [[ -n $level ]] || level=1
+
+ [[ $DEBUG -lt $level ]] || log "[DEBUG] $msg"
+}
+
+function warning {
+ local msg=$1
+
+ log "[WARNING] ##### $msg #####"
+}
+
+function print_help {
+ (
+ echo "Usage: $SCRIPT_NAME [options]"
+ echo
+ echo "Build Galaxy docker compose images for different purposes."
+ echo
+ echo "Options:"
+ echo " -g, --debug Debug mode. [false]"
+ echo " -h, --help Print this help message."
+ echo " -p, --push Push docker images. You can also set environment variable DOCKER_PUSH_ENABLED to \"true\". [true]"
+ echo " +p, --no-push Do not push dockers. You can also set environment variable DOCKER_PUSH_ENABLED to \"false\"."
+ echo " --push-intermediate Also push intermediate images [false]"
+ echo " --no-cache Tell Docker not to use cached images. [false]"
+ echo " --init-tag Set the tag for the Galaxy Init Flavour container image."
+ echo " --postgres-tag Set the tag for the Galaxy Postgres container image."
+ echo " --proftpd-tag Set the tag for the Galaxy Proftpd container image."
+ echo " --web-tag Set the tag for the Galaxy Web k8s container image."
+ echo " --k8s Set settings for Kubernetes usage."
+ echo " --grafana Build grafana container."
+ echo " --condor Build condor containers."
+ echo " --slurm Build slurm containers."
+ echo " -u, --container-user Set the container user. You can also set the environment variable CONTAINER_USER."
+ echo " -r, --container-registry Set the container registry. You can also set the environment variable CONTAINER_REGISTRY."
+ echo
+ ) >&2
+}
+
+function read_args {
+
+ local args="$*" # save arguments for debugging purpose
+
+ # Read options
+ while [[ $# > 0 ]] ; do
+ shift_count=1
+ case $1 in
+ -g|--debug) DEBUG=$((DEBUG + 1)) ;;
+ -h|--help) print_help ; exit 0 ;;
+ -p|--push) DOCKER_PUSH_ENABLED=$TRUE ;;
+ +p|--no-push) DOCKER_PUSH_ENABLED=$FALSE ;;
+ --k8s) BUILD_FOR_K8S=$TRUE ;;
+ --condor) BUILD_FOR_CONDOR=$TRUE ;;
+ --slurm) BUILD_FOR_SLURM=$TRUE ;;
+ --grafana) BUILD_FOR_GRAFANA=$TRUE ;;
+ --push-intermediate) PUSH_INTERMEDIATE_IMAGES=$TRUE ;;
+ --no-cache) NO_CACHE="--no-cache" ;;
+ --postgres-tag) OVERRIDE_POSTGRES_TAG="$2" ; shift_count=2 ;;
+ --proftpd-tag) OVERRIDE_PROFTPD_TAG="$2" ; shift_count=2 ;;
+ --init-tag) OVERRIDE_GALAXY_INIT_PHENO_FLAVOURED_TAG="$2" ; shift_count=2 ;;
+ --web-tag) OVERRIDE_GALAXY_WEB_TAG="$2" ; shift_count=2 ;;
+ -u|--container-user) CONTAINER_USER="$2" ; shift_count=2 ;;
+ -r|--container-registry) CONTAINER_REGISTRY="$2" ; shift_count=2 ;;
+ *) error "Illegal option $1." 98 ;;
+ esac
+ shift $shift_count
+ done
+
+ # Debug messages
+ debug "Command line arguments: $args"
+ debug "Argument DEBUG=$DEBUG"
+ debug "Argument DOCKER_PUSH_ENABLED=$DOCKER_PUSH_ENABLED"
+ debug "Argument DOCKER_USER=$DOCKER_USER"
+ debug "Argument OVERRIDE_GALAXY_WEB_TAG=$OVERRIDE_GALAXY_WEB_TAG"
+ debug "Argument OVERRIDE_POSTGRES_TAG=$OVERRIDE_POSTGRES_TAG"
+ debug "Argument OVERRIDE_PROFTPD_TAG=$OVERRIDE_PROFTPD_TAG"
+ debug "shift_count=$shift_count"
+}
+
+### MAIN ###
+BUILD_FOR_K8S=$FALSE
+BUILD_FOR_CONDOR=$FALSE
+BUILD_FOR_GRAFANA=$FALSE
+BUILD_FOR_SLURM=$FALSE
+
+read_args "$@"
+
+DOCKER_PUSH_ENABLED=${DOCKER_PUSH_ENABLED:-$DFT_DOCKER_PUSH_ENABLED}
+
+DOCKER_REPO=${CONTAINER_REGISTRY:-}
+DOCKER_USER=${CONTAINER_USER:-pcm32}
+
+ANSIBLE_REPO=${ANSIBLE_REPO:-galaxyproject/ansible-galaxy-extras}
+ANSIBLE_RELEASE=${ANSIBLE_RELEASE:-18.09}
+
+GALAXY_VERSION=${GALAXY_VERSION:-18.09}
+
+# GALAXY_BASE_FROM_TO_REPLACE=${GALAXY_BASE_FROM_TO_REPLACE:-quay.io/bgruening/galaxy-base:$GALAXY_VERSION}
+GALAXY_BASE_FROM_TO_REPLACE=$(grep ^FROM galaxy-init/Dockerfile | awk '{ print $2 }') # init starts from base, so we get it from there.
+CONDOR_BASE_FROM_TO_REPLACE=quay.io/bgruening/galaxy-htcondor-base:$GALAXY_VERSION
+
+GALAXY_RELEASE=${GALAXY_RELEASE:-release_$GALAXY_VERSION}
+GALAXY_REPO=${GALAXY_REPO:-galaxyproject/galaxy}
+
+GALAXY_VER_FOR_POSTGRES=$GALAXY_VERSION
+# Uncomment to push intermediate images, otherwise only the images needed for the helm chart are pushed.
+#PUSH_INTERMEDIATE_IMAGES=yes
+
+# Set tags
+TAG=${GALAXY_TAG:-v$GALAXY_VERSION}
+
+# TODO This is PhenoMeNal Jenkins specific, should be removed at some point.
+if [[ -n ${CONTAINER_TAG_PREFIX:-} && -n ${BUILD_NUMBER:-} ]]; then
+ TAG=${CONTAINER_TAG_PREFIX}.${BUILD_NUMBER}
+fi
+
+if [[ -n "${DOCKER_REPO}" ]]; then
+ # Append slash, avoiding double slash
+ DOCKER_REPO="${DOCKER_REPO%/}/"
+fi
+
+GALAXY_BASE_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-base:$TAG
+GALAXY_INIT_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-init:$TAG
+GALAXY_WEB_TAG=${OVERRIDE_GALAXY_WEB_TAG:-$DOCKER_REPO$DOCKER_USER/galaxy-web:$TAG}
+
+if [[ -n "${OVERRIDE_POSTGRES_TAG:-}" ]]; then
+ POSTGRES_TAG="${OVERRIDE_POSTGRES_TAG}"
+else
+ PG_Dockerfile="galaxy-postgres/Dockerfile"
+
+ [[ -f "${PG_Dockerfile}" ]] || error "The galaxy-postgres Dockerfile is missing under galaxy-Postgres." 99
+ POSTGRES_VERSION=$(grep FROM "${PG_Dockerfile}" | awk -F":" '{ print $2 }')
+
+ POSTGRES_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-postgres:$POSTGRES_VERSION"_for_"$GALAXY_VER_FOR_POSTGRES
+fi
+
+if [[ -n "${OVERRIDE_PROFTPD_TAG:-}" ]]; then
+ PROFTPD_TAG="${OVERRIDE_PROFTPD_TAG}"
+else
+ PROFTPD_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-proftpd:for_galaxy_v$GALAXY_VER_FOR_POSTGRES
+fi
+
+CONDOR_BASE_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor-base:$TAG
+CONDOR_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor:$TAG
+CONDOR_EXEC_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-htcondor-executor:$TAG
+GRAFANA_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-grafana:$TAG
+SLURM_TAG=$DOCKER_REPO$DOCKER_USER/galaxy-slurm:$TAG
+### do work
+
+if [ -n $ANSIBLE_REPO ]
+ then
+ log "Making custom galaxy-base:$TAG from $ANSIBLE_REPO at $ANSIBLE_RELEASE"
+ docker build $NO_CACHE --build-arg ANSIBLE_REPO=$ANSIBLE_REPO --build-arg ANSIBLE_RELEASE=$ANSIBLE_RELEASE -t $GALAXY_BASE_TAG galaxy-base/
+ if [[ ! -z ${PUSH_INTERMEDIATE_IMAGES+x} ]];
+ then
+ log "Pushing intermediate image $DOCKER_REPO$DOCKER_USER/galaxy-base:$TAG"
+ docker push $GALAXY_BASE_TAG
+ fi
+fi
+
+
+if [ -n $GALAXY_REPO ]
+ then
+ log "Making custom galaxy-init:$TAG from $GALAXY_REPO at $GALAXY_RELEASE"
+ DOCKERFILE_INIT_1=Dockerfile
+ if [ -n $ANSIBLE_REPO ]
+ then
+ sed s+$GALAXY_BASE_FROM_TO_REPLACE+$GALAXY_BASE_TAG+ galaxy-init/Dockerfile > galaxy-init/Dockerfile_init
+ FROM=`grep ^FROM galaxy-init/Dockerfile_init | awk '{ print $2 }'`
+ log "Using FROM $FROM for galaxy init"
+ DOCKERFILE_INIT_1=Dockerfile_init
+ fi
+ docker build $NO_CACHE --build-arg GALAXY_REPO=$GALAXY_REPO --build-arg GALAXY_RELEASE=$GALAXY_RELEASE -t $GALAXY_INIT_TAG -f galaxy-init/$DOCKERFILE_INIT_1 galaxy-init/
+ if [[ "${DOCKER_PUSH_ENABLED:-}" = "true" ]]; then
+ log "Pushing image $GALAXY_INIT_TAG"
+ docker push $GALAXY_INIT_TAG
+ fi
+fi
+
+
+DOCKERFILE_WEB=Dockerfile
+if [ -n $GALAXY_REPO ]
+then
+ log "Making custom galaxy-web:$TAG from $GALAXY_REPO at $GALAXY_RELEASE"
+ GALAXY_BASE_FROM_TO_REPLACE=$(grep ^FROM galaxy-web/Dockerfile | awk '{ print $2 }')
+ sed s+$GALAXY_BASE_FROM_TO_REPLACE+$GALAXY_BASE_TAG+ galaxy-web/Dockerfile > galaxy-web/Dockerfile_web
+ FROM=$(grep ^FROM galaxy-web/Dockerfile_web | awk '{ print $2 }')
+ log "Using FROM $FROM for galaxy web"
+ DOCKERFILE_WEB=Dockerfile_web
+fi
+K8S_ANSIBLE_TAGS=""
+if $BUILD_FOR_K8S; then
+ K8S_ANSIBLE_TAGS=,k8,k8s
+fi
+docker build $NO_CACHE --build-arg GALAXY_ANSIBLE_TAGS=supervisor,startup,scripts,nginx,cvmfs$K8S_ANSIBLE_TAGS -t $GALAXY_WEB_TAG -f galaxy-web/$DOCKERFILE_WEB galaxy-web/
+if $DOCKER_PUSH_ENABLED; then
+ docker push $GALAXY_WEB_TAG
+fi
+
+# Build postgres
+docker build -t $POSTGRES_TAG -f galaxy-postgres/Dockerfile galaxy-postgres/
+if $DOCKER_PUSH_ENABLED; then
+ docker push $POSTGRES_TAG
+fi
+
+# Build proftpd
+docker build -t $PROFTPD_TAG -f galaxy-proftpd/Dockerfile galaxy-proftpd/
+if $DOCKER_PUSH_ENABLED; then
+ docker push $PROFTPD_TAG
+fi
+
+# Build condor
+if $BUILD_FOR_CONDOR; then
+ docker build $NO_CACHE -t $CONDOR_BASE_TAG galaxy-htcondor-base
+ sed s+$CONDOR_BASE_FROM_TO_REPLACE+$CONDOR_BASE_TAG+ galaxy-htcondor/Dockerfile > galaxy-htcondor/Dockerfile_condor
+ FROM=`grep ^FROM galaxy-htcondor/Dockerfile_condor | awk '{ print $2 }'`
+ log "Using FROM $FROM for condor"
+ docker build $NO_CACHE -t $CONDOR_TAG -f galaxy-htcondor/Dockerfile_condor galaxy-htcondor/
+ sed s+$CONDOR_BASE_FROM_TO_REPLACE+$CONDOR_BASE_TAG+ galaxy-htcondor-executor/Dockerfile > galaxy-htcondor-executor/Dockerfile_condor
+ FROM=`grep ^FROM galaxy-htcondor-executor/Dockerfile_condor | awk '{ print $2 }'`
+ log "Using FROM $FROM for condor-executor"
+ docker build $NO_CACHE -t $CONDOR_EXEC_TAG -f galaxy-htcondor-executor/Dockerfile_condor galaxy-htcondor-executor/
+ if $DOCKER_PUSH_ENABLED; then
+ docker push $CONDOR_TAG
+ docker push $CONDOR_EXEC_TAG
+ fi
+fi
+
+# Build for slurm
+if $BUILD_FOR_SLURM; then
+ docker build -t $SLURM_TAG ./galaxy-slurm
+fi
+
+# Build for grafana
+if $BUILD_FOR_GRAFANA; then
+ docker build -t $GRAFANA_TAG ./galaxy-grafana
+fi
+
+log "Relevant containers:"
+log "Web: $GALAXY_WEB_TAG"
+log "Init: $GALAXY_INIT_TAG"
+log "Postgres: $POSTGRES_TAG"
+log "Proftpd: $PROFTPD_TAG"
+if $BUILD_FOR_CONDOR; then
+ log "Condor: $CONDOR_TAG"
+ log "Condor-exec: $CONDOR_EXEC_TAG"
+fi
+if $BUILD_FOR_SLURM; then
+ log "Slurm: $SLURM_TAG"
+fi
+if $BUILD_FOR_GRAFANA; then
+ log "Grafana: $GRAFANA_TAG"
+fi
+
+log "Now build your own Galaxy init container starting FROM $GALAXY_INIT_TAG to add you own flavour, tools, workflows, etc."
+if $BUILD_FOR_K8S; then
+ log ""
+ log "For k8s: Once you have built your own init container use it within the galaxy-stable Helm chart at https://github.com/galaxyproject/galaxy-kubernetes together with:"
+ log " - Web: $GALAXY_WEB_TAG"
+ log " - Postgres: $POSTGRES_TAG"
+ log " - Proftpd: $PROFTPD_TAG"
+fi
+
+echo "export TAG="$(echo $GALAXY_WEB_TAG | awk -F':' '{print $2}') > tags-for-compose-to-source.sh
+echo "export TAG_POSTGRES="$(echo $POSTGRES_TAG | awk -F':' '{print $2}') >> tags-for-compose-to-source.sh
+echo "export TAG_PROFTPD="$(echo $PROFTPD_TAG | awk -F':' '{print $2}') >> tags-for-compose-to-source.sh
diff --git a/compose/buildlocal.sh b/compose/buildlocal.sh
index e651c5da3..0c6a47c82 100755
--- a/compose/buildlocal.sh
+++ b/compose/buildlocal.sh
@@ -1,17 +1,21 @@
#!/bin/bash
set -x -e
+echo "***************** Notice **************************************************"
+echo "This script is deprecated, please use build-orchestration-images.sh instead."
+echo "***************** Notice **************************************************"
+
ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras
-ANSIBLE_RELEASE=14c60b66f10326c7d158627b004d58e38ca503e4
+ANSIBLE_RELEASE=18.09
-GALAXY_RELEASE=release_18.05
+GALAXY_RELEASE=release_18.09
GALAXY_REPO=galaxyproject/galaxy
DOCKER_ADDITIONAL_BUILD_ARGS=""
#"--no-cache"
# For using latest simply leave this variable empty or set to ":latest". This should be the case on the master branch.
-TAG=":18.05"
+TAG=":18.09"
docker pull postgres
@@ -35,3 +39,7 @@ docker build -t quay.io/bgruening/galaxy-htcondor$TAG ./galaxy-htcondor
docker build -t quay.io/bgruening/galaxy-htcondor-executor$TAG ./galaxy-htcondor-executor
docker build -t quay.io/bgruening/galaxy-grafana$TAG ./galaxy-grafana
+
+echo "***************** Notice **************************************************"
+echo "This script is deprecated, please use build-orchestration-images.sh instead."
+echo "***************** Notice **************************************************"
diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml
index 8e32dd950..8517a3dd4 100644
--- a/compose/docker-compose.yml
+++ b/compose/docker-compose.yml
@@ -3,7 +3,7 @@ services:
# proftpd container
galaxy-proftpd:
# build: galaxy-proftpd
- image: quay.io/bgruening/galaxy-proftpd:${TAG:-latest}
+ image: quay.io/bgruening/galaxy-proftpd:${TAG_PROFTPD:-latest}
env_file: .env
environment:
- proftpd_db_connection=galaxy@galaxy-postgres
@@ -43,7 +43,7 @@ services:
# If you are using the official postgres image, it needs to be populated by calling
# docker-compose run galaxy install_db.sh
# on first run
- image: quay.io/bgruening/galaxy-postgres:${TAG:-latest}
+ image: quay.io/bgruening/galaxy-postgres:${TAG_POSTGRES:-latest}
container_name: galaxy-postgres
hostname: galaxy-postgres
env_file: .env
@@ -324,7 +324,6 @@ services:
volumes:
# This is the directory where all your files from Galaxy will be stored
# on your host system
- - ${EXPORT_DIR:-/export}/galaxy-central/config/galaxy.yml.sample:/etc/galaxy/galaxy.yml
- ${EXPORT_DIR:-/export}/:/export/
- /var/run/docker.sock:/var/run/docker.sock
depends_on:
diff --git a/compose/dumpsql.sh b/compose/dumpsql.sh
index 4a35af44c..e18fba5f1 100755
--- a/compose/dumpsql.sh
+++ b/compose/dumpsql.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-TAG=18.05
+TAG=v18.09
# Sets the image of postgres to use
POSTGRES=postgres:9.6.5
diff --git a/compose/galaxy-base/Dockerfile b/compose/galaxy-base/Dockerfile
index 4d3b5c10d..51ade8aa6 100644
--- a/compose/galaxy-base/Dockerfile
+++ b/compose/galaxy-base/Dockerfile
@@ -2,12 +2,12 @@
#
# VERSION Galaxy-central
-FROM ubuntu:14.04
+FROM ubuntu:18.04
MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com
ARG ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras
-ARG ANSIBLE_RELEASE=14c60b66f10326c7d158627b004d58e38ca503e4
+ARG ANSIBLE_RELEASE=18.09
ENV DEBIAN_FRONTEND=noninteractive \
GALAXY_USER=galaxy \
@@ -36,8 +36,6 @@ GALAXY_CONFIG_FTP_UPLOAD_SITE=galaxy.docker.org \
GALAXY_CONFIG_USE_PBKDF2=False \
GALAXY_CONFIG_NGINX_X_ACCEL_REDIRECT_BASE=/_x_accel_redirect \
GALAXY_CONFIG_NGINX_X_ARCHIVE_FILES_BASE=/_x_accel_redirect \
-GALAXY_CONFIG_NGINX_UPLOAD_STORE=/export/nginx_upload_store \
-GALAXY_CONFIG_NGINX_UPLOAD_PATH=/_upload \
GALAXY_CONFIG_DYNAMIC_PROXY_MANAGE=False \
GALAXY_CONFIG_VISUALIZATION_PLUGINS_DIRECTORY=config/plugins/visualizations \
GALAXY_CONFIG_TRUST_IPYTHON_NOTEBOOK_CONVERSION=True \
@@ -64,24 +62,18 @@ HOME=$GALAXY_HOME
RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
echo 'Acquire::http::Timeout "20";' > /etc/apt/apt.conf.d/98AcquireTimeout && \
echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/99AcquireRetries && \
+ apt-get -qq update && apt-get install --no-install-recommends -y locales && \
locale-gen en_US.UTF-8 && dpkg-reconfigure locales && \
- apt-get -qq update && apt-get install --no-install-recommends -y apt-transport-https wget && \
- # Condor mirror is really unreliable and times out too often, using a copy of the deb from cargo port
- #echo "deb [arch=amd64] http://research.cs.wisc.edu/htcondor/ubuntu/stable/ trusty contrib" > /etc/apt/sources.list.d/htcondor.list
- #wget -qO - http://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - && \
+ apt-get install --no-install-recommends -y apt-transport-https gnupg && \
echo "deb [arch=amd64] http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list && \
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 7BB9C367 && \
apt-get update -qq && apt-get upgrade -y && \
- apt-get install --no-install-recommends -y sudo python-virtualenv \
- supervisor linux-image-extra-virtual munge \
- ansible nano python-pip \
- unattended-upgrades \
- gridengine-drmaa1.0 && \
- wget -q --no-check-certificate https://depot.galaxyproject.org/software/condor/condor_8.6.3_linux_all.deb && \
- dpkg -i condor_8.6.3_linux_all.deb ; apt-get install -f -y && \
- rm condor_8.6.3_linux_all.deb && \
- pip install --upgrade pip setuptools && \
- pip install ephemeris && \
+ apt-get install --no-install-recommends -y sudo \
+ supervisor linux-image-extra-virtual munge \
+ ansible nano python-pip wget htcondor \
+ unattended-upgrades \
+ gridengine-drmaa1.0 && \
+ pip install ephemeris virtualenv && \
apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
mkdir -p /tmp/download && \
wget -qO - https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz | tar -xz -C /tmp/download && \
@@ -89,9 +81,13 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
rm -rf /tmp/download && \
rm -rf ~/.cache/
+RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \
+ mkdir -p /var/run/condor/ /var/lock/condor/ && \
+ chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/
+
RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \
useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" $GALAXY_USER && \
- mkdir $EXPORT_DIR $GALAXY_HOME $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $GALAXY_LOGS_DIR $EXPORT_DIR
+ mkdir $EXPORT_DIR $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $GALAXY_LOGS_DIR $EXPORT_DIR
ADD ./bashrc $GALAXY_HOME/.bashrc
# Container Style
diff --git a/compose/galaxy-grafana/requirements.txt b/compose/galaxy-grafana/requirements.txt
index 3f79978d5..8c9d6c834 100644
--- a/compose/galaxy-grafana/requirements.txt
+++ b/compose/galaxy-grafana/requirements.txt
@@ -3,7 +3,7 @@ bx-python==0.7.3
MarkupSafe==0.23
PyYAML==3.11
SQLAlchemy==1.0.15
-mercurial==3.7.3
+mercurial==4.1.3
numpy==1.9.2
pycrypto==2.6.1
@@ -38,7 +38,7 @@ Markdown==2.6.3
# BioBlend and dependencies
bioblend==0.7.0
boto==2.38.0
-requests==2.8.1
+requests>=2.20.0
requests-toolbelt==0.4.0
# kombu and dependencies
@@ -63,7 +63,7 @@ pyparsing==2.1.1
# Fabric and dependencies
Fabric==1.10.2
-paramiko==1.15.2
+paramiko>=2.0.9
ecdsa==0.13
# Flexible BAM index naming
diff --git a/compose/galaxy-htcondor-base/Dockerfile b/compose/galaxy-htcondor-base/Dockerfile
index 2469c4434..afe67f87e 100644
--- a/compose/galaxy-htcondor-base/Dockerfile
+++ b/compose/galaxy-htcondor-base/Dockerfile
@@ -1,18 +1,14 @@
-FROM ubuntu:14.04
+FROM ubuntu:18.04
ENV DEBIAN_FRONTEND noninteractive
RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
echo 'Acquire::http::Timeout "20";' > /etc/apt/apt.conf.d/98AcquireTimeout && \
echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/99AcquireRetries && \
+ apt-get update -qq && apt-get install -y --no-install-recommends locales && \
locale-gen en_US.UTF-8 && dpkg-reconfigure locales && \
- apt-get update -qq && apt-get install -y --no-install-recommends \
- apt-transport-https wget supervisor unattended-upgrades ca-certificates && \
- # Condor mirror is really unreliable and times out too often, using a copy of the deb from cargo port
- #echo "deb [arch=amd64] http://research.cs.wisc.edu/htcondor/ubuntu/stable/ trusty contrib" > /etc/apt/sources.list.d/htcondor.list
- wget -q https://depot.galaxyproject.org/software/condor/condor_8.6.3_linux_all.deb && \
- dpkg -i condor_8.6.3_linux_all.deb ; apt-get install -f -y && \
- rm condor_8.6.3_linux_all.deb && \
+ apt-get install -y --no-install-recommends \
+ apt-transport-https wget supervisor unattended-upgrades ca-certificates htcondor && \
rm -rf /var/lib/apt/lists/*
RUN wget https://dl.influxdata.com/telegraf/releases/telegraf-1.5.0_linux_amd64.tar.gz && \
@@ -22,3 +18,8 @@ RUN wget https://dl.influxdata.com/telegraf/releases/telegraf-1.5.0_linux_amd64.
rm telegraf-1.5.0_linux_amd64.tar.gz
ADD telegraf.conf /etc/telegraf/telegraf.conf
+
+
+RUN touch /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog && \
+ mkdir -p /var/run/condor/ /var/lock/condor/ && \
+ chown -R condor: /var/log/condor/StartLog /var/log/condor/StarterLog /var/log/condor/CollectorLog /var/log/condor/NegotiatorLog /var/run/condor/ /var/lock/condor/
diff --git a/compose/galaxy-htcondor-executor/Dockerfile b/compose/galaxy-htcondor-executor/Dockerfile
index 3030383c6..46f4a6e27 100644
--- a/compose/galaxy-htcondor-executor/Dockerfile
+++ b/compose/galaxy-htcondor-executor/Dockerfile
@@ -1,4 +1,4 @@
-FROM quay.io/bgruening/galaxy-htcondor-base:18.05
+FROM quay.io/bgruening/galaxy-htcondor-base:18.09
ENV GALAXY_USER=galaxy \
GALAXY_UID=1450 \
diff --git a/compose/galaxy-htcondor-executor/startup.sh b/compose/galaxy-htcondor-executor/startup.sh
index cf7f728ab..147a1d417 100644
--- a/compose/galaxy-htcondor-executor/startup.sh
+++ b/compose/galaxy-htcondor-executor/startup.sh
@@ -30,8 +30,6 @@ SCHED_NAME = $CONDOR_HOST
" > /etc/condor/condor_config.local
/usr/bin/telegraf --config /etc/telegraf/telegraf.conf &
-sudo -u condor touch /var/log/condor/StartLog
-sudo -u condor touch /var/log/condor/StarterLog
tail -f -n 1000 /var/log/condor/StartLog /var/log/condor/StarterLog &
# Mysterious bug? Docker doesn't output its version as condor user if this is not executed before
diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile
index 7c9e414e8..4c26d3265 100644
--- a/compose/galaxy-htcondor/Dockerfile
+++ b/compose/galaxy-htcondor/Dockerfile
@@ -1,4 +1,4 @@
-FROM quay.io/bgruening/galaxy-htcondor-base:18.05
+FROM quay.io/bgruening/galaxy-htcondor-base:18.09
ADD condor_config.local /etc/condor/condor_config.local
ADD supervisord.conf /etc/supervisord.conf
diff --git a/compose/galaxy-init/Dockerfile b/compose/galaxy-init/Dockerfile
index b7c395536..975212a12 100644
--- a/compose/galaxy-init/Dockerfile
+++ b/compose/galaxy-init/Dockerfile
@@ -2,13 +2,18 @@
#
# VERSION Galaxy-central
-FROM quay.io/bgruening/galaxy-base:18.05
+FROM quay.io/bgruening/galaxy-base:18.09
MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com
-ARG GALAXY_RELEASE=release_18.05
+ARG GALAXY_RELEASE=18.09
ARG GALAXY_REPO=galaxyproject/galaxy
+RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
+ echo "Acquire::http {No-Cache=True;};" > /etc/apt/apt.conf.d/no-cache && \
+ apt-get -qq update && apt-get install --no-install-recommends -y && \
+ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/
+
# Create these folders and link to target directory for installation
RUN mkdir -p /export /galaxy-export && \
mkdir /galaxy-export/shed_tools && \
@@ -38,6 +43,7 @@ ADD config $GALAXY_ROOT/config
ADD welcome /galaxy-export/welcome
RUN ansible-playbook /ansible/provision.yml \
+ --extra-vars galaxy_extras_config_ie_proxy=False \
--extra-vars galaxy_server_dir=$GALAXY_ROOT \
--extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \
--extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \
@@ -56,12 +62,8 @@ RUN ansible-playbook /ansible/provision.yml \
#--extra-vars container_resolution_cached_mulled=False \
#--extra-vars container_resolution_build_mulled=False \
--tags=ie,pbs,slurm,uwsgi,metrics,k8s -c local && \
- apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-#--extra-vars galaxy_extras_config_container_resolution=True \
-#--extra-vars container_resolution_explicit=True \
-#--extra-vars container_resolution_cached_mulled=False \
-#--extra-vars container_resolution_build_mulled=False \
+ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+ chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR
# The following commands will be executed as User galaxy
@@ -74,14 +76,25 @@ WORKDIR $GALAXY_ROOT
# Updating genome informations from UCSC
#RUN export GALAXY=$GALAXY_ROOT && sh ./cron/updateucsc.sh.sample
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \
+ /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \
+ rm ~/miniconda.sh && \
+ echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
+ echo "conda activate base" >> ~/.bashrc
+
# prefetch Python wheels
# Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments
-RUN ./scripts/common_startup.sh --skip-client-build && \
+RUN export PATH=/tool_deps/_conda/bin/:$PATH && \
+ conda install -c conda-forge nodejs=9.11.1 yarn=1.12.1 git && \
+ ./scripts/common_startup.sh --skip-client-build && \
. $GALAXY_VIRTUAL_ENV/bin/activate && \
python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \
# Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments
cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \
- npm install
+ npm install && \
+ cd $GALAXY_ROOT/client && yarn install --network-timeout 120000 --check-files && yarn run build-production-maps && \
+ rm -rf /home/galaxy/.cache/
# Switch back to User root
USER root
diff --git a/compose/galaxy-init/startup.sh b/compose/galaxy-init/startup.sh
index 15d8f6afb..ec8f674d8 100644
--- a/compose/galaxy-init/startup.sh
+++ b/compose/galaxy-init/startup.sh
@@ -41,13 +41,16 @@ done
# Optional, might not work
{
- if [ ! -d "/export/var/lib/docker" ]
+ if [ -d "/var/lib/docker" ]
then
- echo "Moving to /export/var/lib/docker"
- mkdir -p /export/var/lib/
- mv /var/lib/docker /export/var/lib/docker
- chown -R $GALAXY_UID:$GALAXY_GID /export/var/lib/docker
- fi
+ if [ ! -d "/export/var/lib/docker" ]
+ then
+ echo "Moving to /export/var/lib/docker"
+ mkdir -p /export/var/lib/
+ mv /var/lib/docker /export/var/lib/docker
+ chown -R $GALAXY_UID:$GALAXY_GID /export/var/lib/docker
+ fi
+ fi
} || echo "Moving docker lib failed, this is not a fatal error"
echo "Initialization complete"
diff --git a/compose/galaxy-postgres/init-galaxy-db.sql.in b/compose/galaxy-postgres/init-galaxy-db.sql.in
index 04099ffd5..eb19cbb2b 100644
--- a/compose/galaxy-postgres/init-galaxy-db.sql.in
+++ b/compose/galaxy-postgres/init-galaxy-db.sql.in
@@ -2090,7 +2090,7 @@ CREATE TABLE job_metric_numeric (
job_id integer,
plugin character varying(255),
metric_name character varying(255),
- metric_value numeric(22,7)
+ metric_value numeric(26,7)
);
@@ -4283,7 +4283,7 @@ CREATE TABLE task_metric_numeric (
task_id integer,
plugin character varying(255),
metric_name character varying(255),
- metric_value numeric(22,7)
+ metric_value numeric(26,7)
);
@@ -8126,7 +8126,7 @@ GalaxyTools lib/tool_shed/galaxy_install/migrate 1
--
COPY migrate_version (repository_id, repository_path, version) FROM stdin;
-Galaxy lib/galaxy/model/migrate 141
+Galaxy lib/galaxy/model/migrate 142
\.
diff --git a/compose/galaxy-proftpd/Dockerfile b/compose/galaxy-proftpd/Dockerfile
index 39255f75f..15861fff7 100644
--- a/compose/galaxy-proftpd/Dockerfile
+++ b/compose/galaxy-proftpd/Dockerfile
@@ -1,9 +1,9 @@
# ProFTP Image, used by the Galaxy Docker project
-FROM ubuntu:14.04
+FROM ubuntu:18.04
ARG ANSIBLE_REPO=galaxyproject/ansible-galaxy-extras
-ARG ANSIBLE_RELEASE=14c60b66f10326c7d158627b004d58e38ca503e4
+ARG ANSIBLE_RELEASE=18.09
ENV GALAXY_USER=galaxy \
GALAXY_UID=1450 \
diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile
index f9b45320c..e640f6951 100644
--- a/compose/galaxy-slurm/Dockerfile
+++ b/compose/galaxy-slurm/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:14.04
+FROM ubuntu:18.04
ENV DEBIAN_FRONTEND noninteractive
@@ -13,12 +13,14 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
groupadd -r $GALAXY_USER -g $GALAXY_GID && \
useradd -u $GALAXY_UID -m -r -g $GALAXY_USER -c "Galaxy user" $GALAXY_USER && \
apt-get update -qq && apt-get install -y --no-install-recommends apt-transport-https \
- slurm-llnl slurm-llnl-torque slurm-drmaa-dev unattended-upgrades \
- python-pip python-psutil supervisor wget build-essential && \
+ unattended-upgrades python-pip python-psutil python-setuptools supervisor wget \
+ build-essential munge locales slurm-wlm slurm-wlm-torque && \
+ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \
+ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \
+ dpkg -i slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \
+ dpkg -i slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \
/usr/sbin/create-munge-key && \
- touch /var/log/slurm-llnl/slurmctld.log /var/log/slurm-llnl/slurmd.log && \
- mkdir /tmp/slurm && pip install --upgrade supervisor virtualenv && \
-
+ pip install --upgrade virtualenv && \
wget https://github.com/singularityware/singularity/releases/download/$SINGULARITY_VERSION/singularity-$SINGULARITY_VERSION.tar.gz && \
tar xvf singularity-$SINGULARITY_VERSION.tar.gz && \
cd singularity-$SINGULARITY_VERSION && \
@@ -26,9 +28,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
make && \
make install && \
rm -rf singularity-$SINGULARITY_VERSION singularity-$SINGULARITY_VERSION.tar.gz && \
-
- apt-get remove -y supervisor build-essential && \
- mkdir /var/log/supervisor && \
+ apt-get remove -y build-essential && \
apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
mkdir -p /tmp/download && \
wget -qO - https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz | tar -xz -C /tmp/download && \
@@ -67,7 +67,7 @@ RUN chown -R $SLURM_UID:$SLURM_GID "$GALAXY_VENV" && \
. "$GALAXY_VENV"/bin/activate && \
pip install --upgrade pip && \
pip install galaxy-lib && \
- pip install -r "$GALAXY_VENV"/requirements.txt --index-url https://wheels.galaxyproject.org/simple && \
+ pip install -r "$GALAXY_VENV"/requirements.txt --index-url https://wheels.galaxyproject.org/simple --extra-index-url https://pypi.python.org/simple && \
rm -rf ~/.cache/
diff --git a/compose/galaxy-slurm/requirements.txt b/compose/galaxy-slurm/requirements.txt
index 3f79978d5..145afdd4d 100644
--- a/compose/galaxy-slurm/requirements.txt
+++ b/compose/galaxy-slurm/requirements.txt
@@ -38,7 +38,7 @@ Markdown==2.6.3
# BioBlend and dependencies
bioblend==0.7.0
boto==2.38.0
-requests==2.8.1
+requests>=2.20.0
requests-toolbelt==0.4.0
# kombu and dependencies
@@ -63,7 +63,7 @@ pyparsing==2.1.1
# Fabric and dependencies
Fabric==1.10.2
-paramiko==1.15.2
+paramiko>=2.0.9
ecdsa==0.13
# Flexible BAM index naming
diff --git a/compose/galaxy-slurm/startup.sh b/compose/galaxy-slurm/startup.sh
index e6ebf8994..799796c6c 100644
--- a/compose/galaxy-slurm/startup.sh
+++ b/compose/galaxy-slurm/startup.sh
@@ -16,6 +16,7 @@ fi
if [ ! -f "$SLURM_CONF_PATH" ]
then
+ mkdir -p /etc/slurm-llnl/
python /usr/local/bin/configure_slurm.py
cp /etc/slurm-llnl/slurm.conf "$SLURM_CONF_PATH"
fi
@@ -23,4 +24,4 @@ mkdir -p /tmp/slurm
chown $SLURM_USER_NAME /tmp/slurm
ln -sf "$GALAXY_DIR" "$SYMLINK_TARGET"
ln -sf "$SLURM_CONF_PATH" /etc/slurm-llnl/slurm.conf
-exec /usr/local/bin/supervisord -n -c /etc/supervisor/supervisord.conf
+exec /usr/bin/supervisord -n -c /etc/supervisor/supervisord.conf
diff --git a/compose/galaxy-web/Dockerfile b/compose/galaxy-web/Dockerfile
index 5baf04a13..5394f628b 100644
--- a/compose/galaxy-web/Dockerfile
+++ b/compose/galaxy-web/Dockerfile
@@ -2,7 +2,7 @@
#
# VERSION Galaxy-central
-FROM quay.io/bgruening/galaxy-base:18.05
+FROM quay.io/bgruening/galaxy-base:18.09
MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com
@@ -22,12 +22,15 @@ GALAXY_HANDLER_NUMPROCS=2
RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
apt-get -qq update && apt-get install --no-install-recommends -y apt-transport-https wget && \
- echo "deb [arch=amd64] http://ppa.launchpad.net/galaxyproject/nginx/ubuntu trusty main" > /etc/apt/sources.list.d/nginx.list && \
- apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 9735427B && \
- apt-get update -qq && apt-get upgrade -y && \
- apt-get install --no-install-recommends -y \
- slurm-drmaa-dev nginx-extras=1.4.6-1ubuntu3.8ppa1 nginx-common=1.4.6-1ubuntu3.8ppa1 supervisor \
- slurm-llnl autofs && \
+ echo "deb https://deb.nodesource.com/node_9.x $(lsb_release -sc) main" > /etc/apt/sources.list.d/nodejs.list && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 68576280 && \
+ apt-get update -qq && \
+ apt-get install --no-install-recommends -y nodejs \
+ nginx-extras nginx-common supervisor autofs slurm-wlm slurm-wlm-torque && \
+ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \
+ wget https://depot.galaxyproject.org/deb/ubuntu/18.04/slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \
+ dpkg -i slurm-drmaa1_1.2.0-dev.deca826_amd64.deb && \
+ dpkg -i slurm-drmaa-dev_1.2.0-dev.deca826_amd64.deb && \
# install and remove supervisor, so get the service config file
apt-get -qq remove -y supervisor && \
apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
@@ -35,7 +38,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
# this can be optimised I think, we should get the SLURM client only and update it
gpasswd -a $GALAXY_USER docker && \
mkdir -p /etc/supervisor/conf.d/ /var/log/supervisor/ && \
- pip install --upgrade pip && \
+ #pip install --upgrade pip && \
pip install ephemeris supervisor --upgrade && \
ln -s /usr/local/bin/supervisord /usr/bin/supervisord
@@ -54,7 +57,7 @@ RUN rm -f /usr/bin/startup && \
--extra-vars galaxy_extras_config_proftpd=False \
--extra-vars galaxy_extras_config_cvmfs=True \
--extra-vars galaxy_tool_data_table_config_file=/etc/galaxy/tool_data_table_conf.xml \
- --extra-vars nginx_upload_store_path=$GALAXY_CONFIG_NGINX_UPLOAD_STORE \
+ --extra-vars galaxy_extras_config_nginx_upload=False \
--extra-vars nginx_welcome_location=$NGINX_WELCOME_LOCATION \
--extra-vars nginx_welcome_path=$NGINX_WELCOME_PATH \
--extra-vars startup_sleeplock=True \
diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile
index b6e54004d..f6995e17c 100644
--- a/galaxy/Dockerfile
+++ b/galaxy/Dockerfile
@@ -9,7 +9,7 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com
ARG GALAXY_RELEASE
ARG GALAXY_REPO
-ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_18.05} \
+ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_18.09} \
GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \
GALAXY_ROOT=/galaxy-central \
GALAXY_CONFIG_DIR=/etc/galaxy \
@@ -53,7 +53,8 @@ GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost \
GALAXY_HANDLER_NUMPROCS=2 \
# Setting a standard encoding. This can get important for things like the unix sort tool.
LC_ALL=en_US.UTF-8 \
-LANG=en_US.UTF-8
+LANG=en_US.UTF-8 \
+GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda
RUN locale-gen en_US.UTF-8 && dpkg-reconfigure locales
@@ -71,7 +72,7 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
apt-add-repository -y ppa:ansible/ansible && \
apt-add-repository -y ppa:galaxyproject/nginx && \
apt-get update -qq && apt-get upgrade -y && \
- apt-get install --no-install-recommends -y postgresql-9.3 sudo python-virtualenv \
+ apt-get install --no-install-recommends -y postgresql-9.3 sudo \
nginx-extras=1.4.6-1ubuntu3.8ppa1 nginx-common=1.4.6-1ubuntu3.8ppa1 docker-ce=17.09.0~ce-0~ubuntu slurm-llnl slurm-llnl-torque \
slurm-drmaa-dev proftpd proftpd-mod-pgsql libyaml-dev ansible munge libmunge-dev \
nano vim curl python-crypto python-pip python-psutil condor python-ldap autofs \
@@ -81,8 +82,9 @@ RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup && \
apt-get purge -y software-properties-common && \
apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ && \
mkdir -p /etc/supervisor/conf.d/ /var/log/supervisor/ && \
- pip install --upgrade pip setuptools && \
+ pip install --upgrade pip && \
pip install ephemeris supervisor --upgrade && \
+ rm -rf ~/.cache/ /galaxy-central/client/node_modules/ && \
ln -s /usr/local/bin/supervisord /usr/bin/supervisord
RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \
@@ -90,10 +92,22 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID && \
mkdir $EXPORT_DIR $GALAXY_HOME $GALAXY_LOGS_DIR && chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_HOME $EXPORT_DIR $GALAXY_LOGS_DIR && \
gpasswd -a $GALAXY_USER docker && \
adduser condor docker
+
ADD ./bashrc $GALAXY_HOME/.bashrc
-# Download latest stable release of Galaxy.
-RUN mkdir $GALAXY_ROOT && \
+# Install miniconda, then virtualenv from conda and then
+# download latest stable release of Galaxy.
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \
+ /bin/bash ~/miniconda.sh -b -p /tool_deps/_conda/ && \
+ rm ~/miniconda.sh && \
+ ln -s /tool_deps/_conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+ echo ". /tool_deps/_conda/etc/profile.d/conda.sh" >> $GALAXY_HOME/.bashrc && \
+ echo "conda activate base" >> $GALAXY_HOME/.bashrc && \
+ export PATH=/tool_deps/_conda/bin/:$PATH && \
+ conda install virtualenv pip && \
+ # conda install conda-canary::conda=4.6 && \
+ chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh && \
+ mkdir $GALAXY_ROOT && \
wget -q -O - $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT && \
virtualenv $GALAXY_VIRTUAL_ENV && \
chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_VIRTUAL_ENV && \
@@ -104,11 +118,10 @@ RUN mkdir $GALAXY_ROOT && \
$GALAXY_VIRTUAL_ENV/bin/pip2 install pip --upgrade && \
$GALAXY_VIRTUAL_ENV/bin/pip2 install --extra-index-url https://wheels.galaxyproject.org/ uwsgi==2.0.15 -v --pre && \
chown -R $GALAXY_USER:$GALAXY_USER $GALAXY_VIRTUAL_ENV && \
- rm -rf ~/.cache/
+ rm -rf ~/.cache/ /galaxy-central/client/node_modules/ && \
+ su $GALAXY_USER -c "cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE"
-RUN su $GALAXY_USER -c "cp $GALAXY_ROOT/config/galaxy.yml.sample $GALAXY_CONFIG_FILE"
ADD ./reports_wsgi.ini.sample $GALAXY_CONFIG_DIR/reports_wsgi.ini
-
ADD sample_tool_list.yaml $GALAXY_HOME/ephemeris/sample_tool_list.yaml
ADD roles/ /ansible/roles
ADD provision.yml /ansible/provision.yml
@@ -138,11 +151,12 @@ RUN ansible-playbook /ansible/postgresql_provision.yml && \
--extra-vars supervisor_postgres_autostart=false \
--extra-vars nginx_use_remote_header=True \
--tags=galaxyextras,cvmfs -c local && \
- apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
+ rm -rf ~/.cache/ /galaxy-central/client/node_modules/
RUN mkdir /shed_tools && chown $GALAXY_USER:$GALAXY_USER /shed_tools && \
mkdir $EXPORT_DIR/ftp/ && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/ftp && \
- mkdir /tool_deps/ && chown $GALAXY_USER:$GALAXY_USER /tool_deps && \
+ mkdir -p /tool_deps/ && chown $GALAXY_USER:$GALAXY_USER -R /tool_deps && \
ln -s /tool_deps/ $EXPORT_DIR/tool_deps && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps # Configure Galaxy to use the Tool Shed
# The following commands will be executed as the galaxy user
@@ -189,14 +203,13 @@ ADD GalaxyDocker.png $GALAXY_CONFIG_DIR/web/welcome_image.png
ADD welcome.html $GALAXY_CONFIG_DIR/web/welcome.html
RUN ./scripts/common_startup.sh && \
+ export PATH=GALAXY_CONDA_PREFIX/bin/:$PATH && \
. $GALAXY_VIRTUAL_ENV/bin/activate && \
python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed && \
# Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments
cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \
- npm install
-
-# the following can be removed with 18.01
-RUN /tool_deps/_conda/bin/conda create -y --override-channels --channel iuc --channel bioconda --channel conda-forge --channel defaults --channel r --name mulled-v1-eb2018fd3ce0fcbcee48a4bf89c57219add691c16729281b6c46b30f08339397 samtools=1.3.1 bcftools=1.5
+ npm install && \
+ rm -rf ~/.cache/ /galaxy-central/client/node_modules/
# Switch back to User root
USER root
@@ -225,7 +238,9 @@ ADD ./tool_sheds_conf.xml $GALAXY_HOME/tool_sheds_conf.xml
ADD install_tools_wrapper.sh /usr/bin/install-tools
# script to install BioJS visualizations
ADD install_biojs_vis.sh /usr/bin/install-biojs
-RUN chmod +x /usr/bin/install-tools /usr/bin/install-biojs
+RUN chmod +x /usr/bin/install-tools /usr/bin/install-biojs && \
+ cd /usr/bin/ && wget https://git.embl.de/grp-gbcs/galaxy-dir-sync/raw/master/src/galaxy-dir-sync.py && \
+ chmod +x galaxy-dir-sync.py
RUN echo "DISCARD_SESSION_KEYRING_ON_STARTUP=False" > /etc/condor/condor_config.local && \
echo "TRUST_UID_DOMAIN=true" >> /etc/condor/condor_config.local
@@ -249,3 +264,4 @@ RUN chmod +x /usr/bin/install_db.sh
# Autostart script that is invoked during container start
CMD ["/usr/bin/startup"]
+
diff --git a/galaxy/install_tools_wrapper.sh b/galaxy/install_tools_wrapper.sh
index ff170f5f1..1a79fdcfa 100644
--- a/galaxy/install_tools_wrapper.sh
+++ b/galaxy/install_tools_wrapper.sh
@@ -24,7 +24,7 @@ else
done
echo "starting Galaxy"
- sudo -E -u galaxy ./run.sh -d $install_log --pidfile galaxy_install.pid --http-timeout 3000
+ sudo -E -u galaxy unset $SUDO_UID; ./run.sh -d $install_log --pidfile galaxy_install.pid --http-timeout 3000
galaxy_install_pid=`cat galaxy_install.pid`
galaxy-wait -g http://localhost:$PORT -v --timeout 120
diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras
index 14c60b66f..21975fcd6 160000
--- a/galaxy/roles/galaxyprojectdotorg.galaxyextras
+++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras
@@ -1 +1 @@
-Subproject commit 14c60b66f10326c7d158627b004d58e38ca503e4
+Subproject commit 21975fcd6808471bc8bc9707d3dc0f11fbbbceaf
diff --git a/test/bioblend/Dockerfile b/test/bioblend/Dockerfile
index 90748868f..2dccfcd76 100644
--- a/test/bioblend/Dockerfile
+++ b/test/bioblend/Dockerfile
@@ -3,18 +3,20 @@ FROM quay.io/bgruening/galaxy
USER galaxy
WORKDIR /home/galaxy
-RUN . /galaxy_venv/bin/activate && \
- wget https://github.com/galaxyproject/bioblend/archive/master.tar.gz && tar xfz master.tar.gz && \
+RUN wget https://github.com/galaxyproject/bioblend/archive/master.tar.gz && tar xfz master.tar.gz && \
cd bioblend-master && \
+ export PATH=/tool_deps/_conda/bin/:$PATH && . activate galaxy_env && \
pip install --upgrade "tox>=1.8.0" "pep8<=1.6.2" && \
python setup.py install && \
- sed -i.bak "s/commands.*$/commands =/" tox.ini
+ sed -i.bak "s/commands.*$/commands =/" tox.ini && \
+ sed -i.bak2 "s/GALAXY_VERSION/GALAXY_VERSION BIOBLEND_TEST_JOB_TIMEOUT/" tox.ini
ENV TOX_ENV=py27 \
BIOBLEND_GALAXY_API_KEY=admin \
- BIOBLEND_GALAXY_URL=http://galaxy
+ BIOBLEND_GALAXY_URL=http://galaxy \
+ BIOBLEND_TEST_JOB_TIMEOUT="240"
-CMD /bin/bash -c ". /galaxy_venv/bin/activate && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -e 'test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_tool_dependency_install'"
+CMD /bin/bash -c "export PATH=/tool_deps/_conda/bin/:$PATH && cd /home/galaxy/bioblend-master && tox -e $TOX_ENV -- -k 'not test_download_dataset and not test_upload_from_galaxy_filesystem and not test_get_datasets and not test_datasets_from_fs and not test_tool_dependency_install and not test_download_history and not test_export_and_download'"
# library tests, needs share /tmp filesystem
# * test_upload_from_galaxy_filesystem
diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh
index 3696ddd79..1f84d7add 100644
--- a/test/bioblend/test.sh
+++ b/test/bioblend/test.sh
@@ -8,11 +8,13 @@ then
pip install --upgrade "tox>=1.8.0" "pep8<=1.6.2" ;
python setup.py install ;
sed -i.bak "s/commands.*$/commands =/" tox.ini ;
+ sed -i.bak2 "s/GALAXY_VERSION/GALAXY_VERSION BIOBLEND_TEST_JOB_TIMEOUT/" tox.ini ;
export TOX_ENV=py27 ;
export BIOBLEND_GALAXY_API_KEY=admin ;
export BIOBLEND_GALAXY_URL=http://galaxy ;
+ export BIOBLEND_TEST_JOB_TIMEOUT="240";
cd /home/galaxy/bioblend-master ;
- tox -e $TOX_ENV -- -e "test_download_dataset|test_upload_from_galaxy_filesystem|test_get_datasets|test_datasets_from_fs|test_existing_history|test_new_history|test_params|test_tool_dependency_install"'
+ tox -e $TOX_ENV -- -k "not test_download_dataset and not test_upload_from_galaxy_filesystem and not test_get_datasets and not test_datasets_from_fs and not test_existing_history and not test_new_history and not test_params and not test_tool_dependency_install and not test_download_history and not test_export_and_download"'
else
docker build -t bioblend_test .