From 77f482d67ace257d405e2447af520a9745af5b69 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 10 Oct 2020 12:57:39 +0200 Subject: [PATCH 01/68] I will sponsor a new Miniconda version and restrict node to 4GB, node is way to greedy. --- galaxy/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 2b46ec66c..ae6c640b4 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -16,8 +16,8 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG GALAXY_RELEASE ARG GALAXY_REPO -ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.05} \ -GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \ +GALAXY_REPO=${GALAXY_REPO:-https://github.com/bgruening/galaxy} \ GALAXY_ROOT=/galaxy-central \ GALAXY_CONFIG_DIR=/etc/galaxy \ EXPORT_DIR=/export \ @@ -69,6 +69,7 @@ GALAXY_HANDLER_NUMPROCS=2 \ # Setting a standard encoding. This can get important for things like the unix sort tool. LC_ALL=en_US.UTF-8 \ LANG=en_US.UTF-8 \ +NODE_OPTIONS=--max-old-space-size=4096 \ GALAXY_CONDA_PREFIX=/tool_deps/_conda # 16MB @@ -118,7 +119,7 @@ ADD ./bashrc $GALAXY_HOME/.bashrc # Install miniconda, then virtualenv from conda and then # download latest stable release of Galaxy. -RUN curl -s -L https://repo.continuum.io/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh > ~/miniconda.sh \ +RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.3-Linux-x86_64.sh > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX/ \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ From b9844137bf42363247f3369a7936c8ce467f2e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sat, 10 Oct 2020 15:18:53 +0200 Subject: [PATCH 02/68] Update .travis.yml --- .travis.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 694321b4c..4353fad27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,11 @@ before_install: # start building this repo - git submodule update --init --recursive - sudo chown 1450 /tmp && sudo chmod a=rwx /tmp + - export WORKING_DIR="$TRAVIS_BUILD_DIR" + - export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy" + - export INSTALL_REPO_ARG="" + - export SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml + - travis_wait 30 cd "$WORKING_DIR" && docker build -t quay.io/bgruening/galaxy galaxy/ - | ## define a container size check function, first parameter is the container name, second the max allowed size in MB container_size_check () { @@ -55,14 +60,6 @@ before_install: #exit fi } - - export WORKING_DIR="$TRAVIS_BUILD_DIR" - export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy" - INSTALL_REPO_ARG="" - SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml - cd "$WORKING_DIR" - docker build -t quay.io/bgruening/galaxy galaxy/ - container_size_check quay.io/bgruening/galaxy 1500 mkdir local_folder From cef854e2ba674ca547c2352749c7722fa0ac8fd1 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 10 Oct 2020 17:29:12 +0200 Subject: [PATCH 03/68] update role to 20.09 --- galaxy/roles/galaxyprojectdotorg.galaxyextras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index b9e47789f..82da900da 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit b9e47789f9b2be2a18d3db3d9585893ddefca062 +Subproject commit 82da900da5667a4d1e06ff976487d30ce460a59d From f9a2b2e4a1465b825bdf068a5c5ab7f7a3566f95 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 10 Oct 2020 17:30:03 +0200 Subject: [PATCH 04/68] update role to 20.09 --- galaxy/roles/galaxy-postgresql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/roles/galaxy-postgresql b/galaxy/roles/galaxy-postgresql index f363e42e3..1507fbe5e 160000 --- a/galaxy/roles/galaxy-postgresql +++ b/galaxy/roles/galaxy-postgresql @@ -1 +1 @@ -Subproject commit f363e42e373dce38e8dceb75acb07e770f71b83c +Subproject commit 1507fbe5eda946fc9f4c9aabef4e73a9b11a8315 From 7c377bad94036ff67fd072eefa694bdd70798c50 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 10 Oct 2020 17:59:17 +0200 Subject: [PATCH 05/68] add new GHA test --- .github/workflows/single.sh | 152 +++++++++++++++++++++++++ .github/workflows/single_container.yml | 11 ++ 2 files changed, 163 insertions(+) create mode 100644 .github/workflows/single.sh create mode 100644 .github/workflows/single_container.yml diff --git a/.github/workflows/single.sh b/.github/workflows/single.sh new file mode 100644 index 000000000..06d891550 --- /dev/null +++ b/.github/workflows/single.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +set -e +export GALAXY_HOME=/home/galaxy +export GALAXY_USER=admin@galaxy.org +export GALAXY_USER_EMAIL=admin@galaxy.org +export GALAXY_USER_PASSWD=password +export BIOBLEND_GALAXY_API_KEY=fakekey +export BIOBLEND_GALAXY_URL=http://localhost:8080 + +sudo apt-get update -qq +#sudo apt-get install docker-ce --no-install-recommends -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" +sudo apt-get install sshpass --no-install-recommends -y + +pip3 install ephemeris + +docker --version +docker info + +# start building this repo +git submodule update --init --recursive +sudo chown 1450 /tmp && sudo chmod a=rwx /tmp + +## define a container size check function, first parameter is the container name, second the max allowed size in MB +container_size_check () { + + # check that the image size is not growing too much between releases + # the 19.05 monolithic image was around 1.500 MB + size=`docker image inspect $1 --format='{{.Size}}'` + size_in_mb=$(($size/(1024*1024))) + if [[ $size_in_mb -ge $2 ]] + then + echo "The new compiled image ($1) is larger than allowed. $size_in_mb vs. $2" + sleep 2 + #exit + fi +} + +export WORKING_DIR=${GITHUB_WORKSPACE:-$PWD} + +export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy" +INSTALL_REPO_ARG="" +SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml +cd "$WORKING_DIR" +docker build -t quay.io/bgruening/galaxy galaxy/ +#container_size_check quay.io/bgruening/galaxy 1500 + +mkdir local_folder +docker run -d -p 8080:80 -p 8021:21 -p 8022:22 \ + --name galaxy \ + --privileged=true \ + -v `pwd`/local_folder:/export/ \ + -e GALAXY_CONFIG_ALLOW_USER_DATASET_PURGE=True \ + -e GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE=True \ + -e GALAXY_CONFIG_ENABLE_USER_DELETION=True \ + -e GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True \ + -v /tmp/:/tmp/ \ + quay.io/bgruening/galaxy + +sleep 30 +docker logs galaxy +# Define start functions +docker_exec() { + cd $WORKING_DIR + docker exec -t -i galaxy "$@" +} +docker_exec_run() { + cd $WORKING_DIR + docker run quay.io/bgruening/galaxy "$@" +} +docker_run() { + cd $WORKING_DIR + docker run "$@" +} + +docker ps + +# Test submitting jobs to an external slurm cluster +cd ${WORKING_DIR}/test/slurm/ && bash test.sh && cd $WORKING_DIR + +# Test submitting jobs to an external gridengine cluster +# TODO 19.05, need to enable this again! +# - cd $WORKING_DIR/test/gridengine/ && bash test.sh && cd $WORKING_DIR + +echo 'Waiting for Galaxy to come up.' +galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 300 + +curl -v --fail $BIOBLEND_GALAXY_URL/api/version + +# Test self-signed HTTPS +docker_run -d --name httpstest -p 443:443 -e "USE_HTTPS=True" $DOCKER_RUN_CONTAINER +# TODO 19.05 +# - sleep 90s && curl -v -k --fail https://127.0.0.1:443/api/version +#- echo | openssl s_client -connect 127.0.0.1:443 2>/dev/null | openssl x509 -issuer -noout| grep selfsigned + +docker logs httpstest && docker stop httpstest && docker rm httpstest + +# Test FTP Server upload +date > time.txt && curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true +# Test FTP Server get +curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD + +# Test CVMFS +docker_exec bash -c "service autofs start" +docker_exec bash -c "cvmfs_config chksetup" +docker_exec bash -c "ls /cvmfs/data.galaxyproject.org/byhand" + +# Test SFTP Server +sshpass -p $GALAXY_USER_PASSWD sftp -v -P 8022 -o User=$GALAXY_USER -o "StrictHostKeyChecking no" localhost <<< $'put time.txt' + +# Run a ton of BioBlend test against our servers. +cd $WORKING_DIR/test/bioblend/ && . ./test.sh && cd $WORKING_DIR/ + +# not working anymore in 18.01 +# executing: /galaxy_venv/bin/uwsgi --yaml /etc/galaxy/galaxy.yml --master --daemonize2 galaxy.log --pidfile2 galaxy.pid --log-file=galaxy_install.log --pid-file=galaxy_install.pid +# [uWSGI] getting YAML configuration from /etc/galaxy/galaxy.yml +# /galaxy_venv/bin/python: unrecognized option '--log-file=galaxy_install.log' +# getopt_long() error +# cat: galaxy_install.pid: No such file or directory +# tail: cannot open ‘galaxy_install.log’ for reading: No such file or directory +#- | +# if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] +# then +# # Test without install-repository wrapper +# sleep 10 +# docker_exec_run bash -c 'cd $GALAXY_ROOT && python ./scripts/api/install_tool_shed_repositories.py --api admin -l http://localhost:80 --url https://toolshed.g2.bx.psu.edu -o devteam --name cut_columns --panel-section-name BEDTools' +# fi + + +# Test the 'new' tool installation script +docker_exec install-tools "$SAMPLE_TOOLS" +# Test the Conda installation +docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/bin/:$PATH && conda --version && conda install samtools -c bioconda --yes' + + +docker stop galaxy +docker rmi galaxy + + + + + + + + + + + + + + + diff --git a/.github/workflows/single_container.yml b/.github/workflows/single_container.yml new file mode 100644 index 000000000..b7026e2b8 --- /dev/null +++ b/.github/workflows/single_container.yml @@ -0,0 +1,11 @@ +name: Single Container Test +on: [push] +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Build and Test + run: ./.github/workflows/single.sh From 3a23265305c141f360d4806c8943e04797856cb4 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 10 Oct 2020 18:22:01 +0200 Subject: [PATCH 06/68] GHA update --- .github/workflows/single.sh | 20 ++++++++++---------- .github/workflows/single_container.yml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) mode change 100644 => 100755 .github/workflows/single.sh diff --git a/.github/workflows/single.sh b/.github/workflows/single.sh old mode 100644 new mode 100755 index 06d891550..fedec34fd --- a/.github/workflows/single.sh +++ b/.github/workflows/single.sh @@ -1,6 +1,6 @@ #!/bin/bash - set -e + export GALAXY_HOME=/home/galaxy export GALAXY_USER=admin@galaxy.org export GALAXY_USER_EMAIL=admin@galaxy.org @@ -26,7 +26,7 @@ container_size_check () { # check that the image size is not growing too much between releases # the 19.05 monolithic image was around 1.500 MB - size=`docker image inspect $1 --format='{{.Size}}'` + size="${docker image inspect $1 --format='{{.Size}}'}" size_in_mb=$(($size/(1024*1024))) if [[ $size_in_mb -ge $2 ]] then @@ -39,7 +39,6 @@ container_size_check () { export WORKING_DIR=${GITHUB_WORKSPACE:-$PWD} export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy" -INSTALL_REPO_ARG="" SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml cd "$WORKING_DIR" docker build -t quay.io/bgruening/galaxy galaxy/ @@ -49,7 +48,7 @@ mkdir local_folder docker run -d -p 8080:80 -p 8021:21 -p 8022:22 \ --name galaxy \ --privileged=true \ - -v `pwd`/local_folder:/export/ \ + -v "$(pwd)/local_folder:/export/" \ -e GALAXY_CONFIG_ALLOW_USER_DATASET_PURGE=True \ -e GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE=True \ -e GALAXY_CONFIG_ENABLE_USER_DELETION=True \ @@ -61,22 +60,22 @@ sleep 30 docker logs galaxy # Define start functions docker_exec() { - cd $WORKING_DIR + cd "$WORKING_DIR" docker exec -t -i galaxy "$@" } docker_exec_run() { - cd $WORKING_DIR + cd "$WORKING_DIR" docker run quay.io/bgruening/galaxy "$@" } docker_run() { - cd $WORKING_DIR + cd "$WORKING_DIR" docker run "$@" } docker ps # Test submitting jobs to an external slurm cluster -cd ${WORKING_DIR}/test/slurm/ && bash test.sh && cd $WORKING_DIR +cd "${WORKING_DIR}/test/slurm/" && bash test.sh && cd "$WORKING_DIR" # Test submitting jobs to an external gridengine cluster # TODO 19.05, need to enable this again! @@ -96,7 +95,8 @@ docker_run -d --name httpstest -p 443:443 -e "USE_HTTPS=True" $DOCKER_RUN_CONTAI docker logs httpstest && docker stop httpstest && docker rm httpstest # Test FTP Server upload -date > time.txt && curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true +date > time.txt +curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true # Test FTP Server get curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD @@ -109,7 +109,7 @@ docker_exec bash -c "ls /cvmfs/data.galaxyproject.org/byhand" sshpass -p $GALAXY_USER_PASSWD sftp -v -P 8022 -o User=$GALAXY_USER -o "StrictHostKeyChecking no" localhost <<< $'put time.txt' # Run a ton of BioBlend test against our servers. -cd $WORKING_DIR/test/bioblend/ && . ./test.sh && cd $WORKING_DIR/ +cd "$WORKING_DIR/test/bioblend/" && . ./test.sh && cd "$WORKING_DIR/" # not working anymore in 18.01 # executing: /galaxy_venv/bin/uwsgi --yaml /etc/galaxy/galaxy.yml --master --daemonize2 galaxy.log --pidfile2 galaxy.pid --log-file=galaxy_install.log --pid-file=galaxy_install.pid diff --git a/.github/workflows/single_container.yml b/.github/workflows/single_container.yml index b7026e2b8..1208b03d4 100644 --- a/.github/workflows/single_container.yml +++ b/.github/workflows/single_container.yml @@ -8,4 +8,4 @@ jobs: uses: actions/checkout@v2 - name: Build and Test - run: ./.github/workflows/single.sh + run: bash .github/workflows/single.sh From bc3c419d768d09a1a90d3754b698ddf9484573a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sat, 10 Oct 2020 18:26:02 +0200 Subject: [PATCH 07/68] Update single_container.yml --- .github/workflows/single_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/single_container.yml b/.github/workflows/single_container.yml index 1208b03d4..cb7ffa6cf 100644 --- a/.github/workflows/single_container.yml +++ b/.github/workflows/single_container.yml @@ -1,7 +1,7 @@ name: Single Container Test on: [push] jobs: - lint: + build_and_test: runs-on: ubuntu-latest steps: - name: Checkout From c4f8fb1f74c87664a556ecfa25075306cc1c76ec Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 10 Oct 2020 18:45:03 +0200 Subject: [PATCH 08/68] add python --- .github/workflows/single_container.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/single_container.yml b/.github/workflows/single_container.yml index 1208b03d4..701297ccc 100644 --- a/.github/workflows/single_container.yml +++ b/.github/workflows/single_container.yml @@ -3,9 +3,14 @@ on: [push] jobs: lint: runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7] steps: - name: Checkout uses: actions/checkout@v2 - + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} - name: Build and Test run: bash .github/workflows/single.sh From 240376b54bc1d77d25d5174825aa8e277c232f8d Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 11 Oct 2020 14:40:30 +0200 Subject: [PATCH 09/68] cleanup JS stuff --- galaxy/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index ae6c640b4..2a3ca6b39 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -213,6 +213,8 @@ RUN mkdir -p /shed_tools $EXPORT_DIR/ftp/ \ --tags=galaxyextras,cvmfs -c local \ # TODO: no clue why this is needed here again && cd $GALAXY_ROOT && ./scripts/common_startup.sh \ + && cd config && find . -name 'node_modules' -type d -prune -exec rm -rf '{}' + \ + && find . -name '.cache' -type d -prune -exec rm -rf '{}' + \ && cd / \ && rm $PG_DATA_DIR_DEFAULT -rf \ && python /usr/local/bin/setup_postgresql.py --dbuser galaxy --dbpassword galaxy --db-name galaxy --dbpath $PG_DATA_DIR_DEFAULT --dbversion $PG_VERSION \ From c497c4fa7f9524fd5be38067405965cb59fe018c Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sun, 11 Oct 2020 14:41:12 +0200 Subject: [PATCH 10/68] add -ex --- .github/workflows/single.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/single.sh b/.github/workflows/single.sh index fedec34fd..d0c5bd1b3 100755 --- a/.github/workflows/single.sh +++ b/.github/workflows/single.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -ex export GALAXY_HOME=/home/galaxy export GALAXY_USER=admin@galaxy.org From e45b9fe08ede06765780625260bebbc1aef0720a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 13 Dec 2020 11:15:14 +0100 Subject: [PATCH 11/68] set thunder-lock true --- compose/base_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/base_config.yml b/compose/base_config.yml index cc2970901..f3640e038 100644 --- a/compose/base_config.yml +++ b/compose/base_config.yml @@ -9,7 +9,7 @@ galaxy_uwsgi: master: false virtualenv: .venv pythonpath: lib - thunder-lock: false + thunder-lock: true die-on-term: true hook-master-start: unix_signal:2 gracefully_kill_them_all hook-master-start: unix_signal:15 gracefully_kill_them_all From e199c6ad99230241a63f47e6b39ca8cbb9f89fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Mon, 1 Feb 2021 00:49:52 +0100 Subject: [PATCH 12/68] add WeasyPrint --- galaxy/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 2a3ca6b39..560589b8f 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -212,6 +212,9 @@ RUN mkdir -p /shed_tools $EXPORT_DIR/ftp/ \ --extra-vars nginx_use_remote_header=True \ --tags=galaxyextras,cvmfs -c local \ # TODO: no clue why this is needed here again + && . $GALAXY_VIRTUAL_ENV/bin/activate \ + && pip install WeasyPrint + && deactivate && cd $GALAXY_ROOT && ./scripts/common_startup.sh \ && cd config && find . -name 'node_modules' -type d -prune -exec rm -rf '{}' + \ && find . -name '.cache' -type d -prune -exec rm -rf '{}' + \ From 80489d7e922862962589859c2e71cde786ca012e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Mon, 1 Feb 2021 10:29:12 +0100 Subject: [PATCH 13/68] Update Dockerfile --- galaxy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 560589b8f..4ee55d19f 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -211,10 +211,10 @@ RUN mkdir -p /shed_tools $EXPORT_DIR/ftp/ \ --extra-vars supervisor_postgres_autostart=false \ --extra-vars nginx_use_remote_header=True \ --tags=galaxyextras,cvmfs -c local \ - # TODO: no clue why this is needed here again - && . $GALAXY_VIRTUAL_ENV/bin/activate \ + && . $GALAXY_VIRTUAL_ENV/bin/activate && pip install WeasyPrint && deactivate + # TODO: no clue why this is needed here again && cd $GALAXY_ROOT && ./scripts/common_startup.sh \ && cd config && find . -name 'node_modules' -type d -prune -exec rm -rf '{}' + \ && find . -name '.cache' -type d -prune -exec rm -rf '{}' + \ From c6409db6fa21c52bd02ec49226ac6700953fd5a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Mon, 1 Feb 2021 10:31:28 +0100 Subject: [PATCH 14/68] Update Dockerfile --- galaxy/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 4ee55d19f..a8c5d1219 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -211,9 +211,9 @@ RUN mkdir -p /shed_tools $EXPORT_DIR/ftp/ \ --extra-vars supervisor_postgres_autostart=false \ --extra-vars nginx_use_remote_header=True \ --tags=galaxyextras,cvmfs -c local \ - && . $GALAXY_VIRTUAL_ENV/bin/activate - && pip install WeasyPrint - && deactivate + && . $GALAXY_VIRTUAL_ENV/bin/activate \ + && pip install WeasyPrint \ + && deactivate \ # TODO: no clue why this is needed here again && cd $GALAXY_ROOT && ./scripts/common_startup.sh \ && cd config && find . -name 'node_modules' -type d -prune -exec rm -rf '{}' + \ From 7a54464a5cfcfce98c9108c15517db6b1546d250 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Mon, 1 Feb 2021 19:30:37 +0100 Subject: [PATCH 15/68] fix repo URL --- galaxy/Dockerfile | 198 +++++++++++++++++++++++----------------------- 1 file changed, 99 insertions(+), 99 deletions(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index a8c5d1219..77e2fe516 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -17,60 +17,60 @@ ARG GALAXY_RELEASE ARG GALAXY_REPO ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \ -GALAXY_REPO=${GALAXY_REPO:-https://github.com/bgruening/galaxy} \ -GALAXY_ROOT=/galaxy-central \ -GALAXY_CONFIG_DIR=/etc/galaxy \ -EXPORT_DIR=/export \ -DEBIAN_FRONTEND=noninteractive \ -PG_VERSION=11 + GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ + GALAXY_ROOT=/galaxy-central \ + GALAXY_CONFIG_DIR=/etc/galaxy \ + EXPORT_DIR=/export \ + DEBIAN_FRONTEND=noninteractive \ + PG_VERSION=11 ENV GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml \ -GALAXY_CONFIG_JOB_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_conf.xml \ -GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_metrics_conf.xml \ -GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml \ -GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \ -GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=$EXPORT_DIR/tool_deps \ -GALAXY_CONFIG_TOOL_PATH=$EXPORT_DIR/galaxy-central/tools \ -GALAXY_VIRTUAL_ENV=/galaxy_venv \ -GALAXY_USER=galaxy \ -GALAXY_UID=1450 \ -GALAXY_GID=1450 \ -GALAXY_POSTGRES_UID=1550 \ -GALAXY_POSTGRES_GID=1550 \ -GALAXY_HOME=/home/galaxy \ -GALAXY_LOGS_DIR=/home/galaxy/logs \ -GALAXY_DEFAULT_ADMIN_USER=admin \ -GALAXY_DEFAULT_ADMIN_EMAIL=admin@galaxy.org \ -GALAXY_DEFAULT_ADMIN_PASSWORD=password \ -GALAXY_DEFAULT_ADMIN_KEY=fakekey \ -GALAXY_DESTINATIONS_DEFAULT=slurm_cluster \ -GALAXY_RUNNERS_ENABLE_SLURM=True \ -GALAXY_RUNNERS_ENABLE_CONDOR=False \ -GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:galaxy@localhost:5432/galaxy?client_encoding=utf8 \ -GALAXY_CONFIG_ADMIN_USERS=admin@galaxy.org \ -GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3 \ -GALAXY_CONFIG_BRAND="Galaxy Docker Build" \ -GALAXY_CONFIG_STATIC_ENABLED=False \ -# Define the default postgresql database path -PG_DATA_DIR_DEFAULT=/var/lib/postgresql/$PG_VERSION/main/ \ -PG_CONF_DIR_DEFAULT=/etc/postgresql/$PG_VERSION/main/ \ -PG_DATA_DIR_HOST=$EXPORT_DIR/postgresql/$PG_VERSION/main/ \ -# The following 2 ENV vars can be used to set the number of uwsgi processes and threads -UWSGI_PROCESSES=2 \ -UWSGI_THREADS=4 \ -# Set HTTPS to use a self-signed certificate (or your own certificate in $EXPORT_DIR/{server.key,server.crt}) -USE_HTTPS=False \ -# Set USE_HTTPS_LENSENCRYPT and GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL to a domain that is reachable to get a letsencrypt certificate -USE_HTTPS_LETSENCRYPT=False \ -GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost \ -# Set the number of Galaxy handlers -GALAXY_HANDLER_NUMPROCS=2 \ -# Setting a standard encoding. This can get important for things like the unix sort tool. -LC_ALL=en_US.UTF-8 \ -LANG=en_US.UTF-8 \ -NODE_OPTIONS=--max-old-space-size=4096 \ -GALAXY_CONDA_PREFIX=/tool_deps/_conda + GALAXY_CONFIG_JOB_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_conf.xml \ + GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_metrics_conf.xml \ + GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml \ + GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \ + GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=$EXPORT_DIR/tool_deps \ + GALAXY_CONFIG_TOOL_PATH=$EXPORT_DIR/galaxy-central/tools \ + GALAXY_VIRTUAL_ENV=/galaxy_venv \ + GALAXY_USER=galaxy \ + GALAXY_UID=1450 \ + GALAXY_GID=1450 \ + GALAXY_POSTGRES_UID=1550 \ + GALAXY_POSTGRES_GID=1550 \ + GALAXY_HOME=/home/galaxy \ + GALAXY_LOGS_DIR=/home/galaxy/logs \ + GALAXY_DEFAULT_ADMIN_USER=admin \ + GALAXY_DEFAULT_ADMIN_EMAIL=admin@galaxy.org \ + GALAXY_DEFAULT_ADMIN_PASSWORD=password \ + GALAXY_DEFAULT_ADMIN_KEY=fakekey \ + GALAXY_DESTINATIONS_DEFAULT=slurm_cluster \ + GALAXY_RUNNERS_ENABLE_SLURM=True \ + GALAXY_RUNNERS_ENABLE_CONDOR=False \ + GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:galaxy@localhost:5432/galaxy?client_encoding=utf8 \ + GALAXY_CONFIG_ADMIN_USERS=admin@galaxy.org \ + GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3 \ + GALAXY_CONFIG_BRAND="Galaxy Docker Build" \ + GALAXY_CONFIG_STATIC_ENABLED=False \ + # Define the default postgresql database path + PG_DATA_DIR_DEFAULT=/var/lib/postgresql/$PG_VERSION/main/ \ + PG_CONF_DIR_DEFAULT=/etc/postgresql/$PG_VERSION/main/ \ + PG_DATA_DIR_HOST=$EXPORT_DIR/postgresql/$PG_VERSION/main/ \ + # The following 2 ENV vars can be used to set the number of uwsgi processes and threads + UWSGI_PROCESSES=2 \ + UWSGI_THREADS=4 \ + # Set HTTPS to use a self-signed certificate (or your own certificate in $EXPORT_DIR/{server.key,server.crt}) + USE_HTTPS=False \ + # Set USE_HTTPS_LENSENCRYPT and GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL to a domain that is reachable to get a letsencrypt certificate + USE_HTTPS_LETSENCRYPT=False \ + GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost \ + # Set the number of Galaxy handlers + GALAXY_HANDLER_NUMPROCS=2 \ + # Setting a standard encoding. This can get important for things like the unix sort tool. + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + NODE_OPTIONS=--max-old-space-size=4096 \ + GALAXY_CONDA_PREFIX=/tool_deps/_conda # 16MB RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup \ @@ -189,28 +189,28 @@ RUN mkdir -p /shed_tools $EXPORT_DIR/ftp/ \ && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps \ && apt update -qq && apt install --no-install-recommends -y ansible \ && ansible-playbook /ansible/provision.yml \ - --extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \ - --extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \ - --extra-vars galaxy_user_name=$GALAXY_USER \ - --extra-vars galaxy_config_file=$GALAXY_CONFIG_FILE \ - --extra-vars galaxy_config_dir=$GALAXY_CONFIG_DIR \ - --extra-vars galaxy_job_conf_path=$GALAXY_CONFIG_JOB_CONFIG_FILE \ - --extra-vars galaxy_job_metrics_conf_path=$GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE \ - --extra-vars supervisor_manage_slurm="" \ - --extra-vars galaxy_extras_config_condor=True \ - --extra-vars galaxy_extras_config_condor_docker=True \ - --extra-vars galaxy_extras_config_rabbitmq=True \ - --extra-vars galaxy_extras_config_cvmfs=True \ - --extra-vars galaxy_extras_config_uwsgi=False \ - --extra-vars proftpd_db_connection=galaxy@galaxy \ - --extra-vars proftpd_files_dir=$EXPORT_DIR/ftp \ - --extra-vars proftpd_use_sftp=True \ - --extra-vars galaxy_extras_docker_legacy=False \ - --extra-vars galaxy_minimum_version=19.01 \ - --extra-vars supervisor_postgres_config_path=$PG_CONF_DIR_DEFAULT/postgresql.conf \ - --extra-vars supervisor_postgres_autostart=false \ - --extra-vars nginx_use_remote_header=True \ - --tags=galaxyextras,cvmfs -c local \ + --extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \ + --extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \ + --extra-vars galaxy_user_name=$GALAXY_USER \ + --extra-vars galaxy_config_file=$GALAXY_CONFIG_FILE \ + --extra-vars galaxy_config_dir=$GALAXY_CONFIG_DIR \ + --extra-vars galaxy_job_conf_path=$GALAXY_CONFIG_JOB_CONFIG_FILE \ + --extra-vars galaxy_job_metrics_conf_path=$GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE \ + --extra-vars supervisor_manage_slurm="" \ + --extra-vars galaxy_extras_config_condor=True \ + --extra-vars galaxy_extras_config_condor_docker=True \ + --extra-vars galaxy_extras_config_rabbitmq=True \ + --extra-vars galaxy_extras_config_cvmfs=True \ + --extra-vars galaxy_extras_config_uwsgi=False \ + --extra-vars proftpd_db_connection=galaxy@galaxy \ + --extra-vars proftpd_files_dir=$EXPORT_DIR/ftp \ + --extra-vars proftpd_use_sftp=True \ + --extra-vars galaxy_extras_docker_legacy=False \ + --extra-vars galaxy_minimum_version=19.01 \ + --extra-vars supervisor_postgres_config_path=$PG_CONF_DIR_DEFAULT/postgresql.conf \ + --extra-vars supervisor_postgres_autostart=false \ + --extra-vars nginx_use_remote_header=True \ + --tags=galaxyextras,cvmfs -c local \ && . $GALAXY_VIRTUAL_ENV/bin/activate \ && pip install WeasyPrint \ && deactivate \ @@ -244,28 +244,28 @@ WORKDIR $GALAXY_ROOT #RUN export GALAXY=$GALAXY_ROOT && sh ./cron/updateucsc.sh.sample ENV GALAXY_CONFIG_JOB_WORKING_DIRECTORY=$EXPORT_DIR/galaxy-central/database/job_working_directory \ -GALAXY_CONFIG_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \ -GALAXY_CONFIG_NEW_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \ -GALAXY_CONFIG_TEMPLATE_CACHE_PATH=$EXPORT_DIR/galaxy-central/database/compiled_templates \ -GALAXY_CONFIG_CITATION_CACHE_DATA_DIR=$EXPORT_DIR/galaxy-central/database/citations/data \ -GALAXY_CONFIG_CLUSTER_FILES_DIRECTORY=$EXPORT_DIR/galaxy-central/database/pbs \ -GALAXY_CONFIG_FTP_UPLOAD_DIR=$EXPORT_DIR/ftp \ -GALAXY_CONFIG_FTP_UPLOAD_SITE=galaxy.docker.org \ -GALAXY_CONFIG_USE_PBKDF2=False \ -GALAXY_CONFIG_NGINX_X_ACCEL_REDIRECT_BASE=/_x_accel_redirect \ -GALAXY_CONFIG_NGINX_X_ARCHIVE_FILES_BASE=/_x_accel_redirect \ -GALAXY_CONFIG_DYNAMIC_PROXY_MANAGE=False \ -GALAXY_CONFIG_VISUALIZATION_PLUGINS_DIRECTORY=config/plugins/visualizations \ -GALAXY_CONFIG_TRUST_IPYTHON_NOTEBOOK_CONVERSION=True \ -GALAXY_CONFIG_TOOLFORM_UPGRADE=True \ -GALAXY_CONFIG_SANITIZE_ALL_HTML=False \ -GALAXY_CONFIG_TOOLFORM_UPGRADE=True \ -GALAXY_CONFIG_WELCOME_URL=$GALAXY_CONFIG_DIR/web/welcome.html \ -GALAXY_CONFIG_OVERRIDE_DEBUG=False \ -GALAXY_CONFIG_ENABLE_QUOTAS=True \ -# We need to set $HOME for some Tool Shed tools (e.g Perl libs with $HOME/.cpan) -HOME=$GALAXY_HOME \ -GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda + GALAXY_CONFIG_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \ + GALAXY_CONFIG_NEW_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \ + GALAXY_CONFIG_TEMPLATE_CACHE_PATH=$EXPORT_DIR/galaxy-central/database/compiled_templates \ + GALAXY_CONFIG_CITATION_CACHE_DATA_DIR=$EXPORT_DIR/galaxy-central/database/citations/data \ + GALAXY_CONFIG_CLUSTER_FILES_DIRECTORY=$EXPORT_DIR/galaxy-central/database/pbs \ + GALAXY_CONFIG_FTP_UPLOAD_DIR=$EXPORT_DIR/ftp \ + GALAXY_CONFIG_FTP_UPLOAD_SITE=galaxy.docker.org \ + GALAXY_CONFIG_USE_PBKDF2=False \ + GALAXY_CONFIG_NGINX_X_ACCEL_REDIRECT_BASE=/_x_accel_redirect \ + GALAXY_CONFIG_NGINX_X_ARCHIVE_FILES_BASE=/_x_accel_redirect \ + GALAXY_CONFIG_DYNAMIC_PROXY_MANAGE=False \ + GALAXY_CONFIG_VISUALIZATION_PLUGINS_DIRECTORY=config/plugins/visualizations \ + GALAXY_CONFIG_TRUST_IPYTHON_NOTEBOOK_CONVERSION=True \ + GALAXY_CONFIG_TOOLFORM_UPGRADE=True \ + GALAXY_CONFIG_SANITIZE_ALL_HTML=False \ + GALAXY_CONFIG_TOOLFORM_UPGRADE=True \ + GALAXY_CONFIG_WELCOME_URL=$GALAXY_CONFIG_DIR/web/welcome.html \ + GALAXY_CONFIG_OVERRIDE_DEBUG=False \ + GALAXY_CONFIG_ENABLE_QUOTAS=True \ + # We need to set $HOME for some Tool Shed tools (e.g Perl libs with $HOME/.cpan) + HOME=$GALAXY_HOME \ + GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda # Container Style ADD GalaxyDocker.png $GALAXY_CONFIG_DIR/web/welcome_image.png @@ -282,10 +282,10 @@ ADD welcome.html $GALAXY_CONFIG_DIR/web/welcome.html # && find $GALAXY_VIRTUAL_ENV -name '*.pyc' -delete \ # && rm -rf /tmp/* $GALAXY_ROOT/client/node_modules/ $GALAXY_VIRTUAL_ENV/src/ /home/galaxy/.cache/ /home/galaxy/.npm - # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments - #cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \ - #npm install && \ - #rm -rf ~/.cache/ $GALAXY_ROOT/client/node_modules/ +# Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments +#cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \ +#npm install && \ +#rm -rf ~/.cache/ $GALAXY_ROOT/client/node_modules/ # Switch back to User root USER root From 15f6079540bdc934f47b48e9e915444843c4bc35 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 09:47:44 +0100 Subject: [PATCH 16/68] compose update to 20.09 --- compose/README.md | 4 ++-- compose/galaxy-htcondor/Dockerfile | 2 +- compose/galaxy-server/Dockerfile | 4 ++-- compose/galaxy-slurm/Dockerfile | 2 +- compose/tests/docker-compose.test.bioblend.yml | 2 +- compose/tests/galaxy-selenium-test/Dockerfile | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compose/README.md b/compose/README.md index 3007747d0..e56d19ac9 100644 --- a/compose/README.md +++ b/compose/README.md @@ -151,9 +151,9 @@ All options are discussed under [configuration reference](#configuration-referen ### Use specific Galaxy version or Docker images The `IMAGE_TAG` environment variable allows to use specific versions of the -setup. Say, you want to stay with Galaxy v20.05 for now: +setup. Say, you want to stay with Galaxy v20.09 for now: -> export IMAGE_TAG=20.05 +> export IMAGE_TAG=20.09 > docker-compose up Without setting this variable, you will always get updated to the newest diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index d10630bde..2d5425d40 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_TAG=latest FROM buildpack-deps:18.04 as galaxy_dependencies -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy ENV GALAXY_ROOT=/galaxy diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 2245216ec..94f94cbaf 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -50,7 +50,7 @@ RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA_VERSI FROM build_base as build_galaxy -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh @@ -73,7 +73,7 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss # --- Final image --- FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-cluster-base:$IMAGE_TAG as final -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 2f1328e8c..fdf66ceaa 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_TAG=latest FROM buildpack-deps:18.04 as galaxy_dependencies -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy ENV GALAXY_ROOT=/galaxy diff --git a/compose/tests/docker-compose.test.bioblend.yml b/compose/tests/docker-compose.test.bioblend.yml index 9ebcaf7b8..6b77fb405 100644 --- a/compose/tests/docker-compose.test.bioblend.yml +++ b/compose/tests/docker-compose.test.bioblend.yml @@ -4,7 +4,7 @@ services: image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-bioblend-test:${IMAGE_TAG:-latest} build: tests/galaxy-bioblend-test environment: - - GALAXY_VERSION=${GALAXY_VERSION:-release_20.05} # TODO: Change to GALAXY_RELEASE + - GALAXY_VERSION=${GALAXY_VERSION:-release_20.09} # TODO: Change to GALAXY_RELEASE - GALAXY_URL=http://nginx${GALAXY_PROXY_PREFIX:-}/ networks: - galaxy diff --git a/compose/tests/galaxy-selenium-test/Dockerfile b/compose/tests/galaxy-selenium-test/Dockerfile index f081a029e..453b12508 100644 --- a/compose/tests/galaxy-selenium-test/Dockerfile +++ b/compose/tests/galaxy-selenium-test/Dockerfile @@ -1,6 +1,6 @@ FROM selenium/standalone-chrome:3.141.59 -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy ENV GALAXY_ROOT=/galaxy From 032d2190b8047734a65c36b6167d20c353b6d697 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 09:50:19 +0100 Subject: [PATCH 17/68] git workflow update env vars --- .github/workflows/compose.yml | 6 +++--- .github/workflows/pull-request.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 2dbf435f6..a824ef01a 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -232,13 +232,13 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Set image tag in env - run: echo "::set-env name=IMAGE_TAG::${GITHUB_REF#refs/heads/}" + run: echo "IMAGE_TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - name: Master branch - Set image to to 'latest' if: github.ref == 'refs/heads/master' - run: echo "::set-env name=IMAGE_TAG::latest" + run: echo "IMAGE_TAG=latest" >> $GITHUB_ENV - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow - run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}" + run: echo "WORKFLOWS=${{ matrix.test.workflow }}" $GITHUB_ENV - name: Run tests for the first time if: steps.run_check.outputs.run run: | diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 9eb289190..c35105fad 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -108,7 +108,7 @@ jobs: uses: actions/checkout@v2 - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow - run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}" + run: echo "WORKFLOWS=${{ matrix.test.workflow }}" $GITHUB_ENV - name: Build galaxy-container-base env: image_name: galaxy-container-base From defa4326ea39b2b5204bd8f69bd0999b4b1d73c7 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 09:51:47 +0100 Subject: [PATCH 18/68] git workflow env correct piping --- .github/workflows/compose.yml | 2 +- .github/workflows/pull-request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index a824ef01a..1df8d1bf5 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -238,7 +238,7 @@ jobs: run: echo "IMAGE_TAG=latest" >> $GITHUB_ENV - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow - run: echo "WORKFLOWS=${{ matrix.test.workflow }}" $GITHUB_ENV + run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV - name: Run tests for the first time if: steps.run_check.outputs.run run: | diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index c35105fad..5b4aafe41 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -108,7 +108,7 @@ jobs: uses: actions/checkout@v2 - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow - run: echo "WORKFLOWS=${{ matrix.test.workflow }}" $GITHUB_ENV + run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV - name: Build galaxy-container-base env: image_name: galaxy-container-base From 2bf03b7d277754e7c25b83377da507471f8d01bc Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 09:53:04 +0100 Subject: [PATCH 19/68] git workflow switch to official galaxy repo --- .github/workflows/compose.yml | 2 +- .github/workflows/pull-request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 1df8d1bf5..7615c6eb6 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -120,7 +120,7 @@ jobs: --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \ --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \ --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \ - --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy \ + --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy \ ${{ matrix.image.subdir }}${{ matrix.image.name }} && break || echo "Fail.. Retrying" done; shell: bash diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 5b4aafe41..050f5de85 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -142,7 +142,7 @@ jobs: echo "Removing export directory if existent"; sudo rm -rf export set +e - docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy + docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }} test_exit_code=$? error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1) From e55dddee96564d3091ad794cb036fe71352ad149 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 10:00:34 +0100 Subject: [PATCH 20/68] Update to ubuntu 20.04 --- README.md | 2 ++ compose/README.md | 2 +- compose/base-images/galaxy-container-base/Dockerfile | 4 ++-- .../galaxy-configurator/templates/galaxy/job_conf.xml.j2 | 2 +- compose/galaxy-htcondor/Dockerfile | 2 +- compose/galaxy-server/Dockerfile | 2 +- compose/galaxy-slurm/Dockerfile | 2 +- compose/pulsar/Dockerfile | 2 +- compose/tests/galaxy-selenium-test/Dockerfile | 9 +++++---- galaxy/Dockerfile | 4 ++-- test/slurm/Dockerfile | 2 +- 11 files changed, 18 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a64c50650..6e23c7b71 100644 --- a/README.md +++ b/README.md @@ -993,6 +993,8 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro - Featuring Galaxy 20.05 - Completely reworked compose setup - The default admin password and apikey (`GALAXY_DEFAULT_ADMIN_PASSWORD` and `GALAXY_DEFAULT_ADMIN_KEY`) have changed: the password is now `password` (instead of `admin`) and the apikey `fakekey` (instead of `admin`). +- 20.09: + - Featuring Galaxy 20.09 # Support & Bug Reports [[toc]](#toc) diff --git a/compose/README.md b/compose/README.md index e56d19ac9..8f9d49576 100644 --- a/compose/README.md +++ b/compose/README.md @@ -378,7 +378,7 @@ The following are settings specific to this docker-compose setup: | `GALAXY_K8S_DOCKER_REPO_DEFAULT` | The Docker Repo/Registry to use if the resolver could not resolve the proper image for a job. Defaults to `docker.io`. | | `GALAXY_K8S_DOCKER_OWNER_DEFAULT` | The Owner/Username to use if the resolver could not resolve the proper image for a job. Is not set by default. | | `GALAXY_K8S_DOCKER_IMAGE_DEFAULT` | The Image to use if the resolver could not resolve the proper image for a job. Defaults to `ubuntu`. | -| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `18.04`. | +| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `20.04`. | ### HTCondor | Variable | Description | diff --git a/compose/base-images/galaxy-container-base/Dockerfile b/compose/base-images/galaxy-container-base/Dockerfile index 20fa3743f..669533266 100644 --- a/compose/base-images/galaxy-container-base/Dockerfile +++ b/compose/base-images/galaxy-container-base/Dockerfile @@ -1,4 +1,4 @@ -FROM buildpack-deps:18.04 as build_singularity +FROM buildpack-deps:20.04 as build_singularity COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh @@ -21,7 +21,7 @@ RUN wget https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_ # --- Final image --- -FROM ubuntu:18.04 as final +FROM ubuntu:20.04 as final COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh diff --git a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 index 461dfb5ff..ed4a295ca 100644 --- a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 +++ b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 @@ -59,7 +59,7 @@ {{ GALAXY_K8S_DOCKER_REPO_DEFAULT | default('docker.io') }} {% if GALAXY_K8S_DOCKER_OWNER_DEFAULT -%}{{ GALAXY_K8S_DOCKER_OWNER_DEFAULT }}{% endif -%} {{ GALAXY_K8S_DOCKER_IMAGE_DEFAULT | default('ubuntu') }} - {{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('18.04') }} + {{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('20.04') }} true {% endif -%} diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index 2d5425d40..c012c75b6 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as galaxy_dependencies +FROM buildpack-deps:20.04 as galaxy_dependencies ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 94f94cbaf..c29e5d4f9 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as build_base +FROM buildpack-deps:20.04 as build_base ENV EXPORT_DIR=/export \ GALAXY_ROOT=/galaxy \ diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index fdf66ceaa..f4c5f616d 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as galaxy_dependencies +FROM buildpack-deps:20.04 as galaxy_dependencies ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy diff --git a/compose/pulsar/Dockerfile b/compose/pulsar/Dockerfile index 2fb2a7d10..1d63ec2cc 100644 --- a/compose/pulsar/Dockerfile +++ b/compose/pulsar/Dockerfile @@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as build_pulsar +FROM buildpack-deps:20.04 as build_pulsar ARG PULSAR_RELEASE=0.13.0 ARG PULSAR_REPO=https://github.com/galaxyproject/pulsar diff --git a/compose/tests/galaxy-selenium-test/Dockerfile b/compose/tests/galaxy-selenium-test/Dockerfile index 453b12508..c72e3ba66 100644 --- a/compose/tests/galaxy-selenium-test/Dockerfile +++ b/compose/tests/galaxy-selenium-test/Dockerfile @@ -3,16 +3,17 @@ FROM selenium/standalone-chrome:3.141.59 ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy -ENV GALAXY_ROOT=/galaxy -ENV GALAXY_PYTHON=/usr/bin/python +ENV GALAXY_ROOT=/galaxy3 +ENV GALAXY_PYTHON=/usr/bin/python3 USER root -RUN apt update && apt install --no-install-recommends python-dev python-pip -y && rm -rf /var/lib/apt/lists/* \ +RUN apt update && apt install --no-install-recommends python3-dev python3-pip -y && rm -rf /var/lib/apt/lists/* \ && mkdir "${GALAXY_ROOT}" \ && chown seluser "${GALAXY_ROOT}" USER seluser -RUN curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ +RUN mkdir -p $GALAXY_ROOT && \ + curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ && cd "${GALAXY_ROOT}" \ && ./scripts/common_startup.sh --skip-client-build --dev-wheels diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 2b46ec66c..a992f1e58 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM ubuntu:18.04 +FROM ubuntu:20.04 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com @@ -16,7 +16,7 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG GALAXY_RELEASE ARG GALAXY_REPO -ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.05} \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \ GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ GALAXY_ROOT=/galaxy-central \ GALAXY_CONFIG_DIR=/etc/galaxy \ diff --git a/test/slurm/Dockerfile b/test/slurm/Dockerfile index 76495a16d..bf74cf193 100644 --- a/test/slurm/Dockerfile +++ b/test/slurm/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 ENV DEBIAN_FRONTEND noninteractive From 1fa182388a8f6d0719d12de5aa5bc8ee02c5ebde Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 10:02:27 +0100 Subject: [PATCH 21/68] git workflow update ubuntu to 20.04 --- .github/workflows/compose.yml | 8 ++++---- .github/workflows/lint.yml | 2 +- .github/workflows/pull-request.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 7615c6eb6..9d96b5d21 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -2,7 +2,7 @@ name: build-and-test on: [push] jobs: build_container_base: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v2 @@ -38,7 +38,7 @@ jobs: working-directory: ./compose/base-images build_cluster_base: needs: build_container_base - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v2 @@ -74,7 +74,7 @@ jobs: working-directory: ./compose/base-images build: needs: build_cluster_base - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: image: @@ -127,7 +127,7 @@ jobs: working-directory: ./compose test: needs: [build] - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: infrastructure: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 63693b86d..f0c4b46d2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,7 +2,7 @@ name: Lint on: [push] jobs: lint: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 050f5de85..c660a7ca2 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -2,7 +2,7 @@ name: pr-test on: pull_request jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: infrastructure: From d0e42f9aa238dca36a5e3f67082097f6538952c5 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 10:03:19 +0100 Subject: [PATCH 22/68] set thunder-lock true --- compose/base_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/base_config.yml b/compose/base_config.yml index cc2970901..f3640e038 100644 --- a/compose/base_config.yml +++ b/compose/base_config.yml @@ -9,7 +9,7 @@ galaxy_uwsgi: master: false virtualenv: .venv pythonpath: lib - thunder-lock: false + thunder-lock: true die-on-term: true hook-master-start: unix_signal:2 gracefully_kill_them_all hook-master-start: unix_signal:15 gracefully_kill_them_all From 6ebbdc4fd91a8625d8496ca968db95358307b012 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 10:03:53 +0100 Subject: [PATCH 23/68] remove py-call-osafterfork --- compose/base_config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/base_config.yml b/compose/base_config.yml index f3640e038..46f37dc7c 100644 --- a/compose/base_config.yml +++ b/compose/base_config.yml @@ -13,7 +13,6 @@ galaxy_uwsgi: die-on-term: true hook-master-start: unix_signal:2 gracefully_kill_them_all hook-master-start: unix_signal:15 gracefully_kill_them_all - py-call-osafterfork: true enable-threads: true galaxy: From 7aa4bef6fa1fb7e8a6e182a9b0fb3c73349d0b22 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 10:23:02 +0100 Subject: [PATCH 24/68] singularity add galaxy env file resolution --- compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 index ed4a295ca..9b1ca49ed 100644 --- a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 +++ b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 @@ -32,6 +32,7 @@ {% if GALAXY_DEPENDENCY_RESOLUTION == 'singularity' -%} + /home/galaxy true {% if GALAXY_JOB_RUNNER == 'local' -%} From 7f898dfd3ce30262494c600aa59b3b955b1f7bb6 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 13:29:34 +0100 Subject: [PATCH 25/68] update Miniconda version --- compose/galaxy-server/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index c29e5d4f9..e3ce4726b 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -24,7 +24,7 @@ ENV GALAXY_USER=galaxy \ GALAXY_HOME=/home/galaxy ENV GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda \ - MINICONDA_VERSION=4.7.10 + MINICONDA_VERSION=py38_4.9.2 RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ && useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" --shell /bin/bash $GALAXY_USER \ @@ -34,7 +34,7 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ FROM build_base as build_miniconda COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Install Miniconda -RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA_VERSION}-Linux-x86_64.sh" > ~/miniconda.sh \ +RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ From 13177af8b7ee4bcadc1f8c2286cd4f86b162acb0 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 13:31:08 +0100 Subject: [PATCH 26/68] upgrade pulsar to v0.14.0 --- compose/pulsar/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/pulsar/Dockerfile b/compose/pulsar/Dockerfile index 1d63ec2cc..5d485cc81 100644 --- a/compose/pulsar/Dockerfile +++ b/compose/pulsar/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_TAG=latest FROM buildpack-deps:20.04 as build_pulsar -ARG PULSAR_RELEASE=0.13.0 +ARG PULSAR_RELEASE=0.14.0 ARG PULSAR_REPO=https://github.com/galaxyproject/pulsar ENV PULSAR_ROOT=/pulsar From a4cabfbef1e0c1ca1246d99750a9b6671fe8331b Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 13:31:34 +0100 Subject: [PATCH 27/68] upgrade container-base sources to focal --- compose/base-images/galaxy-cluster-base/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/base-images/galaxy-cluster-base/Dockerfile b/compose/base-images/galaxy-cluster-base/Dockerfile index f9d668009..dc5a2cb08 100644 --- a/compose/base-images/galaxy-cluster-base/Dockerfile +++ b/compose/base-images/galaxy-cluster-base/Dockerfile @@ -20,8 +20,8 @@ ENV MUNGER_USER=munge \ MUNGE_GID=1200 RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ && useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \ - && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \ - && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \ + && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \ + && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \ && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8DE68488997C5C6BA19021136F2CC56412788738 \ && apt update \ && apt install --no-install-recommends slurm-client slurmd slurmctld slurm-drmaa1 -y \ From 332357f27a8b18852f7eae62e335822f31c58b47 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 13:36:34 +0100 Subject: [PATCH 28/68] update to python3 --- compose/base-images/galaxy-cluster-base/Dockerfile | 2 +- compose/base-images/galaxy-container-base/Dockerfile | 2 +- compose/galaxy-server/Dockerfile | 3 +-- compose/pulsar/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/compose/base-images/galaxy-cluster-base/Dockerfile b/compose/base-images/galaxy-cluster-base/Dockerfile index dc5a2cb08..ff7c91cce 100644 --- a/compose/base-images/galaxy-cluster-base/Dockerfile +++ b/compose/base-images/galaxy-cluster-base/Dockerfile @@ -24,7 +24,7 @@ RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \ && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8DE68488997C5C6BA19021136F2CC56412788738 \ && apt update \ - && apt install --no-install-recommends slurm-client slurmd slurmctld slurm-drmaa1 -y \ + && apt install --no-install-recommends python3-distutils slurm-client slurmd slurmctld slurm-drmaa1 -y \ && apt --no-install-recommends install munge libmunge-dev -y \ && ln -s /usr/lib/slurm-drmaa/lib/libdrmaa.so.1 /usr/lib/slurm-drmaa/lib/libdrmaa.so \ && /usr/bin/common_cleanup.sh diff --git a/compose/base-images/galaxy-container-base/Dockerfile b/compose/base-images/galaxy-container-base/Dockerfile index 669533266..6cf33a516 100644 --- a/compose/base-images/galaxy-container-base/Dockerfile +++ b/compose/base-images/galaxy-container-base/Dockerfile @@ -26,7 +26,7 @@ FROM ubuntu:20.04 as final COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Base dependencies -RUN apt update && apt install --no-install-recommends ca-certificates squashfs-tools -y \ +RUN apt update && apt install --no-install-recommends ca-certificates python3-distutils squashfs-tools -y \ && /usr/bin/common_cleanup.sh # Install Docker diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index e3ce4726b..a7521ce4e 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -120,8 +120,7 @@ RUN apt update && apt install --no-install-recommends curl gcc gnupg2 libgomp1 l RUN usermod -aG docker $GALAXY_USER # Make Python3 standard -RUN mv /usr/bin/python /usr/bin/python2 \ - && ln /usr/bin/python3 /usr/bin/python +RUN ln /usr/bin/python3 /usr/bin/python COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_galaxy ${GALAXY_ROOT} ${GALAXY_ROOT} COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} diff --git a/compose/pulsar/Dockerfile b/compose/pulsar/Dockerfile index 5d485cc81..dac9b1503 100644 --- a/compose/pulsar/Dockerfile +++ b/compose/pulsar/Dockerfile @@ -34,7 +34,7 @@ ENV PULSAR_VIRTUALENV=$PULSAR_ROOT/.venv \ PULSAR_TOOL_DEPENDENCY_DIR=$PULSAR_ROOT/dependencies RUN apt update \ - && apt install --no-install-recommends ca-certificates curl libxml2-dev libpython3.6 python3 -y \ + && apt install --no-install-recommends ca-certificates curl libxml2-dev python3 -y \ && /usr/bin/common_cleanup.sh COPY --from=build_pulsar /pulsar /pulsar From 8face0342e9072347641bf184f5e67df87ca859d Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 20:33:46 +0100 Subject: [PATCH 29/68] workflow tests ignore filetype --- compose/tests/galaxy-workflow-test/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/tests/galaxy-workflow-test/Dockerfile b/compose/tests/galaxy-workflow-test/Dockerfile index b392babbb..a5ca295ba 100644 --- a/compose/tests/galaxy-workflow-test/Dockerfile +++ b/compose/tests/galaxy-workflow-test/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.11 -ENV TEST_REPO=${TEST_REPO:-https://github.com/usegalaxy-eu/workflow-testing} \ - TEST_RELEASE=${TEST_RELEASE:-master} +ENV TEST_REPO=${TEST_REPO:-https://github.com/jonas27/workflow-testing} \ + TEST_RELEASE=${TEST_RELEASE:-20.09-comment-filetype} RUN apk add --no-cache bash python3 curl \ && apk add --no-cache --virtual build-dep gcc libxml2-dev libxslt-dev musl-dev linux-headers python3-dev \ From e2f69c1972e5da72587b63535980ff7df80b9fdc Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 06:39:47 +0100 Subject: [PATCH 30/68] python3 standard --- compose/galaxy-htcondor/Dockerfile | 2 ++ compose/galaxy-server/Dockerfile | 2 +- compose/galaxy-slurm/Dockerfile | 2 ++ compose/tests/galaxy-workflow-test/Dockerfile | 3 +++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index c012c75b6..7fc2c5d06 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -67,4 +67,6 @@ COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT $ COPY start.sh /usr/bin/start.sh RUN apt update && apt install python3 -y +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 + ENTRYPOINT /usr/bin/start.sh diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index a7521ce4e..06b40b236 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -120,7 +120,7 @@ RUN apt update && apt install --no-install-recommends curl gcc gnupg2 libgomp1 l RUN usermod -aG docker $GALAXY_USER # Make Python3 standard -RUN ln /usr/bin/python3 /usr/bin/python +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_galaxy ${GALAXY_ROOT} ${GALAXY_ROOT} COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index f4c5f616d..760749ab2 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -50,6 +50,8 @@ RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ # Copy Galaxy dependencies COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT $GALAXY_ROOT +# Make Python3 standard +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 COPY start.sh /usr/bin/start.sh diff --git a/compose/tests/galaxy-workflow-test/Dockerfile b/compose/tests/galaxy-workflow-test/Dockerfile index a5ca295ba..1b9385da9 100644 --- a/compose/tests/galaxy-workflow-test/Dockerfile +++ b/compose/tests/galaxy-workflow-test/Dockerfile @@ -10,6 +10,9 @@ RUN apk add --no-cache bash python3 curl \ && curl -L -s $TEST_REPO/archive/$TEST_RELEASE.tar.gz | tar xzf - --strip-components=1 \ && apk del build-dep +# Make Python3 standard +RUN ln /usr/bin/python3 /usr/bin/python && ln /usr/bin/python3 /usr/bin/python2 + ADD ./run.sh /usr/bin/run.sh WORKDIR /src From 7432ab43b143af5d56ffb9f55031fa8cf0daf84c Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 07:32:41 +0100 Subject: [PATCH 31/68] galaxy dependency add files --- compose/galaxy-htcondor/Dockerfile | 2 +- compose/galaxy-slurm/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index 7fc2c5d06..ac49a826b 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -18,7 +18,7 @@ RUN mkdir "${GALAXY_ROOT}" \ && cd $GALAXY_ROOT/lib \ && ls . | grep -v "galaxy\|galaxy_ext" | xargs rm -rf \ && cd $GALAXY_ROOT/lib/galaxy \ - && ls . | grep -v "__init__.py\|datatypes\|exceptions\|metadata\|model\|util\|security" | xargs rm -rf + && ls . | grep -v "__init__.py\|datatypes\|exceptions\|files\|metadata\|model\|util\|security" | xargs rm -rf FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-container-base:$IMAGE_TAG as final diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 760749ab2..13d6b6ce9 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -17,7 +17,7 @@ RUN mkdir "${GALAXY_ROOT}" \ && cd $GALAXY_ROOT/lib \ && ls . | grep -v "galaxy\|galaxy_ext" | xargs rm -rf \ && cd $GALAXY_ROOT/lib/galaxy \ - && ls . | grep -v "__init__.py\|datatypes\|exceptions\|metadata\|model\|util\|security" | xargs rm -rf + && ls . | grep -v "__init__.py\|datatypes\|exceptions\|files\|metadata\|model\|util\|security" | xargs rm -rf FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-container-base:$IMAGE_TAG as final From 4ac618ecae5985c6ca0060dbc2ee3c15fda2a55f Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 10:16:46 +0100 Subject: [PATCH 32/68] galaxy-server set repo env --- compose/galaxy-server/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 06b40b236..314ba269a 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -73,9 +73,6 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss # --- Final image --- FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-cluster-base:$IMAGE_TAG as final -ARG GALAXY_RELEASE=release_20.09 -ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy - COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh COPY ./files/create_galaxy_user.py /usr/local/bin/create_galaxy_user.py @@ -84,7 +81,9 @@ ENV EXPORT_DIR=/export \ GALAXY_PYTHON=/usr/bin/python3 \ HTCONDOR_ROOT=/opt/htcondor -ENV GALAXY_STATIC_DIR=$GALAXY_ROOT/static \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \ + GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ + GALAXY_STATIC_DIR=$GALAXY_ROOT/static \ GALAXY_EXPORT=$EXPORT_DIR/galaxy \ GALAXY_CONFIG_DIR=$GALAXY_ROOT/config \ GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=/tool_deps \ From 656d75b620f511df512108ebd1c6f81e265a98a1 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 14:53:05 +0100 Subject: [PATCH 33/68] htcondor and slurm include galaxy lib tools --- compose/docker-compose.htcondor.yml | 1 + compose/docker-compose.slurm.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/compose/docker-compose.htcondor.yml b/compose/docker-compose.htcondor.yml index 1b7c36702..c0579a14e 100644 --- a/compose/docker-compose.htcondor.yml +++ b/compose/docker-compose.htcondor.yml @@ -28,6 +28,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/htcondor:/config - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database + - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/lib/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv diff --git a/compose/docker-compose.slurm.yml b/compose/docker-compose.slurm.yml index 134f5e06d..5d55c7512 100644 --- a/compose/docker-compose.slurm.yml +++ b/compose/docker-compose.slurm.yml @@ -40,6 +40,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro + - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/lib/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv - ${EXPORT_DIR:-./export}/tool_deps:/tool_deps From 09a842f1ff8ea0308897479194db03be1ea7d09f Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 14:55:18 +0100 Subject: [PATCH 34/68] galaxy make ython3 standard --- compose/galaxy-server/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 314ba269a..19feaeb18 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -56,6 +56,7 @@ ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Install Galaxy RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libssl-dev python3-dev python3-pip -y \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3 9 \ && mkdir "${GALAXY_ROOT}" \ && curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ && cd $GALAXY_ROOT \ From 06a0877d0fccc464b185e4f6e71bf7d449d6d7c0 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 14:58:05 +0100 Subject: [PATCH 35/68] galaxy-server pip install deps --- compose/galaxy-server/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 19feaeb18..d789c2250 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -63,6 +63,7 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss && ./scripts/common_startup.sh \ && . $GALAXY_ROOT/.venv/bin/activate \ && pip3 install drmaa psycopg2 pycurl pykube \ + && pip3 install importlib-metadata importlib-resources pathlib2 ruamel.yaml.clib typing zipp \ && deactivate \ && rm -rf .ci .circleci .coveragerc .gitignore .travis.yml CITATION CODE_OF_CONDUCT.md CONTRIBUTING.md CONTRIBUTORS.md \ LICENSE.txt Makefile README.rst SECURITY_POLICY.md pytest.ini tox.ini \ From 31066d0b7c67aba122fd51f0de32dece78b62a20 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 14:59:40 +0100 Subject: [PATCH 36/68] slurm python3 --- compose/galaxy-slurm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 13d6b6ce9..6f54338b2 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -45,7 +45,7 @@ ENV SLURM_USER=galaxy \ RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ && useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \ && apt update \ - && apt install --no-install-recommends gosu munge python python-dev slurm-wlm -y \ + && apt install --no-install-recommends gosu munge python3 python3-dev slurm-wlm -y \ && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/* && find / -name '*.pyc' -delete # Copy Galaxy dependencies From 3ecdf5044c1e558cd63ca562087cc2da671dd956 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 15:01:25 +0100 Subject: [PATCH 37/68] selenium test correct typo --- compose/tests/galaxy-selenium-test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/tests/galaxy-selenium-test/Dockerfile b/compose/tests/galaxy-selenium-test/Dockerfile index c72e3ba66..370264852 100644 --- a/compose/tests/galaxy-selenium-test/Dockerfile +++ b/compose/tests/galaxy-selenium-test/Dockerfile @@ -3,7 +3,7 @@ FROM selenium/standalone-chrome:3.141.59 ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy -ENV GALAXY_ROOT=/galaxy3 +ENV GALAXY_ROOT=/galaxy ENV GALAXY_PYTHON=/usr/bin/python3 USER root From b16127cef3110d3b297e6fc0857f7762d195f4cf Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 16:43:18 +0100 Subject: [PATCH 38/68] readme add tests matrix --- compose/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/compose/README.md b/compose/README.md index 8f9d49576..41aaaf46c 100644 --- a/compose/README.md +++ b/compose/README.md @@ -393,3 +393,20 @@ The following are settings specific to this docker-compose setup: | `SLURM_NODE_CPUS` | Number of CPUs per node. Defaults to 1. | | `SLURM_NODE_MEMORY` | Amount of memory per node. Defaults to 1024. | | `SLURM_NODE_HOSTNAME` | Docker Compose adds a prefix in front of the container names by default. Change this value to the name of your setup and `_slurm_node` (e.g. `compose_slurm_node`) to ensure a correct mapping of the Slurm nodes. | + +### Github Workflow Tests (Branch 20.09) +| Setup | bioblend | workflow ard | workflow mapping_by_sequencing | workflow wf3-shed-tools (example1) | selenium | +|------------------------|--------------------|--------------------|--------------------------------|------------------------------------|--------------------| +| Galaxy Base | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| Galaxy Proxy Prefix | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| HTCondor | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| Slurm | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| Pulsar | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | +| k8s | :x: | :x: | :x: | :heavy_check_mark: | :x: | +| Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | +| Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | +| HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | + + +Implemented: :heavy_check_mark: +Not Implemented: :x: From 257b1c1eb939f9812d31cc820fdb7397ab3a7126 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 17:15:42 +0100 Subject: [PATCH 39/68] galaxy tools volume export in galaxy and mount in htcondor and slurm --- compose/docker-compose.htcondor.yml | 2 +- compose/docker-compose.slurm.yml | 2 +- compose/galaxy-server/files/start.sh | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/compose/docker-compose.htcondor.yml b/compose/docker-compose.htcondor.yml index c0579a14e..35084397f 100644 --- a/compose/docker-compose.htcondor.yml +++ b/compose/docker-compose.htcondor.yml @@ -28,7 +28,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/htcondor:/config - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database - - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/lib/galaxy/tools:ro + - ${EXPORT_DIR:-./export}/galaxy/lib/galaxy/tools:/galaxy/lib/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv diff --git a/compose/docker-compose.slurm.yml b/compose/docker-compose.slurm.yml index 5d55c7512..4cf6e29cb 100644 --- a/compose/docker-compose.slurm.yml +++ b/compose/docker-compose.slurm.yml @@ -40,7 +40,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro - - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/lib/galaxy/tools:ro + - ${EXPORT_DIR:-./export}/galaxy/lib/galaxy/tools:/galaxy/lib/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv - ${EXPORT_DIR:-./export}/tool_deps:/tool_deps diff --git a/compose/galaxy-server/files/start.sh b/compose/galaxy-server/files/start.sh index baf672165..860280856 100755 --- a/compose/galaxy-server/files/start.sh +++ b/compose/galaxy-server/files/start.sh @@ -13,6 +13,14 @@ create_user() { deactivate } +# start copy lib/tools. Looks very hacky. +tools_dir="/galaxy/lib/galaxy/tools/" +exp_dir="/export$tools_dir" +mkdir -p $exp_dir +chown "$GALAXY_USER:$GALAXY_USER" $exp_dir +cp -rf $tools_dir/* $exp_dir +# end copy lib/tools. + # First start?? Check if something exists that indicates that environment is not new.. Config file? Something in DB maybe?? echo "Initialization: Check if files already exist, export otherwise." From 23bb51b00eab436f63372bb1e6b4136ad6a99349 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 18:18:52 +0100 Subject: [PATCH 40/68] README tiny delete --- compose/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/README.md b/compose/README.md index 41aaaf46c..a2c913d67 100644 --- a/compose/README.md +++ b/compose/README.md @@ -407,6 +407,5 @@ The following are settings specific to this docker-compose setup: | Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | | HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | - Implemented: :heavy_check_mark: Not Implemented: :x: From 1b7eda1bb24351a477faf6d62af88774fca422c3 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 19:35:30 +0100 Subject: [PATCH 41/68] github workflow use ubuntu-latest again --- .github/workflows/compose.yml | 6 +++--- .github/workflows/lint.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 9d96b5d21..475bbff93 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -38,7 +38,7 @@ jobs: working-directory: ./compose/base-images build_cluster_base: needs: build_container_base - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 @@ -74,7 +74,7 @@ jobs: working-directory: ./compose/base-images build: needs: build_cluster_base - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: image: @@ -127,7 +127,7 @@ jobs: working-directory: ./compose test: needs: [build] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: infrastructure: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f0c4b46d2..63693b86d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,7 +2,7 @@ name: Lint on: [push] jobs: lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 From 272c2062ae290f5d99b8adc57f50d864d5fda94e Mon Sep 17 00:00:00 2001 From: jonas27 Date: Wed, 10 Feb 2021 06:30:28 +0100 Subject: [PATCH 42/68] Revert "README tiny delete" This reverts commit 23bb51b00eab436f63372bb1e6b4136ad6a99349. --- compose/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/README.md b/compose/README.md index a2c913d67..41aaaf46c 100644 --- a/compose/README.md +++ b/compose/README.md @@ -407,5 +407,6 @@ The following are settings specific to this docker-compose setup: | Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | | HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | + Implemented: :heavy_check_mark: Not Implemented: :x: From 85be8c5ecb0890e747c2ee299e00879f6a647521 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Wed, 10 Feb 2021 11:11:15 +0100 Subject: [PATCH 43/68] exclude pulsar and k8s not working test --- .github/workflows/pull-request.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index c660a7ca2..12baa530c 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -30,6 +30,7 @@ jobs: exclude_test: - workflow_example1 - workflow_mapping_by_sequencing + - workflow_ard # - name: galaxy-pulsar-mq # files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml # exclude_test: @@ -41,6 +42,7 @@ jobs: - bioblend - workflow_ard - workflow_mapping_by_sequencing + - workflow_example1 - selenium - name: galaxy-singularity files: -f docker-compose.yml -f docker-compose.singularity.yml From 17d57b500032d1d9fa9889fc270dfdbf00b698b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 10 Feb 2021 13:09:43 +0100 Subject: [PATCH 44/68] Update Dockerfile --- galaxy/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 77e2fe516..d6ee7325e 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -24,7 +24,6 @@ ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \ DEBIAN_FRONTEND=noninteractive \ PG_VERSION=11 - ENV GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml \ GALAXY_CONFIG_JOB_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_conf.xml \ GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_metrics_conf.xml \ From 004bc785a07544f1421daf835d25acf39f81ff05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 10 Feb 2021 13:24:34 +0100 Subject: [PATCH 45/68] update conda --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index d6ee7325e..a7b7972c7 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -118,7 +118,7 @@ ADD ./bashrc $GALAXY_HOME/.bashrc # Install miniconda, then virtualenv from conda and then # download latest stable release of Galaxy. -RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.3-Linux-x86_64.sh > ~/miniconda.sh \ +RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py37_4.9.2-Linux-x86_64.sh > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX/ \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ From 9c16da37e7295a4039e475c7fcca254f8085d4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 10 Feb 2021 18:19:33 +0100 Subject: [PATCH 46/68] Update Dockerfile --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index a7b7972c7..b49f1dc40 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -118,7 +118,7 @@ ADD ./bashrc $GALAXY_HOME/.bashrc # Install miniconda, then virtualenv from conda and then # download latest stable release of Galaxy. -RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py37_4.9.2-Linux-x86_64.sh > ~/miniconda.sh \ +RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py39_4.9.2-Linux-x86_64.sh > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX/ \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ From 52e10582f5edcc3d0c413ce42dd1179545ff8da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 10 Feb 2021 18:39:23 +0100 Subject: [PATCH 47/68] Update Dockerfile --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index b49f1dc40..3eeae088b 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -118,7 +118,7 @@ ADD ./bashrc $GALAXY_HOME/.bashrc # Install miniconda, then virtualenv from conda and then # download latest stable release of Galaxy. -RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py39_4.9.2-Linux-x86_64.sh > ~/miniconda.sh \ +RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py38_4.9.2-Linux-x86_64.sh > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX/ \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ From 700abc8ddf620d2a27c13d5bf1433a1e247f2c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 10 Feb 2021 19:36:21 +0100 Subject: [PATCH 48/68] Update Dockerfile --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 3eeae088b..d6ee7325e 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -118,7 +118,7 @@ ADD ./bashrc $GALAXY_HOME/.bashrc # Install miniconda, then virtualenv from conda and then # download latest stable release of Galaxy. -RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py38_4.9.2-Linux-x86_64.sh > ~/miniconda.sh \ +RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.3-Linux-x86_64.sh > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX/ \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ From 8ebbe358750d2ab0703764def58a70791707b0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 10 Feb 2021 20:38:07 +0100 Subject: [PATCH 49/68] Update Dockerfile --- galaxy/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index d6ee7325e..f3dcf83ff 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -129,6 +129,7 @@ RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.3-Linux-x && conda config --add channels bioconda \ && conda config --add channels conda-forge \ && conda install virtualenv pip ephemeris \ + && conda install python=3.6 -y \ && chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh \ && conda clean --packages -t -i \ # cleanup dance From f215661808bc361b7902bee1c1fa97fb350466a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 10 Feb 2021 22:47:57 +0100 Subject: [PATCH 50/68] Update Dockerfile --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index f3dcf83ff..ca28950f4 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -118,7 +118,7 @@ ADD ./bashrc $GALAXY_HOME/.bashrc # Install miniconda, then virtualenv from conda and then # download latest stable release of Galaxy. -RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.3-Linux-x86_64.sh > ~/miniconda.sh \ +RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX/ \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ From e167f773cc777b618bebec16ae369a3c64767a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 10 Feb 2021 23:02:44 +0100 Subject: [PATCH 51/68] Update Dockerfile --- galaxy/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index ca28950f4..103437d29 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -129,7 +129,6 @@ RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86 && conda config --add channels bioconda \ && conda config --add channels conda-forge \ && conda install virtualenv pip ephemeris \ - && conda install python=3.6 -y \ && chown $GALAXY_USER:$GALAXY_USER -R /tool_deps/ /etc/profile.d/conda.sh \ && conda clean --packages -t -i \ # cleanup dance From e0362fc1b7f140297e8c1c6235fc5e78208163ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Thu, 11 Feb 2021 00:17:50 +0100 Subject: [PATCH 52/68] Update Dockerfile --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 103437d29..848d581fc 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -118,7 +118,7 @@ ADD ./bashrc $GALAXY_HOME/.bashrc # Install miniconda, then virtualenv from conda and then # download latest stable release of Galaxy. -RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh > ~/miniconda.sh \ +RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX/ \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ From 68bc2cfeddb14b987fe5e2adf0807f6270d50b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Thu, 11 Feb 2021 10:14:14 +0100 Subject: [PATCH 53/68] Update Dockerfile --- galaxy/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 848d581fc..af107797e 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -309,6 +309,9 @@ RUN echo "DISCARD_SESSION_KEYRING_ON_STARTUP=False" > /etc/condor/condor_config. ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini /sbin/tini RUN chmod +x /sbin/tini +# https://stackoverflow.com/questions/62250160/uwsgi-runtimeerror-cannot-release-un-acquired-lock +ADD sed -i 's/py-call-osafterfork.*//g' /galaxy-central/config/galaxy.yml.sample + # This needs to happen here and not above, otherwise the Galaxy start # (without running the startup.sh script) will crash because integrated_tool_panel.xml could not be found. ENV GALAXY_CONFIG_INTEGRATED_TOOL_PANEL_CONFIG $EXPORT_DIR/galaxy-central/integrated_tool_panel.xml From fbf008dd3257b2f25b84c6c2e3c93ae6807bec85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Thu, 11 Feb 2021 10:26:12 +0100 Subject: [PATCH 54/68] Update Dockerfile --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index af107797e..c47092831 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -310,7 +310,7 @@ ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini /sbin/tini RUN chmod +x /sbin/tini # https://stackoverflow.com/questions/62250160/uwsgi-runtimeerror-cannot-release-un-acquired-lock -ADD sed -i 's/py-call-osafterfork.*//g' /galaxy-central/config/galaxy.yml.sample +RUN sed -i 's/py-call-osafterfork.*//g' /galaxy-central/config/galaxy.yml.sample # This needs to happen here and not above, otherwise the Galaxy start # (without running the startup.sh script) will crash because integrated_tool_panel.xml could not be found. From 9e1c0a7e60f947a32ed99fadb7f77d446f5da992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Thu, 11 Feb 2021 10:52:04 +0100 Subject: [PATCH 55/68] Update Dockerfile --- galaxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index c47092831..85f426616 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -310,7 +310,7 @@ ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini /sbin/tini RUN chmod +x /sbin/tini # https://stackoverflow.com/questions/62250160/uwsgi-runtimeerror-cannot-release-un-acquired-lock -RUN sed -i 's/py-call-osafterfork.*//g' /galaxy-central/config/galaxy.yml.sample +RUN sed -i 's/py-call-osafterfork.*//g' /etc/galaxy/galaxy.yml # This needs to happen here and not above, otherwise the Galaxy start # (without running the startup.sh script) will crash because integrated_tool_panel.xml could not be found. From 1a255419562abb9ce2cfa5263556a6a513e70df3 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Thu, 11 Feb 2021 12:42:32 +0100 Subject: [PATCH 56/68] add hack --- galaxy/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 85f426616..ffe96856e 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -310,7 +310,8 @@ ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini /sbin/tini RUN chmod +x /sbin/tini # https://stackoverflow.com/questions/62250160/uwsgi-runtimeerror-cannot-release-un-acquired-lock -RUN sed -i 's/py-call-osafterfork.*//g' /etc/galaxy/galaxy.yml +ADD run.sh $GALAXY_ROOT/run.sh +RUN chmod +x $GALAXY_ROOT/run.sh && sed -i 's/py-call-osafterfork.*//g' /etc/galaxy/galaxy.yml # This needs to happen here and not above, otherwise the Galaxy start # (without running the startup.sh script) will crash because integrated_tool_panel.xml could not be found. From a4f73f67f0c32aa68aaca0af017028b2b1f6a932 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Thu, 11 Feb 2021 12:42:41 +0100 Subject: [PATCH 57/68] add hack --- galaxy/run.sh | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100755 galaxy/run.sh diff --git a/galaxy/run.sh b/galaxy/run.sh new file mode 100755 index 000000000..fa54e0f25 --- /dev/null +++ b/galaxy/run.sh @@ -0,0 +1,98 @@ +#!/bin/sh + + +# Usage: ./run.sh +# +# +# Description: This script can be used to start or stop the galaxy +# web application. + +cd "$(dirname "$0")" + +. ./scripts/common_startup_functions.sh + + +# If there is a file that defines a shell environment specific to this +# instance of Galaxy, source the file. +if [ -z "$GALAXY_LOCAL_ENV_FILE" ]; +then + GALAXY_LOCAL_ENV_FILE='./config/local_env.sh' +fi + +if [ -f "$GALAXY_LOCAL_ENV_FILE" ]; +then + . "$GALAXY_LOCAL_ENV_FILE" +fi + +GALAXY_PID=${GALAXY_PID:-galaxy.pid} +GALAXY_LOG=${GALAXY_LOG:-galaxy.log} +PID_FILE=$GALAXY_PID +LOG_FILE=$GALAXY_LOG + +parse_common_args $@ + +run_common_start_up + +setup_python + + +if [ ! -z "$GALAXY_RUN_WITH_TEST_TOOLS" ]; +then + export GALAXY_CONFIG_OVERRIDE_TOOL_CONFIG_FILE="$(pwd)/test/functional/tools/samples_tool_conf.xml" + export GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES="true" + export GALAXY_CONFIG_OVERRIDE_ENABLE_BETA_TOOL_FORMATS="true" + export GALAXY_CONFIG_INTERACTIVETOOLS_ENABLE="true" + export GALAXY_CONFIG_OVERRIDE_WEBHOOKS_DIR="test/functional/webhooks" +fi + +set_galaxy_config_file_var + + +if [ "$INITIALIZE_TOOL_DEPENDENCIES" -eq 1 ]; then + # Install Conda environment if needed. + python ./scripts/manage_tool_dependencies.py init_if_needed +fi + +[ -n "$GALAXY_UWSGI" ] && APP_WEBSERVER='uwsgi' +find_server "${GALAXY_CONFIG_FILE:-none}" galaxy +server_args=`echo $server_args | sed 's/--py-call-osafterfork//g'` + + +if [ "$run_server" = "python" -a -n "$GALAXY_RUN_ALL" ]; then + servers=$(sed -n 's/^\[server:\(.*\)\]/\1/ p' "$GALAXY_CONFIG_FILE" | xargs echo) + if [ -z "$stop_daemon_arg_set" -a -z "$daemon_or_restart_arg_set" ]; then + echo "ERROR: \$GALAXY_RUN_ALL cannot be used without the '--daemon', '--stop-daemon', 'restart', 'start' or 'stop' arguments to run.sh" + exit 1 + fi + for server in $servers; do + echo "Executing: python $server_args --server-name=\"$server\" --pid-file=\"$server.pid\" --log-file=\"$server.log\"" + eval python $server_args --server-name="$server" --pid-file="$server.pid" --log-file="$server.log" + if [ -n "$wait_arg_set" -a -n "$daemon_or_restart_arg_set" ]; then + while true; do + sleep 1 + # Grab the current pid from the pid file and remove any trailing space + if ! current_pid_in_file=$(sed -e 's/[[:space:]]*$//' "$server.pid"); then + echo "A Galaxy process died, interrupting" >&2 + exit 1 + fi + if [ -n "$current_pid_in_file" ]; then + echo "Found PID $current_pid_in_file in '$server.pid', monitoring '$server.log'" + else + echo "No PID found in '$server.pid' yet" + continue + fi + # Search for all pids in the logs and tail for the last one + latest_pid=$(grep '^Starting server in PID [0-9]\+\.$' "$server.log" | sed 's/^Starting server in PID \([0-9]\{1,\}\).$/\1/' | tail -n 1) + # If they're equivalent, then the current pid file agrees with our logs + # and we've succesfully started + [ -n "$latest_pid" ] && [ "$latest_pid" -eq "$current_pid_in_file" ] && break + done + echo + fi + done +else + + echo "Executing: $run_server $server_args" + # args are properly quoted so use eval + eval $run_server $server_args +fi From 8af26008ffd4f67cbe109364500197cd8853fd3b Mon Sep 17 00:00:00 2001 From: jonas27 Date: Wed, 17 Feb 2021 07:29:38 +0100 Subject: [PATCH 58/68] Trigger PR Build --- compose/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/README.md b/compose/README.md index 41aaaf46c..d8e6eda1b 100644 --- a/compose/README.md +++ b/compose/README.md @@ -410,3 +410,4 @@ The following are settings specific to this docker-compose setup: Implemented: :heavy_check_mark: Not Implemented: :x: + From 7dd620ec87b03a5e40ff5eed431c36ffca5447ce Mon Sep 17 00:00:00 2001 From: jonas27 Date: Wed, 17 Feb 2021 08:23:50 +0100 Subject: [PATCH 59/68] README test matrix add not implemented tests --- compose/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/README.md b/compose/README.md index d8e6eda1b..d73d39a5d 100644 --- a/compose/README.md +++ b/compose/README.md @@ -401,8 +401,8 @@ The following are settings specific to this docker-compose setup: | Galaxy Proxy Prefix | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | | HTCondor | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | | Slurm | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | -| Pulsar | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | -| k8s | :x: | :x: | :x: | :heavy_check_mark: | :x: | +| Pulsar | :heavy_check_mark: | :x: | :x: | :x: | :heavy_check_mark: | +| k8s | :x: | :x: | :x: | :x: | :x: | | Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | | Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | | HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | From f72d8977b65e00691c0ca1598d4668a41d60d80c Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 16 Apr 2021 11:54:06 +0200 Subject: [PATCH 60/68] update hadolint action --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 63693b86d..1677c1094 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ jobs: level: warning pattern: "*.sh" - name: Run hadolint with reviewdog - uses: reviewdog/action-hadolint@v1 + uses: reviewdog/action-hadolint@v1.17.1 with: github_token: ${{ secrets.GITHUB_TOKEN }} reporter: github-check From 64d2618ef8a87660be641222e235476eee09f05b Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 16 Apr 2021 11:56:10 +0200 Subject: [PATCH 61/68] hadolint --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1677c1094..4cbe31dc4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ jobs: level: warning pattern: "*.sh" - name: Run hadolint with reviewdog - uses: reviewdog/action-hadolint@v1.17.1 + uses: reviewdog/action-hadolint@v1.16.1 with: github_token: ${{ secrets.GITHUB_TOKEN }} reporter: github-check From 0aa5631ac521d6aaedafad22aec125cfca2a150b Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 16 Apr 2021 12:19:10 +0200 Subject: [PATCH 62/68] hadolint --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4cbe31dc4..36692201a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ jobs: level: warning pattern: "*.sh" - name: Run hadolint with reviewdog - uses: reviewdog/action-hadolint@v1.16.1 + uses: reviewdog/action-hadolint@v1.16.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} reporter: github-check From 0bea7137a0616c3f651804ac19cd358702763c21 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 16 Apr 2021 14:19:22 +0200 Subject: [PATCH 63/68] disable impossible ftp test + switch back to ubuntu 18.04 --- .github/workflows/single.sh | 20 +++----------------- galaxy/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/.github/workflows/single.sh b/.github/workflows/single.sh index d0c5bd1b3..f992a886e 100755 --- a/.github/workflows/single.sh +++ b/.github/workflows/single.sh @@ -96,9 +96,10 @@ docker logs httpstest && docker stop httpstest && docker rm httpstest # Test FTP Server upload date > time.txt -curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true +# FIXME passive mode does not work, it would require the container to run with --net=host +#curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true # Test FTP Server get -curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD +#curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD # Test CVMFS docker_exec bash -c "service autofs start" @@ -135,18 +136,3 @@ docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/b docker stop galaxy docker rmi galaxy - - - - - - - - - - - - - - - diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index e2ef0d6ea..ffe96856e 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM ubuntu:20.04 +FROM ubuntu:18.04 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com From e113d39ef7e1e7d56e52206c3ad5fd6cf57dd43e Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 16 Apr 2021 14:50:49 +0200 Subject: [PATCH 64/68] not a tty in GHA --- .github/workflows/single.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/single.sh b/.github/workflows/single.sh index f992a886e..9abbf42b5 100755 --- a/.github/workflows/single.sh +++ b/.github/workflows/single.sh @@ -61,7 +61,7 @@ docker logs galaxy # Define start functions docker_exec() { cd "$WORKING_DIR" - docker exec -t -i galaxy "$@" + docker exec galaxy "$@" } docker_exec_run() { cd "$WORKING_DIR" From 4dc42d9f0f7b1b44163e3f5df5bf30cd8946fc04 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 16 Apr 2021 16:26:03 +0200 Subject: [PATCH 65/68] fix --- .github/workflows/single.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/single.sh b/.github/workflows/single.sh index 9abbf42b5..e47c8e527 100755 --- a/.github/workflows/single.sh +++ b/.github/workflows/single.sh @@ -135,4 +135,4 @@ docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/b docker stop galaxy -docker rmi galaxy +docker rmi $DOCKER_RUN_CONTAINER From 5c2751eacfb55e62b12c62c4948f8f0e21cbdfd1 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 16 Apr 2021 17:06:33 +0200 Subject: [PATCH 66/68] fix --- .github/workflows/single.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/single.sh b/.github/workflows/single.sh index e47c8e527..6e6f86a0e 100755 --- a/.github/workflows/single.sh +++ b/.github/workflows/single.sh @@ -135,4 +135,5 @@ docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/b docker stop galaxy +docker rm -f galaxy docker rmi $DOCKER_RUN_CONTAINER From 9dd2bd682eba7e439432754372973388e226fb32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Fri, 16 Apr 2021 18:42:07 +0200 Subject: [PATCH 67/68] Update single.sh --- .github/workflows/single.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/single.sh b/.github/workflows/single.sh index 6e6f86a0e..446d867ae 100755 --- a/.github/workflows/single.sh +++ b/.github/workflows/single.sh @@ -136,4 +136,4 @@ docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/b docker stop galaxy docker rm -f galaxy -docker rmi $DOCKER_RUN_CONTAINER +docker rmi -f $DOCKER_RUN_CONTAINER From 7d122a6bc220541840a7ba49d2697e0326d84bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Fri, 16 Apr 2021 23:48:21 +0200 Subject: [PATCH 68/68] exclude other k8s test --- .github/workflows/compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 475bbff93..c076434d3 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -164,6 +164,7 @@ jobs: files: -f docker-compose.yml -f docker-compose.k8s.yml exclude_test: - bioblend + - workflow_example1 - workflow_ard - workflow_mapping_by_sequencing - selenium