diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml
index 2dbf435f6..c076434d3 100644
--- a/.github/workflows/compose.yml
+++ b/.github/workflows/compose.yml
@@ -2,7 +2,7 @@ name: build-and-test
on: [push]
jobs:
build_container_base:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v2
@@ -120,7 +120,7 @@ jobs:
--build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \
--build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \
--build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \
- --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy \
+ --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy \
${{ matrix.image.subdir }}${{ matrix.image.name }} && break || echo "Fail.. Retrying"
done;
shell: bash
@@ -164,6 +164,7 @@ jobs:
files: -f docker-compose.yml -f docker-compose.k8s.yml
exclude_test:
- bioblend
+ - workflow_example1
- workflow_ard
- workflow_mapping_by_sequencing
- selenium
@@ -232,13 +233,13 @@ jobs:
- name: Checkout
uses: actions/checkout@v2
- name: Set image tag in env
- run: echo "::set-env name=IMAGE_TAG::${GITHUB_REF#refs/heads/}"
+ run: echo "IMAGE_TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
- name: Master branch - Set image to to 'latest'
if: github.ref == 'refs/heads/master'
- run: echo "::set-env name=IMAGE_TAG::latest"
+ run: echo "IMAGE_TAG=latest" >> $GITHUB_ENV
- name: Set WORKFLOWS env for worfklows-test
if: matrix.test.workflow
- run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}"
+ run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV
- name: Run tests for the first time
if: steps.run_check.outputs.run
run: |
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 63693b86d..36692201a 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -16,7 +16,7 @@ jobs:
level: warning
pattern: "*.sh"
- name: Run hadolint with reviewdog
- uses: reviewdog/action-hadolint@v1
+ uses: reviewdog/action-hadolint@v1.16.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
reporter: github-check
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 9eb289190..12baa530c 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -2,7 +2,7 @@ name: pr-test
on: pull_request
jobs:
test:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
strategy:
matrix:
infrastructure:
@@ -30,6 +30,7 @@ jobs:
exclude_test:
- workflow_example1
- workflow_mapping_by_sequencing
+ - workflow_ard
# - name: galaxy-pulsar-mq
# files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml
# exclude_test:
@@ -41,6 +42,7 @@ jobs:
- bioblend
- workflow_ard
- workflow_mapping_by_sequencing
+ - workflow_example1
- selenium
- name: galaxy-singularity
files: -f docker-compose.yml -f docker-compose.singularity.yml
@@ -108,7 +110,7 @@ jobs:
uses: actions/checkout@v2
- name: Set WORKFLOWS env for worfklows-test
if: matrix.test.workflow
- run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}"
+ run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV
- name: Build galaxy-container-base
env:
image_name: galaxy-container-base
@@ -142,7 +144,7 @@ jobs:
echo "Removing export directory if existent";
sudo rm -rf export
set +e
- docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy
+ docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy
docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }}
test_exit_code=$?
error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1)
diff --git a/.github/workflows/single.sh b/.github/workflows/single.sh
new file mode 100755
index 000000000..446d867ae
--- /dev/null
+++ b/.github/workflows/single.sh
@@ -0,0 +1,139 @@
+#!/bin/bash
+set -ex
+
+export GALAXY_HOME=/home/galaxy
+export GALAXY_USER=admin@galaxy.org
+export GALAXY_USER_EMAIL=admin@galaxy.org
+export GALAXY_USER_PASSWD=password
+export BIOBLEND_GALAXY_API_KEY=fakekey
+export BIOBLEND_GALAXY_URL=http://localhost:8080
+
+sudo apt-get update -qq
+#sudo apt-get install docker-ce --no-install-recommends -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew"
+sudo apt-get install sshpass --no-install-recommends -y
+
+pip3 install ephemeris
+
+docker --version
+docker info
+
+# start building this repo
+git submodule update --init --recursive
+sudo chown 1450 /tmp && sudo chmod a=rwx /tmp
+
+## define a container size check function, first parameter is the container name, second the max allowed size in MB
+container_size_check () {
+
+ # check that the image size is not growing too much between releases
+ # the 19.05 monolithic image was around 1.500 MB
+ size="${docker image inspect $1 --format='{{.Size}}'}"
+ size_in_mb=$(($size/(1024*1024)))
+ if [[ $size_in_mb -ge $2 ]]
+ then
+ echo "The new compiled image ($1) is larger than allowed. $size_in_mb vs. $2"
+ sleep 2
+ #exit
+ fi
+}
+
+export WORKING_DIR=${GITHUB_WORKSPACE:-$PWD}
+
+export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy"
+SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml
+cd "$WORKING_DIR"
+docker build -t quay.io/bgruening/galaxy galaxy/
+#container_size_check quay.io/bgruening/galaxy 1500
+
+mkdir local_folder
+docker run -d -p 8080:80 -p 8021:21 -p 8022:22 \
+ --name galaxy \
+ --privileged=true \
+ -v "$(pwd)/local_folder:/export/" \
+ -e GALAXY_CONFIG_ALLOW_USER_DATASET_PURGE=True \
+ -e GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE=True \
+ -e GALAXY_CONFIG_ENABLE_USER_DELETION=True \
+ -e GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True \
+ -v /tmp/:/tmp/ \
+ quay.io/bgruening/galaxy
+
+sleep 30
+docker logs galaxy
+# Define start functions
+docker_exec() {
+ cd "$WORKING_DIR"
+ docker exec galaxy "$@"
+}
+docker_exec_run() {
+ cd "$WORKING_DIR"
+ docker run quay.io/bgruening/galaxy "$@"
+}
+docker_run() {
+ cd "$WORKING_DIR"
+ docker run "$@"
+}
+
+docker ps
+
+# Test submitting jobs to an external slurm cluster
+cd "${WORKING_DIR}/test/slurm/" && bash test.sh && cd "$WORKING_DIR"
+
+# Test submitting jobs to an external gridengine cluster
+# TODO 19.05, need to enable this again!
+# - cd $WORKING_DIR/test/gridengine/ && bash test.sh && cd $WORKING_DIR
+
+echo 'Waiting for Galaxy to come up.'
+galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 300
+
+curl -v --fail $BIOBLEND_GALAXY_URL/api/version
+
+# Test self-signed HTTPS
+docker_run -d --name httpstest -p 443:443 -e "USE_HTTPS=True" $DOCKER_RUN_CONTAINER
+# TODO 19.05
+# - sleep 90s && curl -v -k --fail https://127.0.0.1:443/api/version
+#- echo | openssl s_client -connect 127.0.0.1:443 2>/dev/null | openssl x509 -issuer -noout| grep selfsigned
+
+docker logs httpstest && docker stop httpstest && docker rm httpstest
+
+# Test FTP Server upload
+date > time.txt
+# FIXME passive mode does not work, it would require the container to run with --net=host
+#curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true
+# Test FTP Server get
+#curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD
+
+# Test CVMFS
+docker_exec bash -c "service autofs start"
+docker_exec bash -c "cvmfs_config chksetup"
+docker_exec bash -c "ls /cvmfs/data.galaxyproject.org/byhand"
+
+# Test SFTP Server
+sshpass -p $GALAXY_USER_PASSWD sftp -v -P 8022 -o User=$GALAXY_USER -o "StrictHostKeyChecking no" localhost <<< $'put time.txt'
+
+# Run a ton of BioBlend test against our servers.
+cd "$WORKING_DIR/test/bioblend/" && . ./test.sh && cd "$WORKING_DIR/"
+
+# not working anymore in 18.01
+# executing: /galaxy_venv/bin/uwsgi --yaml /etc/galaxy/galaxy.yml --master --daemonize2 galaxy.log --pidfile2 galaxy.pid --log-file=galaxy_install.log --pid-file=galaxy_install.pid
+# [uWSGI] getting YAML configuration from /etc/galaxy/galaxy.yml
+# /galaxy_venv/bin/python: unrecognized option '--log-file=galaxy_install.log'
+# getopt_long() error
+# cat: galaxy_install.pid: No such file or directory
+# tail: cannot open ‘galaxy_install.log’ for reading: No such file or directory
+#- |
+# if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ]
+# then
+# # Test without install-repository wrapper
+# sleep 10
+# docker_exec_run bash -c 'cd $GALAXY_ROOT && python ./scripts/api/install_tool_shed_repositories.py --api admin -l http://localhost:80 --url https://toolshed.g2.bx.psu.edu -o devteam --name cut_columns --panel-section-name BEDTools'
+# fi
+
+
+# Test the 'new' tool installation script
+docker_exec install-tools "$SAMPLE_TOOLS"
+# Test the Conda installation
+docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/bin/:$PATH && conda --version && conda install samtools -c bioconda --yes'
+
+
+docker stop galaxy
+docker rm -f galaxy
+docker rmi -f $DOCKER_RUN_CONTAINER
diff --git a/.github/workflows/single_container.yml b/.github/workflows/single_container.yml
new file mode 100644
index 000000000..31223bc18
--- /dev/null
+++ b/.github/workflows/single_container.yml
@@ -0,0 +1,16 @@
+name: Single Container Test
+on: [push]
+jobs:
+ build_and_test:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: [3.7]
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v2
+ - uses: actions/setup-python@v1
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Build and Test
+ run: bash .github/workflows/single.sh
diff --git a/.travis.yml b/.travis.yml
index 694321b4c..4353fad27 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,6 +40,11 @@ before_install:
# start building this repo
- git submodule update --init --recursive
- sudo chown 1450 /tmp && sudo chmod a=rwx /tmp
+ - export WORKING_DIR="$TRAVIS_BUILD_DIR"
+ - export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy"
+ - export INSTALL_REPO_ARG=""
+ - export SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml
+ - travis_wait 30 cd "$WORKING_DIR" && docker build -t quay.io/bgruening/galaxy galaxy/
- |
## define a container size check function, first parameter is the container name, second the max allowed size in MB
container_size_check () {
@@ -55,14 +60,6 @@ before_install:
#exit
fi
}
-
- export WORKING_DIR="$TRAVIS_BUILD_DIR"
- export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy"
- INSTALL_REPO_ARG=""
- SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml
- cd "$WORKING_DIR"
- docker build -t quay.io/bgruening/galaxy galaxy/
-
container_size_check quay.io/bgruening/galaxy 1500
mkdir local_folder
diff --git a/README.md b/README.md
index a64c50650..6e23c7b71 100644
--- a/README.md
+++ b/README.md
@@ -993,6 +993,8 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro
- Featuring Galaxy 20.05
- Completely reworked compose setup
- The default admin password and apikey (`GALAXY_DEFAULT_ADMIN_PASSWORD` and `GALAXY_DEFAULT_ADMIN_KEY`) have changed: the password is now `password` (instead of `admin`) and the apikey `fakekey` (instead of `admin`).
+- 20.09:
+ - Featuring Galaxy 20.09
# Support & Bug Reports [[toc]](#toc)
diff --git a/compose/README.md b/compose/README.md
index 3007747d0..d73d39a5d 100644
--- a/compose/README.md
+++ b/compose/README.md
@@ -151,9 +151,9 @@ All options are discussed under [configuration reference](#configuration-referen
### Use specific Galaxy version or Docker images
The `IMAGE_TAG` environment variable allows to use specific versions of the
-setup. Say, you want to stay with Galaxy v20.05 for now:
+setup. Say, you want to stay with Galaxy v20.09 for now:
-> export IMAGE_TAG=20.05
+> export IMAGE_TAG=20.09
> docker-compose up
Without setting this variable, you will always get updated to the newest
@@ -378,7 +378,7 @@ The following are settings specific to this docker-compose setup:
| `GALAXY_K8S_DOCKER_REPO_DEFAULT` | The Docker Repo/Registry to use if the resolver could not resolve the proper image for a job. Defaults to `docker.io`. |
| `GALAXY_K8S_DOCKER_OWNER_DEFAULT` | The Owner/Username to use if the resolver could not resolve the proper image for a job. Is not set by default. |
| `GALAXY_K8S_DOCKER_IMAGE_DEFAULT` | The Image to use if the resolver could not resolve the proper image for a job. Defaults to `ubuntu`. |
-| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `18.04`. |
+| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `20.04`. |
### HTCondor
| Variable | Description |
@@ -393,3 +393,21 @@ The following are settings specific to this docker-compose setup:
| `SLURM_NODE_CPUS` | Number of CPUs per node. Defaults to 1. |
| `SLURM_NODE_MEMORY` | Amount of memory per node. Defaults to 1024. |
| `SLURM_NODE_HOSTNAME` | Docker Compose adds a prefix in front of the container names by default. Change this value to the name of your setup and `_slurm_node` (e.g. `compose_slurm_node`) to ensure a correct mapping of the Slurm nodes. |
+
+### Github Workflow Tests (Branch 20.09)
+| Setup | bioblend | workflow ard | workflow mapping_by_sequencing | workflow wf3-shed-tools (example1) | selenium |
+|------------------------|--------------------|--------------------|--------------------------------|------------------------------------|--------------------|
+| Galaxy Base | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: |
+| Galaxy Proxy Prefix | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: |
+| HTCondor | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: |
+| Slurm | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: |
+| Pulsar | :heavy_check_mark: | :x: | :x: | :x: | :heavy_check_mark: |
+| k8s | :x: | :x: | :x: | :x: | :x: |
+| Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: |
+| Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: |
+| HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: |
+
+
+Implemented: :heavy_check_mark:
+Not Implemented: :x:
+
diff --git a/compose/base-images/galaxy-cluster-base/Dockerfile b/compose/base-images/galaxy-cluster-base/Dockerfile
index f9d668009..ff7c91cce 100644
--- a/compose/base-images/galaxy-cluster-base/Dockerfile
+++ b/compose/base-images/galaxy-cluster-base/Dockerfile
@@ -20,11 +20,11 @@ ENV MUNGER_USER=munge \
MUNGE_GID=1200
RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \
&& useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \
- && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \
- && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \
+ && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \
+ && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \
&& apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8DE68488997C5C6BA19021136F2CC56412788738 \
&& apt update \
- && apt install --no-install-recommends slurm-client slurmd slurmctld slurm-drmaa1 -y \
+ && apt install --no-install-recommends python3-distutils slurm-client slurmd slurmctld slurm-drmaa1 -y \
&& apt --no-install-recommends install munge libmunge-dev -y \
&& ln -s /usr/lib/slurm-drmaa/lib/libdrmaa.so.1 /usr/lib/slurm-drmaa/lib/libdrmaa.so \
&& /usr/bin/common_cleanup.sh
diff --git a/compose/base-images/galaxy-container-base/Dockerfile b/compose/base-images/galaxy-container-base/Dockerfile
index 20fa3743f..6cf33a516 100644
--- a/compose/base-images/galaxy-container-base/Dockerfile
+++ b/compose/base-images/galaxy-container-base/Dockerfile
@@ -1,4 +1,4 @@
-FROM buildpack-deps:18.04 as build_singularity
+FROM buildpack-deps:20.04 as build_singularity
COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
@@ -21,12 +21,12 @@ RUN wget https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_
# --- Final image ---
-FROM ubuntu:18.04 as final
+FROM ubuntu:20.04 as final
COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
# Base dependencies
-RUN apt update && apt install --no-install-recommends ca-certificates squashfs-tools -y \
+RUN apt update && apt install --no-install-recommends ca-certificates python3-distutils squashfs-tools -y \
&& /usr/bin/common_cleanup.sh
# Install Docker
diff --git a/compose/base_config.yml b/compose/base_config.yml
index cc2970901..46f37dc7c 100644
--- a/compose/base_config.yml
+++ b/compose/base_config.yml
@@ -9,11 +9,10 @@ galaxy_uwsgi:
master: false
virtualenv: .venv
pythonpath: lib
- thunder-lock: false
+ thunder-lock: true
die-on-term: true
hook-master-start: unix_signal:2 gracefully_kill_them_all
hook-master-start: unix_signal:15 gracefully_kill_them_all
- py-call-osafterfork: true
enable-threads: true
galaxy:
diff --git a/compose/docker-compose.htcondor.yml b/compose/docker-compose.htcondor.yml
index 1b7c36702..35084397f 100644
--- a/compose/docker-compose.htcondor.yml
+++ b/compose/docker-compose.htcondor.yml
@@ -28,6 +28,7 @@ services:
volumes:
- ${EXPORT_DIR:-./export}/htcondor:/config
- ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database
+ - ${EXPORT_DIR:-./export}/galaxy/lib/galaxy/tools:/galaxy/lib/galaxy/tools:ro
- ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro
- ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data
- ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv
diff --git a/compose/docker-compose.slurm.yml b/compose/docker-compose.slurm.yml
index 134f5e06d..4cf6e29cb 100644
--- a/compose/docker-compose.slurm.yml
+++ b/compose/docker-compose.slurm.yml
@@ -40,6 +40,7 @@ services:
volumes:
- ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database
- ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro
+ - ${EXPORT_DIR:-./export}/galaxy/lib/galaxy/tools:/galaxy/lib/galaxy/tools:ro
- ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data
- ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv
- ${EXPORT_DIR:-./export}/tool_deps:/tool_deps
diff --git a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2
index 461dfb5ff..9b1ca49ed 100644
--- a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2
+++ b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2
@@ -32,6 +32,7 @@
{% if GALAXY_DEPENDENCY_RESOLUTION == 'singularity' -%}
+
/home/galaxy
true
{% if GALAXY_JOB_RUNNER == 'local' -%}
@@ -59,7 +60,7 @@
{{ GALAXY_K8S_DOCKER_REPO_DEFAULT | default('docker.io') }}
{% if GALAXY_K8S_DOCKER_OWNER_DEFAULT -%}{{ GALAXY_K8S_DOCKER_OWNER_DEFAULT }}{% endif -%}
{{ GALAXY_K8S_DOCKER_IMAGE_DEFAULT | default('ubuntu') }}
- {{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('18.04') }}
+ {{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('20.04') }}
true
{% endif -%}
diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile
index d10630bde..ac49a826b 100644
--- a/compose/galaxy-htcondor/Dockerfile
+++ b/compose/galaxy-htcondor/Dockerfile
@@ -2,9 +2,9 @@ ARG DOCKER_REGISTRY=quay.io
ARG DOCKER_REGISTRY_USERNAME=bgruening
ARG IMAGE_TAG=latest
-FROM buildpack-deps:18.04 as galaxy_dependencies
+FROM buildpack-deps:20.04 as galaxy_dependencies
-ARG GALAXY_RELEASE=release_20.05
+ARG GALAXY_RELEASE=release_20.09
ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy
ENV GALAXY_ROOT=/galaxy
@@ -18,7 +18,7 @@ RUN mkdir "${GALAXY_ROOT}" \
&& cd $GALAXY_ROOT/lib \
&& ls . | grep -v "galaxy\|galaxy_ext" | xargs rm -rf \
&& cd $GALAXY_ROOT/lib/galaxy \
- && ls . | grep -v "__init__.py\|datatypes\|exceptions\|metadata\|model\|util\|security" | xargs rm -rf
+ && ls . | grep -v "__init__.py\|datatypes\|exceptions\|files\|metadata\|model\|util\|security" | xargs rm -rf
FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-container-base:$IMAGE_TAG as final
@@ -67,4 +67,6 @@ COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT $
COPY start.sh /usr/bin/start.sh
RUN apt update && apt install python3 -y
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9
+
ENTRYPOINT /usr/bin/start.sh
diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile
index 2245216ec..d789c2250 100644
--- a/compose/galaxy-server/Dockerfile
+++ b/compose/galaxy-server/Dockerfile
@@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=quay.io
ARG DOCKER_REGISTRY_USERNAME=bgruening
ARG IMAGE_TAG=latest
-FROM buildpack-deps:18.04 as build_base
+FROM buildpack-deps:20.04 as build_base
ENV EXPORT_DIR=/export \
GALAXY_ROOT=/galaxy \
@@ -24,7 +24,7 @@ ENV GALAXY_USER=galaxy \
GALAXY_HOME=/home/galaxy
ENV GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda \
- MINICONDA_VERSION=4.7.10
+ MINICONDA_VERSION=py38_4.9.2
RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \
&& useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" --shell /bin/bash $GALAXY_USER \
@@ -34,7 +34,7 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \
FROM build_base as build_miniconda
COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
# Install Miniconda
-RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA_VERSION}-Linux-x86_64.sh" > ~/miniconda.sh \
+RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" > ~/miniconda.sh \
&& /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX \
&& rm ~/miniconda.sh \
&& ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \
@@ -50,18 +50,20 @@ RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA_VERSI
FROM build_base as build_galaxy
-ARG GALAXY_RELEASE=release_20.05
+ARG GALAXY_RELEASE=release_20.09
ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy
COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
# Install Galaxy
RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libssl-dev python3-dev python3-pip -y \
+ && update-alternatives --install /usr/bin/python python /usr/bin/python3 9 \
&& mkdir "${GALAXY_ROOT}" \
&& curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \
&& cd $GALAXY_ROOT \
&& ./scripts/common_startup.sh \
&& . $GALAXY_ROOT/.venv/bin/activate \
&& pip3 install drmaa psycopg2 pycurl pykube \
+ && pip3 install importlib-metadata importlib-resources pathlib2 ruamel.yaml.clib typing zipp \
&& deactivate \
&& rm -rf .ci .circleci .coveragerc .gitignore .travis.yml CITATION CODE_OF_CONDUCT.md CONTRIBUTING.md CONTRIBUTORS.md \
LICENSE.txt Makefile README.rst SECURITY_POLICY.md pytest.ini tox.ini \
@@ -73,9 +75,6 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss
# --- Final image ---
FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-cluster-base:$IMAGE_TAG as final
-ARG GALAXY_RELEASE=release_20.05
-ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy
-
COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
COPY ./files/create_galaxy_user.py /usr/local/bin/create_galaxy_user.py
@@ -84,7 +83,9 @@ ENV EXPORT_DIR=/export \
GALAXY_PYTHON=/usr/bin/python3 \
HTCONDOR_ROOT=/opt/htcondor
-ENV GALAXY_STATIC_DIR=$GALAXY_ROOT/static \
+ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \
+ GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \
+ GALAXY_STATIC_DIR=$GALAXY_ROOT/static \
GALAXY_EXPORT=$EXPORT_DIR/galaxy \
GALAXY_CONFIG_DIR=$GALAXY_ROOT/config \
GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=/tool_deps \
@@ -120,8 +121,7 @@ RUN apt update && apt install --no-install-recommends curl gcc gnupg2 libgomp1 l
RUN usermod -aG docker $GALAXY_USER
# Make Python3 standard
-RUN mv /usr/bin/python /usr/bin/python2 \
- && ln /usr/bin/python3 /usr/bin/python
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9
COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_galaxy ${GALAXY_ROOT} ${GALAXY_ROOT}
COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR}
diff --git a/compose/galaxy-server/files/start.sh b/compose/galaxy-server/files/start.sh
index baf672165..860280856 100755
--- a/compose/galaxy-server/files/start.sh
+++ b/compose/galaxy-server/files/start.sh
@@ -13,6 +13,14 @@ create_user() {
deactivate
}
+# start copy lib/tools. Looks very hacky.
+tools_dir="/galaxy/lib/galaxy/tools/"
+exp_dir="/export$tools_dir"
+mkdir -p $exp_dir
+chown "$GALAXY_USER:$GALAXY_USER" $exp_dir
+cp -rf $tools_dir/* $exp_dir
+# end copy lib/tools.
+
# First start?? Check if something exists that indicates that environment is not new.. Config file? Something in DB maybe??
echo "Initialization: Check if files already exist, export otherwise."
diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile
index 2f1328e8c..6f54338b2 100644
--- a/compose/galaxy-slurm/Dockerfile
+++ b/compose/galaxy-slurm/Dockerfile
@@ -2,9 +2,9 @@ ARG DOCKER_REGISTRY=quay.io
ARG DOCKER_REGISTRY_USERNAME=bgruening
ARG IMAGE_TAG=latest
-FROM buildpack-deps:18.04 as galaxy_dependencies
+FROM buildpack-deps:20.04 as galaxy_dependencies
-ARG GALAXY_RELEASE=release_20.05
+ARG GALAXY_RELEASE=release_20.09
ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy
ENV GALAXY_ROOT=/galaxy
@@ -17,7 +17,7 @@ RUN mkdir "${GALAXY_ROOT}" \
&& cd $GALAXY_ROOT/lib \
&& ls . | grep -v "galaxy\|galaxy_ext" | xargs rm -rf \
&& cd $GALAXY_ROOT/lib/galaxy \
- && ls . | grep -v "__init__.py\|datatypes\|exceptions\|metadata\|model\|util\|security" | xargs rm -rf
+ && ls . | grep -v "__init__.py\|datatypes\|exceptions\|files\|metadata\|model\|util\|security" | xargs rm -rf
FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-container-base:$IMAGE_TAG as final
@@ -45,11 +45,13 @@ ENV SLURM_USER=galaxy \
RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \
&& useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \
&& apt update \
- && apt install --no-install-recommends gosu munge python python-dev slurm-wlm -y \
+ && apt install --no-install-recommends gosu munge python3 python3-dev slurm-wlm -y \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/* && find / -name '*.pyc' -delete
# Copy Galaxy dependencies
COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT $GALAXY_ROOT
+# Make Python3 standard
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9
COPY start.sh /usr/bin/start.sh
diff --git a/compose/pulsar/Dockerfile b/compose/pulsar/Dockerfile
index 2fb2a7d10..dac9b1503 100644
--- a/compose/pulsar/Dockerfile
+++ b/compose/pulsar/Dockerfile
@@ -2,9 +2,9 @@ ARG DOCKER_REGISTRY=quay.io
ARG DOCKER_REGISTRY_USERNAME=bgruening
ARG IMAGE_TAG=latest
-FROM buildpack-deps:18.04 as build_pulsar
+FROM buildpack-deps:20.04 as build_pulsar
-ARG PULSAR_RELEASE=0.13.0
+ARG PULSAR_RELEASE=0.14.0
ARG PULSAR_REPO=https://github.com/galaxyproject/pulsar
ENV PULSAR_ROOT=/pulsar
@@ -34,7 +34,7 @@ ENV PULSAR_VIRTUALENV=$PULSAR_ROOT/.venv \
PULSAR_TOOL_DEPENDENCY_DIR=$PULSAR_ROOT/dependencies
RUN apt update \
- && apt install --no-install-recommends ca-certificates curl libxml2-dev libpython3.6 python3 -y \
+ && apt install --no-install-recommends ca-certificates curl libxml2-dev python3 -y \
&& /usr/bin/common_cleanup.sh
COPY --from=build_pulsar /pulsar /pulsar
diff --git a/compose/tests/docker-compose.test.bioblend.yml b/compose/tests/docker-compose.test.bioblend.yml
index 9ebcaf7b8..6b77fb405 100644
--- a/compose/tests/docker-compose.test.bioblend.yml
+++ b/compose/tests/docker-compose.test.bioblend.yml
@@ -4,7 +4,7 @@ services:
image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-bioblend-test:${IMAGE_TAG:-latest}
build: tests/galaxy-bioblend-test
environment:
- - GALAXY_VERSION=${GALAXY_VERSION:-release_20.05} # TODO: Change to GALAXY_RELEASE
+ - GALAXY_VERSION=${GALAXY_VERSION:-release_20.09} # TODO: Change to GALAXY_RELEASE
- GALAXY_URL=http://nginx${GALAXY_PROXY_PREFIX:-}/
networks:
- galaxy
diff --git a/compose/tests/galaxy-selenium-test/Dockerfile b/compose/tests/galaxy-selenium-test/Dockerfile
index f081a029e..370264852 100644
--- a/compose/tests/galaxy-selenium-test/Dockerfile
+++ b/compose/tests/galaxy-selenium-test/Dockerfile
@@ -1,18 +1,19 @@
FROM selenium/standalone-chrome:3.141.59
-ARG GALAXY_RELEASE=release_20.05
+ARG GALAXY_RELEASE=release_20.09
ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy
ENV GALAXY_ROOT=/galaxy
-ENV GALAXY_PYTHON=/usr/bin/python
+ENV GALAXY_PYTHON=/usr/bin/python3
USER root
-RUN apt update && apt install --no-install-recommends python-dev python-pip -y && rm -rf /var/lib/apt/lists/* \
+RUN apt update && apt install --no-install-recommends python3-dev python3-pip -y && rm -rf /var/lib/apt/lists/* \
&& mkdir "${GALAXY_ROOT}" \
&& chown seluser "${GALAXY_ROOT}"
USER seluser
-RUN curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \
+RUN mkdir -p $GALAXY_ROOT && \
+ curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \
&& cd "${GALAXY_ROOT}" \
&& ./scripts/common_startup.sh --skip-client-build --dev-wheels
diff --git a/compose/tests/galaxy-workflow-test/Dockerfile b/compose/tests/galaxy-workflow-test/Dockerfile
index b392babbb..1b9385da9 100644
--- a/compose/tests/galaxy-workflow-test/Dockerfile
+++ b/compose/tests/galaxy-workflow-test/Dockerfile
@@ -1,7 +1,7 @@
FROM alpine:3.11
-ENV TEST_REPO=${TEST_REPO:-https://github.com/usegalaxy-eu/workflow-testing} \
- TEST_RELEASE=${TEST_RELEASE:-master}
+ENV TEST_REPO=${TEST_REPO:-https://github.com/jonas27/workflow-testing} \
+ TEST_RELEASE=${TEST_RELEASE:-20.09-comment-filetype}
RUN apk add --no-cache bash python3 curl \
&& apk add --no-cache --virtual build-dep gcc libxml2-dev libxslt-dev musl-dev linux-headers python3-dev \
@@ -10,6 +10,9 @@ RUN apk add --no-cache bash python3 curl \
&& curl -L -s $TEST_REPO/archive/$TEST_RELEASE.tar.gz | tar xzf - --strip-components=1 \
&& apk del build-dep
+# Make Python3 standard
+RUN ln /usr/bin/python3 /usr/bin/python && ln /usr/bin/python3 /usr/bin/python2
+
ADD ./run.sh /usr/bin/run.sh
WORKDIR /src
diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile
index 2b46ec66c..ffe96856e 100644
--- a/galaxy/Dockerfile
+++ b/galaxy/Dockerfile
@@ -16,60 +16,60 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com
ARG GALAXY_RELEASE
ARG GALAXY_REPO
-ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.05} \
-GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \
-GALAXY_ROOT=/galaxy-central \
-GALAXY_CONFIG_DIR=/etc/galaxy \
-EXPORT_DIR=/export \
-DEBIAN_FRONTEND=noninteractive \
-PG_VERSION=11
-
+ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \
+ GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \
+ GALAXY_ROOT=/galaxy-central \
+ GALAXY_CONFIG_DIR=/etc/galaxy \
+ EXPORT_DIR=/export \
+ DEBIAN_FRONTEND=noninteractive \
+ PG_VERSION=11
ENV GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml \
-GALAXY_CONFIG_JOB_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_conf.xml \
-GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_metrics_conf.xml \
-GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml \
-GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \
-GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=$EXPORT_DIR/tool_deps \
-GALAXY_CONFIG_TOOL_PATH=$EXPORT_DIR/galaxy-central/tools \
-GALAXY_VIRTUAL_ENV=/galaxy_venv \
-GALAXY_USER=galaxy \
-GALAXY_UID=1450 \
-GALAXY_GID=1450 \
-GALAXY_POSTGRES_UID=1550 \
-GALAXY_POSTGRES_GID=1550 \
-GALAXY_HOME=/home/galaxy \
-GALAXY_LOGS_DIR=/home/galaxy/logs \
-GALAXY_DEFAULT_ADMIN_USER=admin \
-GALAXY_DEFAULT_ADMIN_EMAIL=admin@galaxy.org \
-GALAXY_DEFAULT_ADMIN_PASSWORD=password \
-GALAXY_DEFAULT_ADMIN_KEY=fakekey \
-GALAXY_DESTINATIONS_DEFAULT=slurm_cluster \
-GALAXY_RUNNERS_ENABLE_SLURM=True \
-GALAXY_RUNNERS_ENABLE_CONDOR=False \
-GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:galaxy@localhost:5432/galaxy?client_encoding=utf8 \
-GALAXY_CONFIG_ADMIN_USERS=admin@galaxy.org \
-GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3 \
-GALAXY_CONFIG_BRAND="Galaxy Docker Build" \
-GALAXY_CONFIG_STATIC_ENABLED=False \
-# Define the default postgresql database path
-PG_DATA_DIR_DEFAULT=/var/lib/postgresql/$PG_VERSION/main/ \
-PG_CONF_DIR_DEFAULT=/etc/postgresql/$PG_VERSION/main/ \
-PG_DATA_DIR_HOST=$EXPORT_DIR/postgresql/$PG_VERSION/main/ \
-# The following 2 ENV vars can be used to set the number of uwsgi processes and threads
-UWSGI_PROCESSES=2 \
-UWSGI_THREADS=4 \
-# Set HTTPS to use a self-signed certificate (or your own certificate in $EXPORT_DIR/{server.key,server.crt})
-USE_HTTPS=False \
-# Set USE_HTTPS_LENSENCRYPT and GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL to a domain that is reachable to get a letsencrypt certificate
-USE_HTTPS_LETSENCRYPT=False \
-GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost \
-# Set the number of Galaxy handlers
-GALAXY_HANDLER_NUMPROCS=2 \
-# Setting a standard encoding. This can get important for things like the unix sort tool.
-LC_ALL=en_US.UTF-8 \
-LANG=en_US.UTF-8 \
-GALAXY_CONDA_PREFIX=/tool_deps/_conda
+ GALAXY_CONFIG_JOB_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_conf.xml \
+ GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_metrics_conf.xml \
+ GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml \
+ GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \
+ GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=$EXPORT_DIR/tool_deps \
+ GALAXY_CONFIG_TOOL_PATH=$EXPORT_DIR/galaxy-central/tools \
+ GALAXY_VIRTUAL_ENV=/galaxy_venv \
+ GALAXY_USER=galaxy \
+ GALAXY_UID=1450 \
+ GALAXY_GID=1450 \
+ GALAXY_POSTGRES_UID=1550 \
+ GALAXY_POSTGRES_GID=1550 \
+ GALAXY_HOME=/home/galaxy \
+ GALAXY_LOGS_DIR=/home/galaxy/logs \
+ GALAXY_DEFAULT_ADMIN_USER=admin \
+ GALAXY_DEFAULT_ADMIN_EMAIL=admin@galaxy.org \
+ GALAXY_DEFAULT_ADMIN_PASSWORD=password \
+ GALAXY_DEFAULT_ADMIN_KEY=fakekey \
+ GALAXY_DESTINATIONS_DEFAULT=slurm_cluster \
+ GALAXY_RUNNERS_ENABLE_SLURM=True \
+ GALAXY_RUNNERS_ENABLE_CONDOR=False \
+ GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:galaxy@localhost:5432/galaxy?client_encoding=utf8 \
+ GALAXY_CONFIG_ADMIN_USERS=admin@galaxy.org \
+ GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3 \
+ GALAXY_CONFIG_BRAND="Galaxy Docker Build" \
+ GALAXY_CONFIG_STATIC_ENABLED=False \
+ # Define the default postgresql database path
+ PG_DATA_DIR_DEFAULT=/var/lib/postgresql/$PG_VERSION/main/ \
+ PG_CONF_DIR_DEFAULT=/etc/postgresql/$PG_VERSION/main/ \
+ PG_DATA_DIR_HOST=$EXPORT_DIR/postgresql/$PG_VERSION/main/ \
+ # The following 2 ENV vars can be used to set the number of uwsgi processes and threads
+ UWSGI_PROCESSES=2 \
+ UWSGI_THREADS=4 \
+ # Set HTTPS to use a self-signed certificate (or your own certificate in $EXPORT_DIR/{server.key,server.crt})
+ USE_HTTPS=False \
+ # Set USE_HTTPS_LENSENCRYPT and GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL to a domain that is reachable to get a letsencrypt certificate
+ USE_HTTPS_LETSENCRYPT=False \
+ GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost \
+ # Set the number of Galaxy handlers
+ GALAXY_HANDLER_NUMPROCS=2 \
+ # Setting a standard encoding. This can get important for things like the unix sort tool.
+ LC_ALL=en_US.UTF-8 \
+ LANG=en_US.UTF-8 \
+ NODE_OPTIONS=--max-old-space-size=4096 \
+ GALAXY_CONDA_PREFIX=/tool_deps/_conda
# 16MB
RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup \
@@ -118,7 +118,7 @@ ADD ./bashrc $GALAXY_HOME/.bashrc
# Install miniconda, then virtualenv from conda and then
# download latest stable release of Galaxy.
-RUN curl -s -L https://repo.continuum.io/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh > ~/miniconda.sh \
+RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh > ~/miniconda.sh \
&& /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX/ \
&& rm ~/miniconda.sh \
&& ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \
@@ -188,30 +188,35 @@ RUN mkdir -p /shed_tools $EXPORT_DIR/ftp/ \
&& chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps \
&& apt update -qq && apt install --no-install-recommends -y ansible \
&& ansible-playbook /ansible/provision.yml \
- --extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \
- --extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \
- --extra-vars galaxy_user_name=$GALAXY_USER \
- --extra-vars galaxy_config_file=$GALAXY_CONFIG_FILE \
- --extra-vars galaxy_config_dir=$GALAXY_CONFIG_DIR \
- --extra-vars galaxy_job_conf_path=$GALAXY_CONFIG_JOB_CONFIG_FILE \
- --extra-vars galaxy_job_metrics_conf_path=$GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE \
- --extra-vars supervisor_manage_slurm="" \
- --extra-vars galaxy_extras_config_condor=True \
- --extra-vars galaxy_extras_config_condor_docker=True \
- --extra-vars galaxy_extras_config_rabbitmq=True \
- --extra-vars galaxy_extras_config_cvmfs=True \
- --extra-vars galaxy_extras_config_uwsgi=False \
- --extra-vars proftpd_db_connection=galaxy@galaxy \
- --extra-vars proftpd_files_dir=$EXPORT_DIR/ftp \
- --extra-vars proftpd_use_sftp=True \
- --extra-vars galaxy_extras_docker_legacy=False \
- --extra-vars galaxy_minimum_version=19.01 \
- --extra-vars supervisor_postgres_config_path=$PG_CONF_DIR_DEFAULT/postgresql.conf \
- --extra-vars supervisor_postgres_autostart=false \
- --extra-vars nginx_use_remote_header=True \
- --tags=galaxyextras,cvmfs -c local \
+ --extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \
+ --extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \
+ --extra-vars galaxy_user_name=$GALAXY_USER \
+ --extra-vars galaxy_config_file=$GALAXY_CONFIG_FILE \
+ --extra-vars galaxy_config_dir=$GALAXY_CONFIG_DIR \
+ --extra-vars galaxy_job_conf_path=$GALAXY_CONFIG_JOB_CONFIG_FILE \
+ --extra-vars galaxy_job_metrics_conf_path=$GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE \
+ --extra-vars supervisor_manage_slurm="" \
+ --extra-vars galaxy_extras_config_condor=True \
+ --extra-vars galaxy_extras_config_condor_docker=True \
+ --extra-vars galaxy_extras_config_rabbitmq=True \
+ --extra-vars galaxy_extras_config_cvmfs=True \
+ --extra-vars galaxy_extras_config_uwsgi=False \
+ --extra-vars proftpd_db_connection=galaxy@galaxy \
+ --extra-vars proftpd_files_dir=$EXPORT_DIR/ftp \
+ --extra-vars proftpd_use_sftp=True \
+ --extra-vars galaxy_extras_docker_legacy=False \
+ --extra-vars galaxy_minimum_version=19.01 \
+ --extra-vars supervisor_postgres_config_path=$PG_CONF_DIR_DEFAULT/postgresql.conf \
+ --extra-vars supervisor_postgres_autostart=false \
+ --extra-vars nginx_use_remote_header=True \
+ --tags=galaxyextras,cvmfs -c local \
+ && . $GALAXY_VIRTUAL_ENV/bin/activate \
+ && pip install WeasyPrint \
+ && deactivate \
# TODO: no clue why this is needed here again
&& cd $GALAXY_ROOT && ./scripts/common_startup.sh \
+ && cd config && find . -name 'node_modules' -type d -prune -exec rm -rf '{}' + \
+ && find . -name '.cache' -type d -prune -exec rm -rf '{}' + \
&& cd / \
&& rm $PG_DATA_DIR_DEFAULT -rf \
&& python /usr/local/bin/setup_postgresql.py --dbuser galaxy --dbpassword galaxy --db-name galaxy --dbpath $PG_DATA_DIR_DEFAULT --dbversion $PG_VERSION \
@@ -238,28 +243,28 @@ WORKDIR $GALAXY_ROOT
#RUN export GALAXY=$GALAXY_ROOT && sh ./cron/updateucsc.sh.sample
ENV GALAXY_CONFIG_JOB_WORKING_DIRECTORY=$EXPORT_DIR/galaxy-central/database/job_working_directory \
-GALAXY_CONFIG_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \
-GALAXY_CONFIG_NEW_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \
-GALAXY_CONFIG_TEMPLATE_CACHE_PATH=$EXPORT_DIR/galaxy-central/database/compiled_templates \
-GALAXY_CONFIG_CITATION_CACHE_DATA_DIR=$EXPORT_DIR/galaxy-central/database/citations/data \
-GALAXY_CONFIG_CLUSTER_FILES_DIRECTORY=$EXPORT_DIR/galaxy-central/database/pbs \
-GALAXY_CONFIG_FTP_UPLOAD_DIR=$EXPORT_DIR/ftp \
-GALAXY_CONFIG_FTP_UPLOAD_SITE=galaxy.docker.org \
-GALAXY_CONFIG_USE_PBKDF2=False \
-GALAXY_CONFIG_NGINX_X_ACCEL_REDIRECT_BASE=/_x_accel_redirect \
-GALAXY_CONFIG_NGINX_X_ARCHIVE_FILES_BASE=/_x_accel_redirect \
-GALAXY_CONFIG_DYNAMIC_PROXY_MANAGE=False \
-GALAXY_CONFIG_VISUALIZATION_PLUGINS_DIRECTORY=config/plugins/visualizations \
-GALAXY_CONFIG_TRUST_IPYTHON_NOTEBOOK_CONVERSION=True \
-GALAXY_CONFIG_TOOLFORM_UPGRADE=True \
-GALAXY_CONFIG_SANITIZE_ALL_HTML=False \
-GALAXY_CONFIG_TOOLFORM_UPGRADE=True \
-GALAXY_CONFIG_WELCOME_URL=$GALAXY_CONFIG_DIR/web/welcome.html \
-GALAXY_CONFIG_OVERRIDE_DEBUG=False \
-GALAXY_CONFIG_ENABLE_QUOTAS=True \
-# We need to set $HOME for some Tool Shed tools (e.g Perl libs with $HOME/.cpan)
-HOME=$GALAXY_HOME \
-GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda
+ GALAXY_CONFIG_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \
+ GALAXY_CONFIG_NEW_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \
+ GALAXY_CONFIG_TEMPLATE_CACHE_PATH=$EXPORT_DIR/galaxy-central/database/compiled_templates \
+ GALAXY_CONFIG_CITATION_CACHE_DATA_DIR=$EXPORT_DIR/galaxy-central/database/citations/data \
+ GALAXY_CONFIG_CLUSTER_FILES_DIRECTORY=$EXPORT_DIR/galaxy-central/database/pbs \
+ GALAXY_CONFIG_FTP_UPLOAD_DIR=$EXPORT_DIR/ftp \
+ GALAXY_CONFIG_FTP_UPLOAD_SITE=galaxy.docker.org \
+ GALAXY_CONFIG_USE_PBKDF2=False \
+ GALAXY_CONFIG_NGINX_X_ACCEL_REDIRECT_BASE=/_x_accel_redirect \
+ GALAXY_CONFIG_NGINX_X_ARCHIVE_FILES_BASE=/_x_accel_redirect \
+ GALAXY_CONFIG_DYNAMIC_PROXY_MANAGE=False \
+ GALAXY_CONFIG_VISUALIZATION_PLUGINS_DIRECTORY=config/plugins/visualizations \
+ GALAXY_CONFIG_TRUST_IPYTHON_NOTEBOOK_CONVERSION=True \
+ GALAXY_CONFIG_TOOLFORM_UPGRADE=True \
+ GALAXY_CONFIG_SANITIZE_ALL_HTML=False \
+ GALAXY_CONFIG_TOOLFORM_UPGRADE=True \
+ GALAXY_CONFIG_WELCOME_URL=$GALAXY_CONFIG_DIR/web/welcome.html \
+ GALAXY_CONFIG_OVERRIDE_DEBUG=False \
+ GALAXY_CONFIG_ENABLE_QUOTAS=True \
+ # We need to set $HOME for some Tool Shed tools (e.g Perl libs with $HOME/.cpan)
+ HOME=$GALAXY_HOME \
+ GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda
# Container Style
ADD GalaxyDocker.png $GALAXY_CONFIG_DIR/web/welcome_image.png
@@ -276,10 +281,10 @@ ADD welcome.html $GALAXY_CONFIG_DIR/web/welcome.html
# && find $GALAXY_VIRTUAL_ENV -name '*.pyc' -delete \
# && rm -rf /tmp/* $GALAXY_ROOT/client/node_modules/ $GALAXY_VIRTUAL_ENV/src/ /home/galaxy/.cache/ /home/galaxy/.npm
- # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments
- #cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \
- #npm install && \
- #rm -rf ~/.cache/ $GALAXY_ROOT/client/node_modules/
+# Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments
+#cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \
+#npm install && \
+#rm -rf ~/.cache/ $GALAXY_ROOT/client/node_modules/
# Switch back to User root
USER root
@@ -304,6 +309,10 @@ RUN echo "DISCARD_SESSION_KEYRING_ON_STARTUP=False" > /etc/condor/condor_config.
ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini /sbin/tini
RUN chmod +x /sbin/tini
+# https://stackoverflow.com/questions/62250160/uwsgi-runtimeerror-cannot-release-un-acquired-lock
+ADD run.sh $GALAXY_ROOT/run.sh
+RUN chmod +x $GALAXY_ROOT/run.sh && sed -i 's/py-call-osafterfork.*//g' /etc/galaxy/galaxy.yml
+
# This needs to happen here and not above, otherwise the Galaxy start
# (without running the startup.sh script) will crash because integrated_tool_panel.xml could not be found.
ENV GALAXY_CONFIG_INTEGRATED_TOOL_PANEL_CONFIG $EXPORT_DIR/galaxy-central/integrated_tool_panel.xml
diff --git a/galaxy/roles/galaxy-postgresql b/galaxy/roles/galaxy-postgresql
index f363e42e3..1507fbe5e 160000
--- a/galaxy/roles/galaxy-postgresql
+++ b/galaxy/roles/galaxy-postgresql
@@ -1 +1 @@
-Subproject commit f363e42e373dce38e8dceb75acb07e770f71b83c
+Subproject commit 1507fbe5eda946fc9f4c9aabef4e73a9b11a8315
diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras
index b9e47789f..82da900da 160000
--- a/galaxy/roles/galaxyprojectdotorg.galaxyextras
+++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras
@@ -1 +1 @@
-Subproject commit b9e47789f9b2be2a18d3db3d9585893ddefca062
+Subproject commit 82da900da5667a4d1e06ff976487d30ce460a59d
diff --git a/galaxy/run.sh b/galaxy/run.sh
new file mode 100755
index 000000000..fa54e0f25
--- /dev/null
+++ b/galaxy/run.sh
@@ -0,0 +1,98 @@
+#!/bin/sh
+
+
+# Usage: ./run.sh
+#
+#
+# Description: This script can be used to start or stop the galaxy
+# web application.
+
+cd "$(dirname "$0")"
+
+. ./scripts/common_startup_functions.sh
+
+
+# If there is a file that defines a shell environment specific to this
+# instance of Galaxy, source the file.
+if [ -z "$GALAXY_LOCAL_ENV_FILE" ];
+then
+ GALAXY_LOCAL_ENV_FILE='./config/local_env.sh'
+fi
+
+if [ -f "$GALAXY_LOCAL_ENV_FILE" ];
+then
+ . "$GALAXY_LOCAL_ENV_FILE"
+fi
+
+GALAXY_PID=${GALAXY_PID:-galaxy.pid}
+GALAXY_LOG=${GALAXY_LOG:-galaxy.log}
+PID_FILE=$GALAXY_PID
+LOG_FILE=$GALAXY_LOG
+
+parse_common_args $@
+
+run_common_start_up
+
+setup_python
+
+
+if [ ! -z "$GALAXY_RUN_WITH_TEST_TOOLS" ];
+then
+ export GALAXY_CONFIG_OVERRIDE_TOOL_CONFIG_FILE="$(pwd)/test/functional/tools/samples_tool_conf.xml"
+ export GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES="true"
+ export GALAXY_CONFIG_OVERRIDE_ENABLE_BETA_TOOL_FORMATS="true"
+ export GALAXY_CONFIG_INTERACTIVETOOLS_ENABLE="true"
+ export GALAXY_CONFIG_OVERRIDE_WEBHOOKS_DIR="test/functional/webhooks"
+fi
+
+set_galaxy_config_file_var
+
+
+if [ "$INITIALIZE_TOOL_DEPENDENCIES" -eq 1 ]; then
+ # Install Conda environment if needed.
+ python ./scripts/manage_tool_dependencies.py init_if_needed
+fi
+
+[ -n "$GALAXY_UWSGI" ] && APP_WEBSERVER='uwsgi'
+find_server "${GALAXY_CONFIG_FILE:-none}" galaxy
+server_args=`echo $server_args | sed 's/--py-call-osafterfork//g'`
+
+
+if [ "$run_server" = "python" -a -n "$GALAXY_RUN_ALL" ]; then
+ servers=$(sed -n 's/^\[server:\(.*\)\]/\1/ p' "$GALAXY_CONFIG_FILE" | xargs echo)
+ if [ -z "$stop_daemon_arg_set" -a -z "$daemon_or_restart_arg_set" ]; then
+ echo "ERROR: \$GALAXY_RUN_ALL cannot be used without the '--daemon', '--stop-daemon', 'restart', 'start' or 'stop' arguments to run.sh"
+ exit 1
+ fi
+ for server in $servers; do
+ echo "Executing: python $server_args --server-name=\"$server\" --pid-file=\"$server.pid\" --log-file=\"$server.log\""
+ eval python $server_args --server-name="$server" --pid-file="$server.pid" --log-file="$server.log"
+ if [ -n "$wait_arg_set" -a -n "$daemon_or_restart_arg_set" ]; then
+ while true; do
+ sleep 1
+ # Grab the current pid from the pid file and remove any trailing space
+ if ! current_pid_in_file=$(sed -e 's/[[:space:]]*$//' "$server.pid"); then
+ echo "A Galaxy process died, interrupting" >&2
+ exit 1
+ fi
+ if [ -n "$current_pid_in_file" ]; then
+ echo "Found PID $current_pid_in_file in '$server.pid', monitoring '$server.log'"
+ else
+ echo "No PID found in '$server.pid' yet"
+ continue
+ fi
+ # Search for all pids in the logs and tail for the last one
+ latest_pid=$(grep '^Starting server in PID [0-9]\+\.$' "$server.log" | sed 's/^Starting server in PID \([0-9]\{1,\}\).$/\1/' | tail -n 1)
+ # If they're equivalent, then the current pid file agrees with our logs
+ # and we've succesfully started
+ [ -n "$latest_pid" ] && [ "$latest_pid" -eq "$current_pid_in_file" ] && break
+ done
+ echo
+ fi
+ done
+else
+
+ echo "Executing: $run_server $server_args"
+ # args are properly quoted so use eval
+ eval $run_server $server_args
+fi
diff --git a/test/slurm/Dockerfile b/test/slurm/Dockerfile
index 76495a16d..bf74cf193 100644
--- a/test/slurm/Dockerfile
+++ b/test/slurm/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
ENV DEBIAN_FRONTEND noninteractive