From f1ec67fe108a9bc00ca0e54e93b420773318880d Mon Sep 17 00:00:00 2001 From: Gianmauro Cuccuru Date: Wed, 11 Nov 2020 12:26:56 +0100 Subject: [PATCH 01/31] Update Dockerfile Move ARG variables in the proper scope --- compose/galaxy-server/Dockerfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 9d4dc8872..2245216ec 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -4,9 +4,6 @@ ARG IMAGE_TAG=latest FROM buildpack-deps:18.04 as build_base -ARG GALAXY_RELEASE=release_20.05 -ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy - ENV EXPORT_DIR=/export \ GALAXY_ROOT=/galaxy \ HTCONDOR_ROOT=/opt/htcondor @@ -52,6 +49,10 @@ RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA_VERSI && /usr/bin/common_cleanup.sh FROM build_base as build_galaxy + +ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy + COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Install Galaxy RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libssl-dev python3-dev python3-pip -y \ @@ -72,6 +73,9 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss # --- Final image --- FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-cluster-base:$IMAGE_TAG as final +ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy + COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh COPY ./files/create_galaxy_user.py /usr/local/bin/create_galaxy_user.py @@ -80,9 +84,7 @@ ENV EXPORT_DIR=/export \ GALAXY_PYTHON=/usr/bin/python3 \ HTCONDOR_ROOT=/opt/htcondor -ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.05} \ - GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ - GALAXY_STATIC_DIR=$GALAXY_ROOT/static \ +ENV GALAXY_STATIC_DIR=$GALAXY_ROOT/static \ GALAXY_EXPORT=$EXPORT_DIR/galaxy \ GALAXY_CONFIG_DIR=$GALAXY_ROOT/config \ GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=/tool_deps \ From 15f6079540bdc934f47b48e9e915444843c4bc35 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 09:47:44 +0100 Subject: [PATCH 02/31] compose update to 20.09 --- compose/README.md | 4 ++-- compose/galaxy-htcondor/Dockerfile | 2 +- compose/galaxy-server/Dockerfile | 4 ++-- compose/galaxy-slurm/Dockerfile | 2 +- compose/tests/docker-compose.test.bioblend.yml | 2 +- compose/tests/galaxy-selenium-test/Dockerfile | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compose/README.md b/compose/README.md index 3007747d0..e56d19ac9 100644 --- a/compose/README.md +++ b/compose/README.md @@ -151,9 +151,9 @@ All options are discussed under [configuration reference](#configuration-referen ### Use specific Galaxy version or Docker images The `IMAGE_TAG` environment variable allows to use specific versions of the -setup. Say, you want to stay with Galaxy v20.05 for now: +setup. Say, you want to stay with Galaxy v20.09 for now: -> export IMAGE_TAG=20.05 +> export IMAGE_TAG=20.09 > docker-compose up Without setting this variable, you will always get updated to the newest diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index d10630bde..2d5425d40 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_TAG=latest FROM buildpack-deps:18.04 as galaxy_dependencies -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy ENV GALAXY_ROOT=/galaxy diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 2245216ec..94f94cbaf 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -50,7 +50,7 @@ RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA_VERSI FROM build_base as build_galaxy -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh @@ -73,7 +73,7 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss # --- Final image --- FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-cluster-base:$IMAGE_TAG as final -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 2f1328e8c..fdf66ceaa 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_TAG=latest FROM buildpack-deps:18.04 as galaxy_dependencies -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy ENV GALAXY_ROOT=/galaxy diff --git a/compose/tests/docker-compose.test.bioblend.yml b/compose/tests/docker-compose.test.bioblend.yml index 9ebcaf7b8..6b77fb405 100644 --- a/compose/tests/docker-compose.test.bioblend.yml +++ b/compose/tests/docker-compose.test.bioblend.yml @@ -4,7 +4,7 @@ services: image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-bioblend-test:${IMAGE_TAG:-latest} build: tests/galaxy-bioblend-test environment: - - GALAXY_VERSION=${GALAXY_VERSION:-release_20.05} # TODO: Change to GALAXY_RELEASE + - GALAXY_VERSION=${GALAXY_VERSION:-release_20.09} # TODO: Change to GALAXY_RELEASE - GALAXY_URL=http://nginx${GALAXY_PROXY_PREFIX:-}/ networks: - galaxy diff --git a/compose/tests/galaxy-selenium-test/Dockerfile b/compose/tests/galaxy-selenium-test/Dockerfile index f081a029e..453b12508 100644 --- a/compose/tests/galaxy-selenium-test/Dockerfile +++ b/compose/tests/galaxy-selenium-test/Dockerfile @@ -1,6 +1,6 @@ FROM selenium/standalone-chrome:3.141.59 -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy ENV GALAXY_ROOT=/galaxy From 032d2190b8047734a65c36b6167d20c353b6d697 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 09:50:19 +0100 Subject: [PATCH 03/31] git workflow update env vars --- .github/workflows/compose.yml | 6 +++--- .github/workflows/pull-request.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 2dbf435f6..a824ef01a 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -232,13 +232,13 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Set image tag in env - run: echo "::set-env name=IMAGE_TAG::${GITHUB_REF#refs/heads/}" + run: echo "IMAGE_TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - name: Master branch - Set image to to 'latest' if: github.ref == 'refs/heads/master' - run: echo "::set-env name=IMAGE_TAG::latest" + run: echo "IMAGE_TAG=latest" >> $GITHUB_ENV - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow - run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}" + run: echo "WORKFLOWS=${{ matrix.test.workflow }}" $GITHUB_ENV - name: Run tests for the first time if: steps.run_check.outputs.run run: | diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 9eb289190..c35105fad 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -108,7 +108,7 @@ jobs: uses: actions/checkout@v2 - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow - run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}" + run: echo "WORKFLOWS=${{ matrix.test.workflow }}" $GITHUB_ENV - name: Build galaxy-container-base env: image_name: galaxy-container-base From defa4326ea39b2b5204bd8f69bd0999b4b1d73c7 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 09:51:47 +0100 Subject: [PATCH 04/31] git workflow env correct piping --- .github/workflows/compose.yml | 2 +- .github/workflows/pull-request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index a824ef01a..1df8d1bf5 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -238,7 +238,7 @@ jobs: run: echo "IMAGE_TAG=latest" >> $GITHUB_ENV - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow - run: echo "WORKFLOWS=${{ matrix.test.workflow }}" $GITHUB_ENV + run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV - name: Run tests for the first time if: steps.run_check.outputs.run run: | diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index c35105fad..5b4aafe41 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -108,7 +108,7 @@ jobs: uses: actions/checkout@v2 - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow - run: echo "WORKFLOWS=${{ matrix.test.workflow }}" $GITHUB_ENV + run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV - name: Build galaxy-container-base env: image_name: galaxy-container-base From 2bf03b7d277754e7c25b83377da507471f8d01bc Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 09:53:04 +0100 Subject: [PATCH 05/31] git workflow switch to official galaxy repo --- .github/workflows/compose.yml | 2 +- .github/workflows/pull-request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 1df8d1bf5..7615c6eb6 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -120,7 +120,7 @@ jobs: --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \ --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \ --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \ - --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy \ + --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy \ ${{ matrix.image.subdir }}${{ matrix.image.name }} && break || echo "Fail.. Retrying" done; shell: bash diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 5b4aafe41..050f5de85 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -142,7 +142,7 @@ jobs: echo "Removing export directory if existent"; sudo rm -rf export set +e - docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy + docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }} test_exit_code=$? error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1) From e55dddee96564d3091ad794cb036fe71352ad149 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 10:00:34 +0100 Subject: [PATCH 06/31] Update to ubuntu 20.04 --- README.md | 2 ++ compose/README.md | 2 +- compose/base-images/galaxy-container-base/Dockerfile | 4 ++-- .../galaxy-configurator/templates/galaxy/job_conf.xml.j2 | 2 +- compose/galaxy-htcondor/Dockerfile | 2 +- compose/galaxy-server/Dockerfile | 2 +- compose/galaxy-slurm/Dockerfile | 2 +- compose/pulsar/Dockerfile | 2 +- compose/tests/galaxy-selenium-test/Dockerfile | 9 +++++---- galaxy/Dockerfile | 4 ++-- test/slurm/Dockerfile | 2 +- 11 files changed, 18 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a64c50650..6e23c7b71 100644 --- a/README.md +++ b/README.md @@ -993,6 +993,8 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro - Featuring Galaxy 20.05 - Completely reworked compose setup - The default admin password and apikey (`GALAXY_DEFAULT_ADMIN_PASSWORD` and `GALAXY_DEFAULT_ADMIN_KEY`) have changed: the password is now `password` (instead of `admin`) and the apikey `fakekey` (instead of `admin`). +- 20.09: + - Featuring Galaxy 20.09 # Support & Bug Reports [[toc]](#toc) diff --git a/compose/README.md b/compose/README.md index e56d19ac9..8f9d49576 100644 --- a/compose/README.md +++ b/compose/README.md @@ -378,7 +378,7 @@ The following are settings specific to this docker-compose setup: | `GALAXY_K8S_DOCKER_REPO_DEFAULT` | The Docker Repo/Registry to use if the resolver could not resolve the proper image for a job. Defaults to `docker.io`. | | `GALAXY_K8S_DOCKER_OWNER_DEFAULT` | The Owner/Username to use if the resolver could not resolve the proper image for a job. Is not set by default. | | `GALAXY_K8S_DOCKER_IMAGE_DEFAULT` | The Image to use if the resolver could not resolve the proper image for a job. Defaults to `ubuntu`. | -| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `18.04`. | +| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `20.04`. | ### HTCondor | Variable | Description | diff --git a/compose/base-images/galaxy-container-base/Dockerfile b/compose/base-images/galaxy-container-base/Dockerfile index 20fa3743f..669533266 100644 --- a/compose/base-images/galaxy-container-base/Dockerfile +++ b/compose/base-images/galaxy-container-base/Dockerfile @@ -1,4 +1,4 @@ -FROM buildpack-deps:18.04 as build_singularity +FROM buildpack-deps:20.04 as build_singularity COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh @@ -21,7 +21,7 @@ RUN wget https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_ # --- Final image --- -FROM ubuntu:18.04 as final +FROM ubuntu:20.04 as final COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh diff --git a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 index 461dfb5ff..ed4a295ca 100644 --- a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 +++ b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 @@ -59,7 +59,7 @@ {{ GALAXY_K8S_DOCKER_REPO_DEFAULT | default('docker.io') }} {% if GALAXY_K8S_DOCKER_OWNER_DEFAULT -%}{{ GALAXY_K8S_DOCKER_OWNER_DEFAULT }}{% endif -%} {{ GALAXY_K8S_DOCKER_IMAGE_DEFAULT | default('ubuntu') }} - {{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('18.04') }} + {{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('20.04') }} true {% endif -%} diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index 2d5425d40..c012c75b6 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as galaxy_dependencies +FROM buildpack-deps:20.04 as galaxy_dependencies ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 94f94cbaf..c29e5d4f9 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as build_base +FROM buildpack-deps:20.04 as build_base ENV EXPORT_DIR=/export \ GALAXY_ROOT=/galaxy \ diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index fdf66ceaa..f4c5f616d 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as galaxy_dependencies +FROM buildpack-deps:20.04 as galaxy_dependencies ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy diff --git a/compose/pulsar/Dockerfile b/compose/pulsar/Dockerfile index 2fb2a7d10..1d63ec2cc 100644 --- a/compose/pulsar/Dockerfile +++ b/compose/pulsar/Dockerfile @@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as build_pulsar +FROM buildpack-deps:20.04 as build_pulsar ARG PULSAR_RELEASE=0.13.0 ARG PULSAR_REPO=https://github.com/galaxyproject/pulsar diff --git a/compose/tests/galaxy-selenium-test/Dockerfile b/compose/tests/galaxy-selenium-test/Dockerfile index 453b12508..c72e3ba66 100644 --- a/compose/tests/galaxy-selenium-test/Dockerfile +++ b/compose/tests/galaxy-selenium-test/Dockerfile @@ -3,16 +3,17 @@ FROM selenium/standalone-chrome:3.141.59 ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy -ENV GALAXY_ROOT=/galaxy -ENV GALAXY_PYTHON=/usr/bin/python +ENV GALAXY_ROOT=/galaxy3 +ENV GALAXY_PYTHON=/usr/bin/python3 USER root -RUN apt update && apt install --no-install-recommends python-dev python-pip -y && rm -rf /var/lib/apt/lists/* \ +RUN apt update && apt install --no-install-recommends python3-dev python3-pip -y && rm -rf /var/lib/apt/lists/* \ && mkdir "${GALAXY_ROOT}" \ && chown seluser "${GALAXY_ROOT}" USER seluser -RUN curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ +RUN mkdir -p $GALAXY_ROOT && \ + curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ && cd "${GALAXY_ROOT}" \ && ./scripts/common_startup.sh --skip-client-build --dev-wheels diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 2b46ec66c..a992f1e58 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -2,7 +2,7 @@ # # VERSION Galaxy-central -FROM ubuntu:18.04 +FROM ubuntu:20.04 MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com @@ -16,7 +16,7 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG GALAXY_RELEASE ARG GALAXY_REPO -ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.05} \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \ GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ GALAXY_ROOT=/galaxy-central \ GALAXY_CONFIG_DIR=/etc/galaxy \ diff --git a/test/slurm/Dockerfile b/test/slurm/Dockerfile index 76495a16d..bf74cf193 100644 --- a/test/slurm/Dockerfile +++ b/test/slurm/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 ENV DEBIAN_FRONTEND noninteractive From 1fa182388a8f6d0719d12de5aa5bc8ee02c5ebde Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 10:02:27 +0100 Subject: [PATCH 07/31] git workflow update ubuntu to 20.04 --- .github/workflows/compose.yml | 8 ++++---- .github/workflows/lint.yml | 2 +- .github/workflows/pull-request.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 7615c6eb6..9d96b5d21 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -2,7 +2,7 @@ name: build-and-test on: [push] jobs: build_container_base: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v2 @@ -38,7 +38,7 @@ jobs: working-directory: ./compose/base-images build_cluster_base: needs: build_container_base - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v2 @@ -74,7 +74,7 @@ jobs: working-directory: ./compose/base-images build: needs: build_cluster_base - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: image: @@ -127,7 +127,7 @@ jobs: working-directory: ./compose test: needs: [build] - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: infrastructure: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 63693b86d..f0c4b46d2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,7 +2,7 @@ name: Lint on: [push] jobs: lint: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 050f5de85..c660a7ca2 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -2,7 +2,7 @@ name: pr-test on: pull_request jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: infrastructure: From d0e42f9aa238dca36a5e3f67082097f6538952c5 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 10:03:19 +0100 Subject: [PATCH 08/31] set thunder-lock true --- compose/base_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/base_config.yml b/compose/base_config.yml index cc2970901..f3640e038 100644 --- a/compose/base_config.yml +++ b/compose/base_config.yml @@ -9,7 +9,7 @@ galaxy_uwsgi: master: false virtualenv: .venv pythonpath: lib - thunder-lock: false + thunder-lock: true die-on-term: true hook-master-start: unix_signal:2 gracefully_kill_them_all hook-master-start: unix_signal:15 gracefully_kill_them_all From 6ebbdc4fd91a8625d8496ca968db95358307b012 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 10:03:53 +0100 Subject: [PATCH 09/31] remove py-call-osafterfork --- compose/base_config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/base_config.yml b/compose/base_config.yml index f3640e038..46f37dc7c 100644 --- a/compose/base_config.yml +++ b/compose/base_config.yml @@ -13,7 +13,6 @@ galaxy_uwsgi: die-on-term: true hook-master-start: unix_signal:2 gracefully_kill_them_all hook-master-start: unix_signal:15 gracefully_kill_them_all - py-call-osafterfork: true enable-threads: true galaxy: From 7aa4bef6fa1fb7e8a6e182a9b0fb3c73349d0b22 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 10:23:02 +0100 Subject: [PATCH 10/31] singularity add galaxy env file resolution --- compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 index ed4a295ca..9b1ca49ed 100644 --- a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 +++ b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 @@ -32,6 +32,7 @@ {% if GALAXY_DEPENDENCY_RESOLUTION == 'singularity' -%} + /home/galaxy true {% if GALAXY_JOB_RUNNER == 'local' -%} From 7f898dfd3ce30262494c600aa59b3b955b1f7bb6 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 13:29:34 +0100 Subject: [PATCH 11/31] update Miniconda version --- compose/galaxy-server/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index c29e5d4f9..e3ce4726b 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -24,7 +24,7 @@ ENV GALAXY_USER=galaxy \ GALAXY_HOME=/home/galaxy ENV GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda \ - MINICONDA_VERSION=4.7.10 + MINICONDA_VERSION=py38_4.9.2 RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ && useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" --shell /bin/bash $GALAXY_USER \ @@ -34,7 +34,7 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ FROM build_base as build_miniconda COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Install Miniconda -RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA_VERSION}-Linux-x86_64.sh" > ~/miniconda.sh \ +RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ From 13177af8b7ee4bcadc1f8c2286cd4f86b162acb0 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 13:31:08 +0100 Subject: [PATCH 12/31] upgrade pulsar to v0.14.0 --- compose/pulsar/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/pulsar/Dockerfile b/compose/pulsar/Dockerfile index 1d63ec2cc..5d485cc81 100644 --- a/compose/pulsar/Dockerfile +++ b/compose/pulsar/Dockerfile @@ -4,7 +4,7 @@ ARG IMAGE_TAG=latest FROM buildpack-deps:20.04 as build_pulsar -ARG PULSAR_RELEASE=0.13.0 +ARG PULSAR_RELEASE=0.14.0 ARG PULSAR_REPO=https://github.com/galaxyproject/pulsar ENV PULSAR_ROOT=/pulsar From a4cabfbef1e0c1ca1246d99750a9b6671fe8331b Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 13:31:34 +0100 Subject: [PATCH 13/31] upgrade container-base sources to focal --- compose/base-images/galaxy-cluster-base/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/base-images/galaxy-cluster-base/Dockerfile b/compose/base-images/galaxy-cluster-base/Dockerfile index f9d668009..dc5a2cb08 100644 --- a/compose/base-images/galaxy-cluster-base/Dockerfile +++ b/compose/base-images/galaxy-cluster-base/Dockerfile @@ -20,8 +20,8 @@ ENV MUNGER_USER=munge \ MUNGE_GID=1200 RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ && useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \ - && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \ - && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \ + && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \ + && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \ && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8DE68488997C5C6BA19021136F2CC56412788738 \ && apt update \ && apt install --no-install-recommends slurm-client slurmd slurmctld slurm-drmaa1 -y \ From 332357f27a8b18852f7eae62e335822f31c58b47 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 13:36:34 +0100 Subject: [PATCH 14/31] update to python3 --- compose/base-images/galaxy-cluster-base/Dockerfile | 2 +- compose/base-images/galaxy-container-base/Dockerfile | 2 +- compose/galaxy-server/Dockerfile | 3 +-- compose/pulsar/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/compose/base-images/galaxy-cluster-base/Dockerfile b/compose/base-images/galaxy-cluster-base/Dockerfile index dc5a2cb08..ff7c91cce 100644 --- a/compose/base-images/galaxy-cluster-base/Dockerfile +++ b/compose/base-images/galaxy-cluster-base/Dockerfile @@ -24,7 +24,7 @@ RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \ && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8DE68488997C5C6BA19021136F2CC56412788738 \ && apt update \ - && apt install --no-install-recommends slurm-client slurmd slurmctld slurm-drmaa1 -y \ + && apt install --no-install-recommends python3-distutils slurm-client slurmd slurmctld slurm-drmaa1 -y \ && apt --no-install-recommends install munge libmunge-dev -y \ && ln -s /usr/lib/slurm-drmaa/lib/libdrmaa.so.1 /usr/lib/slurm-drmaa/lib/libdrmaa.so \ && /usr/bin/common_cleanup.sh diff --git a/compose/base-images/galaxy-container-base/Dockerfile b/compose/base-images/galaxy-container-base/Dockerfile index 669533266..6cf33a516 100644 --- a/compose/base-images/galaxy-container-base/Dockerfile +++ b/compose/base-images/galaxy-container-base/Dockerfile @@ -26,7 +26,7 @@ FROM ubuntu:20.04 as final COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Base dependencies -RUN apt update && apt install --no-install-recommends ca-certificates squashfs-tools -y \ +RUN apt update && apt install --no-install-recommends ca-certificates python3-distutils squashfs-tools -y \ && /usr/bin/common_cleanup.sh # Install Docker diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index e3ce4726b..a7521ce4e 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -120,8 +120,7 @@ RUN apt update && apt install --no-install-recommends curl gcc gnupg2 libgomp1 l RUN usermod -aG docker $GALAXY_USER # Make Python3 standard -RUN mv /usr/bin/python /usr/bin/python2 \ - && ln /usr/bin/python3 /usr/bin/python +RUN ln /usr/bin/python3 /usr/bin/python COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_galaxy ${GALAXY_ROOT} ${GALAXY_ROOT} COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} diff --git a/compose/pulsar/Dockerfile b/compose/pulsar/Dockerfile index 5d485cc81..dac9b1503 100644 --- a/compose/pulsar/Dockerfile +++ b/compose/pulsar/Dockerfile @@ -34,7 +34,7 @@ ENV PULSAR_VIRTUALENV=$PULSAR_ROOT/.venv \ PULSAR_TOOL_DEPENDENCY_DIR=$PULSAR_ROOT/dependencies RUN apt update \ - && apt install --no-install-recommends ca-certificates curl libxml2-dev libpython3.6 python3 -y \ + && apt install --no-install-recommends ca-certificates curl libxml2-dev python3 -y \ && /usr/bin/common_cleanup.sh COPY --from=build_pulsar /pulsar /pulsar From 8face0342e9072347641bf184f5e67df87ca859d Mon Sep 17 00:00:00 2001 From: jonas27 Date: Mon, 8 Feb 2021 20:33:46 +0100 Subject: [PATCH 15/31] workflow tests ignore filetype --- compose/tests/galaxy-workflow-test/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/tests/galaxy-workflow-test/Dockerfile b/compose/tests/galaxy-workflow-test/Dockerfile index b392babbb..a5ca295ba 100644 --- a/compose/tests/galaxy-workflow-test/Dockerfile +++ b/compose/tests/galaxy-workflow-test/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.11 -ENV TEST_REPO=${TEST_REPO:-https://github.com/usegalaxy-eu/workflow-testing} \ - TEST_RELEASE=${TEST_RELEASE:-master} +ENV TEST_REPO=${TEST_REPO:-https://github.com/jonas27/workflow-testing} \ + TEST_RELEASE=${TEST_RELEASE:-20.09-comment-filetype} RUN apk add --no-cache bash python3 curl \ && apk add --no-cache --virtual build-dep gcc libxml2-dev libxslt-dev musl-dev linux-headers python3-dev \ From e2f69c1972e5da72587b63535980ff7df80b9fdc Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 06:39:47 +0100 Subject: [PATCH 16/31] python3 standard --- compose/galaxy-htcondor/Dockerfile | 2 ++ compose/galaxy-server/Dockerfile | 2 +- compose/galaxy-slurm/Dockerfile | 2 ++ compose/tests/galaxy-workflow-test/Dockerfile | 3 +++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index c012c75b6..7fc2c5d06 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -67,4 +67,6 @@ COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT $ COPY start.sh /usr/bin/start.sh RUN apt update && apt install python3 -y +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 + ENTRYPOINT /usr/bin/start.sh diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index a7521ce4e..06b40b236 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -120,7 +120,7 @@ RUN apt update && apt install --no-install-recommends curl gcc gnupg2 libgomp1 l RUN usermod -aG docker $GALAXY_USER # Make Python3 standard -RUN ln /usr/bin/python3 /usr/bin/python +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_galaxy ${GALAXY_ROOT} ${GALAXY_ROOT} COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index f4c5f616d..760749ab2 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -50,6 +50,8 @@ RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ # Copy Galaxy dependencies COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT $GALAXY_ROOT +# Make Python3 standard +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 COPY start.sh /usr/bin/start.sh diff --git a/compose/tests/galaxy-workflow-test/Dockerfile b/compose/tests/galaxy-workflow-test/Dockerfile index a5ca295ba..1b9385da9 100644 --- a/compose/tests/galaxy-workflow-test/Dockerfile +++ b/compose/tests/galaxy-workflow-test/Dockerfile @@ -10,6 +10,9 @@ RUN apk add --no-cache bash python3 curl \ && curl -L -s $TEST_REPO/archive/$TEST_RELEASE.tar.gz | tar xzf - --strip-components=1 \ && apk del build-dep +# Make Python3 standard +RUN ln /usr/bin/python3 /usr/bin/python && ln /usr/bin/python3 /usr/bin/python2 + ADD ./run.sh /usr/bin/run.sh WORKDIR /src From 7432ab43b143af5d56ffb9f55031fa8cf0daf84c Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 07:32:41 +0100 Subject: [PATCH 17/31] galaxy dependency add files --- compose/galaxy-htcondor/Dockerfile | 2 +- compose/galaxy-slurm/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index 7fc2c5d06..ac49a826b 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -18,7 +18,7 @@ RUN mkdir "${GALAXY_ROOT}" \ && cd $GALAXY_ROOT/lib \ && ls . | grep -v "galaxy\|galaxy_ext" | xargs rm -rf \ && cd $GALAXY_ROOT/lib/galaxy \ - && ls . | grep -v "__init__.py\|datatypes\|exceptions\|metadata\|model\|util\|security" | xargs rm -rf + && ls . | grep -v "__init__.py\|datatypes\|exceptions\|files\|metadata\|model\|util\|security" | xargs rm -rf FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-container-base:$IMAGE_TAG as final diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 760749ab2..13d6b6ce9 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -17,7 +17,7 @@ RUN mkdir "${GALAXY_ROOT}" \ && cd $GALAXY_ROOT/lib \ && ls . | grep -v "galaxy\|galaxy_ext" | xargs rm -rf \ && cd $GALAXY_ROOT/lib/galaxy \ - && ls . | grep -v "__init__.py\|datatypes\|exceptions\|metadata\|model\|util\|security" | xargs rm -rf + && ls . | grep -v "__init__.py\|datatypes\|exceptions\|files\|metadata\|model\|util\|security" | xargs rm -rf FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-container-base:$IMAGE_TAG as final From 4ac618ecae5985c6ca0060dbc2ee3c15fda2a55f Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 10:16:46 +0100 Subject: [PATCH 18/31] galaxy-server set repo env --- compose/galaxy-server/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 06b40b236..314ba269a 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -73,9 +73,6 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss # --- Final image --- FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-cluster-base:$IMAGE_TAG as final -ARG GALAXY_RELEASE=release_20.09 -ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy - COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh COPY ./files/create_galaxy_user.py /usr/local/bin/create_galaxy_user.py @@ -84,7 +81,9 @@ ENV EXPORT_DIR=/export \ GALAXY_PYTHON=/usr/bin/python3 \ HTCONDOR_ROOT=/opt/htcondor -ENV GALAXY_STATIC_DIR=$GALAXY_ROOT/static \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \ + GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ + GALAXY_STATIC_DIR=$GALAXY_ROOT/static \ GALAXY_EXPORT=$EXPORT_DIR/galaxy \ GALAXY_CONFIG_DIR=$GALAXY_ROOT/config \ GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=/tool_deps \ From 656d75b620f511df512108ebd1c6f81e265a98a1 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 14:53:05 +0100 Subject: [PATCH 19/31] htcondor and slurm include galaxy lib tools --- compose/docker-compose.htcondor.yml | 1 + compose/docker-compose.slurm.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/compose/docker-compose.htcondor.yml b/compose/docker-compose.htcondor.yml index 1b7c36702..c0579a14e 100644 --- a/compose/docker-compose.htcondor.yml +++ b/compose/docker-compose.htcondor.yml @@ -28,6 +28,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/htcondor:/config - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database + - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/lib/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv diff --git a/compose/docker-compose.slurm.yml b/compose/docker-compose.slurm.yml index 134f5e06d..5d55c7512 100644 --- a/compose/docker-compose.slurm.yml +++ b/compose/docker-compose.slurm.yml @@ -40,6 +40,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro + - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/lib/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv - ${EXPORT_DIR:-./export}/tool_deps:/tool_deps From 09a842f1ff8ea0308897479194db03be1ea7d09f Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 14:55:18 +0100 Subject: [PATCH 20/31] galaxy make ython3 standard --- compose/galaxy-server/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 314ba269a..19feaeb18 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -56,6 +56,7 @@ ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Install Galaxy RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libssl-dev python3-dev python3-pip -y \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3 9 \ && mkdir "${GALAXY_ROOT}" \ && curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ && cd $GALAXY_ROOT \ From 06a0877d0fccc464b185e4f6e71bf7d449d6d7c0 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 14:58:05 +0100 Subject: [PATCH 21/31] galaxy-server pip install deps --- compose/galaxy-server/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 19feaeb18..d789c2250 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -63,6 +63,7 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss && ./scripts/common_startup.sh \ && . $GALAXY_ROOT/.venv/bin/activate \ && pip3 install drmaa psycopg2 pycurl pykube \ + && pip3 install importlib-metadata importlib-resources pathlib2 ruamel.yaml.clib typing zipp \ && deactivate \ && rm -rf .ci .circleci .coveragerc .gitignore .travis.yml CITATION CODE_OF_CONDUCT.md CONTRIBUTING.md CONTRIBUTORS.md \ LICENSE.txt Makefile README.rst SECURITY_POLICY.md pytest.ini tox.ini \ From 31066d0b7c67aba122fd51f0de32dece78b62a20 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 14:59:40 +0100 Subject: [PATCH 22/31] slurm python3 --- compose/galaxy-slurm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 13d6b6ce9..6f54338b2 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -45,7 +45,7 @@ ENV SLURM_USER=galaxy \ RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ && useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \ && apt update \ - && apt install --no-install-recommends gosu munge python python-dev slurm-wlm -y \ + && apt install --no-install-recommends gosu munge python3 python3-dev slurm-wlm -y \ && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/* && find / -name '*.pyc' -delete # Copy Galaxy dependencies From 3ecdf5044c1e558cd63ca562087cc2da671dd956 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 15:01:25 +0100 Subject: [PATCH 23/31] selenium test correct typo --- compose/tests/galaxy-selenium-test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/tests/galaxy-selenium-test/Dockerfile b/compose/tests/galaxy-selenium-test/Dockerfile index c72e3ba66..370264852 100644 --- a/compose/tests/galaxy-selenium-test/Dockerfile +++ b/compose/tests/galaxy-selenium-test/Dockerfile @@ -3,7 +3,7 @@ FROM selenium/standalone-chrome:3.141.59 ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy -ENV GALAXY_ROOT=/galaxy3 +ENV GALAXY_ROOT=/galaxy ENV GALAXY_PYTHON=/usr/bin/python3 USER root From b16127cef3110d3b297e6fc0857f7762d195f4cf Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 16:43:18 +0100 Subject: [PATCH 24/31] readme add tests matrix --- compose/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/compose/README.md b/compose/README.md index 8f9d49576..41aaaf46c 100644 --- a/compose/README.md +++ b/compose/README.md @@ -393,3 +393,20 @@ The following are settings specific to this docker-compose setup: | `SLURM_NODE_CPUS` | Number of CPUs per node. Defaults to 1. | | `SLURM_NODE_MEMORY` | Amount of memory per node. Defaults to 1024. | | `SLURM_NODE_HOSTNAME` | Docker Compose adds a prefix in front of the container names by default. Change this value to the name of your setup and `_slurm_node` (e.g. `compose_slurm_node`) to ensure a correct mapping of the Slurm nodes. | + +### Github Workflow Tests (Branch 20.09) +| Setup | bioblend | workflow ard | workflow mapping_by_sequencing | workflow wf3-shed-tools (example1) | selenium | +|------------------------|--------------------|--------------------|--------------------------------|------------------------------------|--------------------| +| Galaxy Base | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| Galaxy Proxy Prefix | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| HTCondor | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| Slurm | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| Pulsar | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | +| k8s | :x: | :x: | :x: | :heavy_check_mark: | :x: | +| Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | +| Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | +| HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | + + +Implemented: :heavy_check_mark: +Not Implemented: :x: From 257b1c1eb939f9812d31cc820fdb7397ab3a7126 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 17:15:42 +0100 Subject: [PATCH 25/31] galaxy tools volume export in galaxy and mount in htcondor and slurm --- compose/docker-compose.htcondor.yml | 2 +- compose/docker-compose.slurm.yml | 2 +- compose/galaxy-server/files/start.sh | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/compose/docker-compose.htcondor.yml b/compose/docker-compose.htcondor.yml index c0579a14e..35084397f 100644 --- a/compose/docker-compose.htcondor.yml +++ b/compose/docker-compose.htcondor.yml @@ -28,7 +28,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/htcondor:/config - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database - - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/lib/galaxy/tools:ro + - ${EXPORT_DIR:-./export}/galaxy/lib/galaxy/tools:/galaxy/lib/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv diff --git a/compose/docker-compose.slurm.yml b/compose/docker-compose.slurm.yml index 5d55c7512..4cf6e29cb 100644 --- a/compose/docker-compose.slurm.yml +++ b/compose/docker-compose.slurm.yml @@ -40,7 +40,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro - - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/lib/galaxy/tools:ro + - ${EXPORT_DIR:-./export}/galaxy/lib/galaxy/tools:/galaxy/lib/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv - ${EXPORT_DIR:-./export}/tool_deps:/tool_deps diff --git a/compose/galaxy-server/files/start.sh b/compose/galaxy-server/files/start.sh index baf672165..860280856 100755 --- a/compose/galaxy-server/files/start.sh +++ b/compose/galaxy-server/files/start.sh @@ -13,6 +13,14 @@ create_user() { deactivate } +# start copy lib/tools. Looks very hacky. +tools_dir="/galaxy/lib/galaxy/tools/" +exp_dir="/export$tools_dir" +mkdir -p $exp_dir +chown "$GALAXY_USER:$GALAXY_USER" $exp_dir +cp -rf $tools_dir/* $exp_dir +# end copy lib/tools. + # First start?? Check if something exists that indicates that environment is not new.. Config file? Something in DB maybe?? echo "Initialization: Check if files already exist, export otherwise." From 23bb51b00eab436f63372bb1e6b4136ad6a99349 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 18:18:52 +0100 Subject: [PATCH 26/31] README tiny delete --- compose/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/README.md b/compose/README.md index 41aaaf46c..a2c913d67 100644 --- a/compose/README.md +++ b/compose/README.md @@ -407,6 +407,5 @@ The following are settings specific to this docker-compose setup: | Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | | HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | - Implemented: :heavy_check_mark: Not Implemented: :x: From 1b7eda1bb24351a477faf6d62af88774fca422c3 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Tue, 9 Feb 2021 19:35:30 +0100 Subject: [PATCH 27/31] github workflow use ubuntu-latest again --- .github/workflows/compose.yml | 6 +++--- .github/workflows/lint.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 9d96b5d21..475bbff93 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -38,7 +38,7 @@ jobs: working-directory: ./compose/base-images build_cluster_base: needs: build_container_base - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 @@ -74,7 +74,7 @@ jobs: working-directory: ./compose/base-images build: needs: build_cluster_base - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: image: @@ -127,7 +127,7 @@ jobs: working-directory: ./compose test: needs: [build] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: infrastructure: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f0c4b46d2..63693b86d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,7 +2,7 @@ name: Lint on: [push] jobs: lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 From 272c2062ae290f5d99b8adc57f50d864d5fda94e Mon Sep 17 00:00:00 2001 From: jonas27 Date: Wed, 10 Feb 2021 06:30:28 +0100 Subject: [PATCH 28/31] Revert "README tiny delete" This reverts commit 23bb51b00eab436f63372bb1e6b4136ad6a99349. --- compose/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/README.md b/compose/README.md index a2c913d67..41aaaf46c 100644 --- a/compose/README.md +++ b/compose/README.md @@ -407,5 +407,6 @@ The following are settings specific to this docker-compose setup: | Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | | HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | + Implemented: :heavy_check_mark: Not Implemented: :x: From 85be8c5ecb0890e747c2ee299e00879f6a647521 Mon Sep 17 00:00:00 2001 From: jonas27 Date: Wed, 10 Feb 2021 11:11:15 +0100 Subject: [PATCH 29/31] exclude pulsar and k8s not working test --- .github/workflows/pull-request.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index c660a7ca2..12baa530c 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -30,6 +30,7 @@ jobs: exclude_test: - workflow_example1 - workflow_mapping_by_sequencing + - workflow_ard # - name: galaxy-pulsar-mq # files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml # exclude_test: @@ -41,6 +42,7 @@ jobs: - bioblend - workflow_ard - workflow_mapping_by_sequencing + - workflow_example1 - selenium - name: galaxy-singularity files: -f docker-compose.yml -f docker-compose.singularity.yml From 8af26008ffd4f67cbe109364500197cd8853fd3b Mon Sep 17 00:00:00 2001 From: jonas27 Date: Wed, 17 Feb 2021 07:29:38 +0100 Subject: [PATCH 30/31] Trigger PR Build --- compose/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/README.md b/compose/README.md index 41aaaf46c..d8e6eda1b 100644 --- a/compose/README.md +++ b/compose/README.md @@ -410,3 +410,4 @@ The following are settings specific to this docker-compose setup: Implemented: :heavy_check_mark: Not Implemented: :x: + From 7dd620ec87b03a5e40ff5eed431c36ffca5447ce Mon Sep 17 00:00:00 2001 From: jonas27 Date: Wed, 17 Feb 2021 08:23:50 +0100 Subject: [PATCH 31/31] README test matrix add not implemented tests --- compose/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/README.md b/compose/README.md index d8e6eda1b..d73d39a5d 100644 --- a/compose/README.md +++ b/compose/README.md @@ -401,8 +401,8 @@ The following are settings specific to this docker-compose setup: | Galaxy Proxy Prefix | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | | HTCondor | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | | Slurm | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | -| Pulsar | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | :heavy_check_mark: | -| k8s | :x: | :x: | :x: | :heavy_check_mark: | :x: | +| Pulsar | :heavy_check_mark: | :x: | :x: | :x: | :heavy_check_mark: | +| k8s | :x: | :x: | :x: | :x: | :x: | | Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | | Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | | HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: |