diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index 2dbf435f6..c076434d3 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -2,7 +2,7 @@ name: build-and-test on: [push] jobs: build_container_base: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v2 @@ -120,7 +120,7 @@ jobs: --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \ --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \ --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \ - --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy \ + --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy \ ${{ matrix.image.subdir }}${{ matrix.image.name }} && break || echo "Fail.. Retrying" done; shell: bash @@ -164,6 +164,7 @@ jobs: files: -f docker-compose.yml -f docker-compose.k8s.yml exclude_test: - bioblend + - workflow_example1 - workflow_ard - workflow_mapping_by_sequencing - selenium @@ -232,13 +233,13 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Set image tag in env - run: echo "::set-env name=IMAGE_TAG::${GITHUB_REF#refs/heads/}" + run: echo "IMAGE_TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - name: Master branch - Set image to to 'latest' if: github.ref == 'refs/heads/master' - run: echo "::set-env name=IMAGE_TAG::latest" + run: echo "IMAGE_TAG=latest" >> $GITHUB_ENV - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow - run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}" + run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV - name: Run tests for the first time if: steps.run_check.outputs.run run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 63693b86d..36692201a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ jobs: level: warning pattern: "*.sh" - name: Run hadolint with reviewdog - uses: reviewdog/action-hadolint@v1 + uses: reviewdog/action-hadolint@v1.16.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} reporter: github-check diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 9eb289190..12baa530c 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -2,7 +2,7 @@ name: pr-test on: pull_request jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: infrastructure: @@ -30,6 +30,7 @@ jobs: exclude_test: - workflow_example1 - workflow_mapping_by_sequencing + - workflow_ard # - name: galaxy-pulsar-mq # files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml # exclude_test: @@ -41,6 +42,7 @@ jobs: - bioblend - workflow_ard - workflow_mapping_by_sequencing + - workflow_example1 - selenium - name: galaxy-singularity files: -f docker-compose.yml -f docker-compose.singularity.yml @@ -108,7 +110,7 @@ jobs: uses: actions/checkout@v2 - name: Set WORKFLOWS env for worfklows-test if: matrix.test.workflow - run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}" + run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV - name: Build galaxy-container-base env: image_name: galaxy-container-base @@ -142,7 +144,7 @@ jobs: echo "Removing export directory if existent"; sudo rm -rf export set +e - docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy + docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }} test_exit_code=$? error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1) diff --git a/.github/workflows/single.sh b/.github/workflows/single.sh new file mode 100755 index 000000000..446d867ae --- /dev/null +++ b/.github/workflows/single.sh @@ -0,0 +1,139 @@ +#!/bin/bash +set -ex + +export GALAXY_HOME=/home/galaxy +export GALAXY_USER=admin@galaxy.org +export GALAXY_USER_EMAIL=admin@galaxy.org +export GALAXY_USER_PASSWD=password +export BIOBLEND_GALAXY_API_KEY=fakekey +export BIOBLEND_GALAXY_URL=http://localhost:8080 + +sudo apt-get update -qq +#sudo apt-get install docker-ce --no-install-recommends -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" +sudo apt-get install sshpass --no-install-recommends -y + +pip3 install ephemeris + +docker --version +docker info + +# start building this repo +git submodule update --init --recursive +sudo chown 1450 /tmp && sudo chmod a=rwx /tmp + +## define a container size check function, first parameter is the container name, second the max allowed size in MB +container_size_check () { + + # check that the image size is not growing too much between releases + # the 19.05 monolithic image was around 1.500 MB + size="${docker image inspect $1 --format='{{.Size}}'}" + size_in_mb=$(($size/(1024*1024))) + if [[ $size_in_mb -ge $2 ]] + then + echo "The new compiled image ($1) is larger than allowed. $size_in_mb vs. $2" + sleep 2 + #exit + fi +} + +export WORKING_DIR=${GITHUB_WORKSPACE:-$PWD} + +export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy" +SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml +cd "$WORKING_DIR" +docker build -t quay.io/bgruening/galaxy galaxy/ +#container_size_check quay.io/bgruening/galaxy 1500 + +mkdir local_folder +docker run -d -p 8080:80 -p 8021:21 -p 8022:22 \ + --name galaxy \ + --privileged=true \ + -v "$(pwd)/local_folder:/export/" \ + -e GALAXY_CONFIG_ALLOW_USER_DATASET_PURGE=True \ + -e GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE=True \ + -e GALAXY_CONFIG_ENABLE_USER_DELETION=True \ + -e GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True \ + -v /tmp/:/tmp/ \ + quay.io/bgruening/galaxy + +sleep 30 +docker logs galaxy +# Define start functions +docker_exec() { + cd "$WORKING_DIR" + docker exec galaxy "$@" +} +docker_exec_run() { + cd "$WORKING_DIR" + docker run quay.io/bgruening/galaxy "$@" +} +docker_run() { + cd "$WORKING_DIR" + docker run "$@" +} + +docker ps + +# Test submitting jobs to an external slurm cluster +cd "${WORKING_DIR}/test/slurm/" && bash test.sh && cd "$WORKING_DIR" + +# Test submitting jobs to an external gridengine cluster +# TODO 19.05, need to enable this again! +# - cd $WORKING_DIR/test/gridengine/ && bash test.sh && cd $WORKING_DIR + +echo 'Waiting for Galaxy to come up.' +galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 300 + +curl -v --fail $BIOBLEND_GALAXY_URL/api/version + +# Test self-signed HTTPS +docker_run -d --name httpstest -p 443:443 -e "USE_HTTPS=True" $DOCKER_RUN_CONTAINER +# TODO 19.05 +# - sleep 90s && curl -v -k --fail https://127.0.0.1:443/api/version +#- echo | openssl s_client -connect 127.0.0.1:443 2>/dev/null | openssl x509 -issuer -noout| grep selfsigned + +docker logs httpstest && docker stop httpstest && docker rm httpstest + +# Test FTP Server upload +date > time.txt +# FIXME passive mode does not work, it would require the container to run with --net=host +#curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true +# Test FTP Server get +#curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD + +# Test CVMFS +docker_exec bash -c "service autofs start" +docker_exec bash -c "cvmfs_config chksetup" +docker_exec bash -c "ls /cvmfs/data.galaxyproject.org/byhand" + +# Test SFTP Server +sshpass -p $GALAXY_USER_PASSWD sftp -v -P 8022 -o User=$GALAXY_USER -o "StrictHostKeyChecking no" localhost <<< $'put time.txt' + +# Run a ton of BioBlend test against our servers. +cd "$WORKING_DIR/test/bioblend/" && . ./test.sh && cd "$WORKING_DIR/" + +# not working anymore in 18.01 +# executing: /galaxy_venv/bin/uwsgi --yaml /etc/galaxy/galaxy.yml --master --daemonize2 galaxy.log --pidfile2 galaxy.pid --log-file=galaxy_install.log --pid-file=galaxy_install.pid +# [uWSGI] getting YAML configuration from /etc/galaxy/galaxy.yml +# /galaxy_venv/bin/python: unrecognized option '--log-file=galaxy_install.log' +# getopt_long() error +# cat: galaxy_install.pid: No such file or directory +# tail: cannot open ‘galaxy_install.log’ for reading: No such file or directory +#- | +# if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] +# then +# # Test without install-repository wrapper +# sleep 10 +# docker_exec_run bash -c 'cd $GALAXY_ROOT && python ./scripts/api/install_tool_shed_repositories.py --api admin -l http://localhost:80 --url https://toolshed.g2.bx.psu.edu -o devteam --name cut_columns --panel-section-name BEDTools' +# fi + + +# Test the 'new' tool installation script +docker_exec install-tools "$SAMPLE_TOOLS" +# Test the Conda installation +docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/bin/:$PATH && conda --version && conda install samtools -c bioconda --yes' + + +docker stop galaxy +docker rm -f galaxy +docker rmi -f $DOCKER_RUN_CONTAINER diff --git a/.github/workflows/single_container.yml b/.github/workflows/single_container.yml new file mode 100644 index 000000000..31223bc18 --- /dev/null +++ b/.github/workflows/single_container.yml @@ -0,0 +1,16 @@ +name: Single Container Test +on: [push] +jobs: + build_and_test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7] + steps: + - name: Checkout + uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Build and Test + run: bash .github/workflows/single.sh diff --git a/.travis.yml b/.travis.yml index 694321b4c..4353fad27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,11 @@ before_install: # start building this repo - git submodule update --init --recursive - sudo chown 1450 /tmp && sudo chmod a=rwx /tmp + - export WORKING_DIR="$TRAVIS_BUILD_DIR" + - export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy" + - export INSTALL_REPO_ARG="" + - export SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml + - travis_wait 30 cd "$WORKING_DIR" && docker build -t quay.io/bgruening/galaxy galaxy/ - | ## define a container size check function, first parameter is the container name, second the max allowed size in MB container_size_check () { @@ -55,14 +60,6 @@ before_install: #exit fi } - - export WORKING_DIR="$TRAVIS_BUILD_DIR" - export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy" - INSTALL_REPO_ARG="" - SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml - cd "$WORKING_DIR" - docker build -t quay.io/bgruening/galaxy galaxy/ - container_size_check quay.io/bgruening/galaxy 1500 mkdir local_folder diff --git a/README.md b/README.md index a64c50650..6e23c7b71 100644 --- a/README.md +++ b/README.md @@ -993,6 +993,8 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro - Featuring Galaxy 20.05 - Completely reworked compose setup - The default admin password and apikey (`GALAXY_DEFAULT_ADMIN_PASSWORD` and `GALAXY_DEFAULT_ADMIN_KEY`) have changed: the password is now `password` (instead of `admin`) and the apikey `fakekey` (instead of `admin`). +- 20.09: + - Featuring Galaxy 20.09 # Support & Bug Reports [[toc]](#toc) diff --git a/compose/README.md b/compose/README.md index 3007747d0..d73d39a5d 100644 --- a/compose/README.md +++ b/compose/README.md @@ -151,9 +151,9 @@ All options are discussed under [configuration reference](#configuration-referen ### Use specific Galaxy version or Docker images The `IMAGE_TAG` environment variable allows to use specific versions of the -setup. Say, you want to stay with Galaxy v20.05 for now: +setup. Say, you want to stay with Galaxy v20.09 for now: -> export IMAGE_TAG=20.05 +> export IMAGE_TAG=20.09 > docker-compose up Without setting this variable, you will always get updated to the newest @@ -378,7 +378,7 @@ The following are settings specific to this docker-compose setup: | `GALAXY_K8S_DOCKER_REPO_DEFAULT` | The Docker Repo/Registry to use if the resolver could not resolve the proper image for a job. Defaults to `docker.io`. | | `GALAXY_K8S_DOCKER_OWNER_DEFAULT` | The Owner/Username to use if the resolver could not resolve the proper image for a job. Is not set by default. | | `GALAXY_K8S_DOCKER_IMAGE_DEFAULT` | The Image to use if the resolver could not resolve the proper image for a job. Defaults to `ubuntu`. | -| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `18.04`. | +| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `20.04`. | ### HTCondor | Variable | Description | @@ -393,3 +393,21 @@ The following are settings specific to this docker-compose setup: | `SLURM_NODE_CPUS` | Number of CPUs per node. Defaults to 1. | | `SLURM_NODE_MEMORY` | Amount of memory per node. Defaults to 1024. | | `SLURM_NODE_HOSTNAME` | Docker Compose adds a prefix in front of the container names by default. Change this value to the name of your setup and `_slurm_node` (e.g. `compose_slurm_node`) to ensure a correct mapping of the Slurm nodes. | + +### Github Workflow Tests (Branch 20.09) +| Setup | bioblend | workflow ard | workflow mapping_by_sequencing | workflow wf3-shed-tools (example1) | selenium | +|------------------------|--------------------|--------------------|--------------------------------|------------------------------------|--------------------| +| Galaxy Base | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| Galaxy Proxy Prefix | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| HTCondor | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| Slurm | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :x: | :heavy_check_mark: | +| Pulsar | :heavy_check_mark: | :x: | :x: | :x: | :heavy_check_mark: | +| k8s | :x: | :x: | :x: | :x: | :x: | +| Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | +| Slurm + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | +| HTCondor + Singularity | :x: | :x: | :x: | :heavy_check_mark: | :x: | + + +Implemented: :heavy_check_mark: +Not Implemented: :x: + diff --git a/compose/base-images/galaxy-cluster-base/Dockerfile b/compose/base-images/galaxy-cluster-base/Dockerfile index f9d668009..ff7c91cce 100644 --- a/compose/base-images/galaxy-cluster-base/Dockerfile +++ b/compose/base-images/galaxy-cluster-base/Dockerfile @@ -20,11 +20,11 @@ ENV MUNGER_USER=munge \ MUNGE_GID=1200 RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ && useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \ - && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \ - && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \ + && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \ + && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \ && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8DE68488997C5C6BA19021136F2CC56412788738 \ && apt update \ - && apt install --no-install-recommends slurm-client slurmd slurmctld slurm-drmaa1 -y \ + && apt install --no-install-recommends python3-distutils slurm-client slurmd slurmctld slurm-drmaa1 -y \ && apt --no-install-recommends install munge libmunge-dev -y \ && ln -s /usr/lib/slurm-drmaa/lib/libdrmaa.so.1 /usr/lib/slurm-drmaa/lib/libdrmaa.so \ && /usr/bin/common_cleanup.sh diff --git a/compose/base-images/galaxy-container-base/Dockerfile b/compose/base-images/galaxy-container-base/Dockerfile index 20fa3743f..6cf33a516 100644 --- a/compose/base-images/galaxy-container-base/Dockerfile +++ b/compose/base-images/galaxy-container-base/Dockerfile @@ -1,4 +1,4 @@ -FROM buildpack-deps:18.04 as build_singularity +FROM buildpack-deps:20.04 as build_singularity COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh @@ -21,12 +21,12 @@ RUN wget https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_ # --- Final image --- -FROM ubuntu:18.04 as final +FROM ubuntu:20.04 as final COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Base dependencies -RUN apt update && apt install --no-install-recommends ca-certificates squashfs-tools -y \ +RUN apt update && apt install --no-install-recommends ca-certificates python3-distutils squashfs-tools -y \ && /usr/bin/common_cleanup.sh # Install Docker diff --git a/compose/base_config.yml b/compose/base_config.yml index cc2970901..46f37dc7c 100644 --- a/compose/base_config.yml +++ b/compose/base_config.yml @@ -9,11 +9,10 @@ galaxy_uwsgi: master: false virtualenv: .venv pythonpath: lib - thunder-lock: false + thunder-lock: true die-on-term: true hook-master-start: unix_signal:2 gracefully_kill_them_all hook-master-start: unix_signal:15 gracefully_kill_them_all - py-call-osafterfork: true enable-threads: true galaxy: diff --git a/compose/docker-compose.htcondor.yml b/compose/docker-compose.htcondor.yml index 1b7c36702..35084397f 100644 --- a/compose/docker-compose.htcondor.yml +++ b/compose/docker-compose.htcondor.yml @@ -28,6 +28,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/htcondor:/config - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database + - ${EXPORT_DIR:-./export}/galaxy/lib/galaxy/tools:/galaxy/lib/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv diff --git a/compose/docker-compose.slurm.yml b/compose/docker-compose.slurm.yml index 134f5e06d..4cf6e29cb 100644 --- a/compose/docker-compose.slurm.yml +++ b/compose/docker-compose.slurm.yml @@ -40,6 +40,7 @@ services: volumes: - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro + - ${EXPORT_DIR:-./export}/galaxy/lib/galaxy/tools:/galaxy/lib/galaxy/tools:ro - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv - ${EXPORT_DIR:-./export}/tool_deps:/tool_deps diff --git a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 index 461dfb5ff..9b1ca49ed 100644 --- a/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 +++ b/compose/galaxy-configurator/templates/galaxy/job_conf.xml.j2 @@ -32,6 +32,7 @@ {% if GALAXY_DEPENDENCY_RESOLUTION == 'singularity' -%} + /home/galaxy true {% if GALAXY_JOB_RUNNER == 'local' -%} @@ -59,7 +60,7 @@ {{ GALAXY_K8S_DOCKER_REPO_DEFAULT | default('docker.io') }} {% if GALAXY_K8S_DOCKER_OWNER_DEFAULT -%}{{ GALAXY_K8S_DOCKER_OWNER_DEFAULT }}{% endif -%} {{ GALAXY_K8S_DOCKER_IMAGE_DEFAULT | default('ubuntu') }} - {{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('18.04') }} + {{ GALAXY_K8S_DOCKER_TAG_DEFAULT | default('20.04') }} true {% endif -%} diff --git a/compose/galaxy-htcondor/Dockerfile b/compose/galaxy-htcondor/Dockerfile index d10630bde..ac49a826b 100644 --- a/compose/galaxy-htcondor/Dockerfile +++ b/compose/galaxy-htcondor/Dockerfile @@ -2,9 +2,9 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as galaxy_dependencies +FROM buildpack-deps:20.04 as galaxy_dependencies -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy ENV GALAXY_ROOT=/galaxy @@ -18,7 +18,7 @@ RUN mkdir "${GALAXY_ROOT}" \ && cd $GALAXY_ROOT/lib \ && ls . | grep -v "galaxy\|galaxy_ext" | xargs rm -rf \ && cd $GALAXY_ROOT/lib/galaxy \ - && ls . | grep -v "__init__.py\|datatypes\|exceptions\|metadata\|model\|util\|security" | xargs rm -rf + && ls . | grep -v "__init__.py\|datatypes\|exceptions\|files\|metadata\|model\|util\|security" | xargs rm -rf FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-container-base:$IMAGE_TAG as final @@ -67,4 +67,6 @@ COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT $ COPY start.sh /usr/bin/start.sh RUN apt update && apt install python3 -y +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 + ENTRYPOINT /usr/bin/start.sh diff --git a/compose/galaxy-server/Dockerfile b/compose/galaxy-server/Dockerfile index 2245216ec..d789c2250 100644 --- a/compose/galaxy-server/Dockerfile +++ b/compose/galaxy-server/Dockerfile @@ -2,7 +2,7 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as build_base +FROM buildpack-deps:20.04 as build_base ENV EXPORT_DIR=/export \ GALAXY_ROOT=/galaxy \ @@ -24,7 +24,7 @@ ENV GALAXY_USER=galaxy \ GALAXY_HOME=/home/galaxy ENV GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda \ - MINICONDA_VERSION=4.7.10 + MINICONDA_VERSION=py38_4.9.2 RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ && useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" --shell /bin/bash $GALAXY_USER \ @@ -34,7 +34,7 @@ RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \ FROM build_base as build_miniconda COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Install Miniconda -RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA_VERSION}-Linux-x86_64.sh" > ~/miniconda.sh \ +RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ @@ -50,18 +50,20 @@ RUN curl -s -L "https://repo.anaconda.com/miniconda/Miniconda2-${MINICONDA_VERSI FROM build_base as build_galaxy -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh # Install Galaxy RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libssl-dev python3-dev python3-pip -y \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3 9 \ && mkdir "${GALAXY_ROOT}" \ && curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ && cd $GALAXY_ROOT \ && ./scripts/common_startup.sh \ && . $GALAXY_ROOT/.venv/bin/activate \ && pip3 install drmaa psycopg2 pycurl pykube \ + && pip3 install importlib-metadata importlib-resources pathlib2 ruamel.yaml.clib typing zipp \ && deactivate \ && rm -rf .ci .circleci .coveragerc .gitignore .travis.yml CITATION CODE_OF_CONDUCT.md CONTRIBUTING.md CONTRIBUTORS.md \ LICENSE.txt Makefile README.rst SECURITY_POLICY.md pytest.ini tox.ini \ @@ -73,9 +75,6 @@ RUN apt update && apt install --no-install-recommends libcurl4-openssl-dev libss # --- Final image --- FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-cluster-base:$IMAGE_TAG as final -ARG GALAXY_RELEASE=release_20.05 -ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy - COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh COPY ./files/create_galaxy_user.py /usr/local/bin/create_galaxy_user.py @@ -84,7 +83,9 @@ ENV EXPORT_DIR=/export \ GALAXY_PYTHON=/usr/bin/python3 \ HTCONDOR_ROOT=/opt/htcondor -ENV GALAXY_STATIC_DIR=$GALAXY_ROOT/static \ +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \ + GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ + GALAXY_STATIC_DIR=$GALAXY_ROOT/static \ GALAXY_EXPORT=$EXPORT_DIR/galaxy \ GALAXY_CONFIG_DIR=$GALAXY_ROOT/config \ GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=/tool_deps \ @@ -120,8 +121,7 @@ RUN apt update && apt install --no-install-recommends curl gcc gnupg2 libgomp1 l RUN usermod -aG docker $GALAXY_USER # Make Python3 standard -RUN mv /usr/bin/python /usr/bin/python2 \ - && ln /usr/bin/python3 /usr/bin/python +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_galaxy ${GALAXY_ROOT} ${GALAXY_ROOT} COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} ${GALAXY_CONFIG_TOOL_DEPENDENCY_DIR} diff --git a/compose/galaxy-server/files/start.sh b/compose/galaxy-server/files/start.sh index baf672165..860280856 100755 --- a/compose/galaxy-server/files/start.sh +++ b/compose/galaxy-server/files/start.sh @@ -13,6 +13,14 @@ create_user() { deactivate } +# start copy lib/tools. Looks very hacky. +tools_dir="/galaxy/lib/galaxy/tools/" +exp_dir="/export$tools_dir" +mkdir -p $exp_dir +chown "$GALAXY_USER:$GALAXY_USER" $exp_dir +cp -rf $tools_dir/* $exp_dir +# end copy lib/tools. + # First start?? Check if something exists that indicates that environment is not new.. Config file? Something in DB maybe?? echo "Initialization: Check if files already exist, export otherwise." diff --git a/compose/galaxy-slurm/Dockerfile b/compose/galaxy-slurm/Dockerfile index 2f1328e8c..6f54338b2 100644 --- a/compose/galaxy-slurm/Dockerfile +++ b/compose/galaxy-slurm/Dockerfile @@ -2,9 +2,9 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as galaxy_dependencies +FROM buildpack-deps:20.04 as galaxy_dependencies -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy ENV GALAXY_ROOT=/galaxy @@ -17,7 +17,7 @@ RUN mkdir "${GALAXY_ROOT}" \ && cd $GALAXY_ROOT/lib \ && ls . | grep -v "galaxy\|galaxy_ext" | xargs rm -rf \ && cd $GALAXY_ROOT/lib/galaxy \ - && ls . | grep -v "__init__.py\|datatypes\|exceptions\|metadata\|model\|util\|security" | xargs rm -rf + && ls . | grep -v "__init__.py\|datatypes\|exceptions\|files\|metadata\|model\|util\|security" | xargs rm -rf FROM $DOCKER_REGISTRY/$DOCKER_REGISTRY_USERNAME/galaxy-container-base:$IMAGE_TAG as final @@ -45,11 +45,13 @@ ENV SLURM_USER=galaxy \ RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \ && useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \ && apt update \ - && apt install --no-install-recommends gosu munge python python-dev slurm-wlm -y \ + && apt install --no-install-recommends gosu munge python3 python3-dev slurm-wlm -y \ && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/* && find / -name '*.pyc' -delete # Copy Galaxy dependencies COPY --chown=$GALAXY_USER:$GALAXY_USER --from=galaxy_dependencies $GALAXY_ROOT $GALAXY_ROOT +# Make Python3 standard +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 9 COPY start.sh /usr/bin/start.sh diff --git a/compose/pulsar/Dockerfile b/compose/pulsar/Dockerfile index 2fb2a7d10..dac9b1503 100644 --- a/compose/pulsar/Dockerfile +++ b/compose/pulsar/Dockerfile @@ -2,9 +2,9 @@ ARG DOCKER_REGISTRY=quay.io ARG DOCKER_REGISTRY_USERNAME=bgruening ARG IMAGE_TAG=latest -FROM buildpack-deps:18.04 as build_pulsar +FROM buildpack-deps:20.04 as build_pulsar -ARG PULSAR_RELEASE=0.13.0 +ARG PULSAR_RELEASE=0.14.0 ARG PULSAR_REPO=https://github.com/galaxyproject/pulsar ENV PULSAR_ROOT=/pulsar @@ -34,7 +34,7 @@ ENV PULSAR_VIRTUALENV=$PULSAR_ROOT/.venv \ PULSAR_TOOL_DEPENDENCY_DIR=$PULSAR_ROOT/dependencies RUN apt update \ - && apt install --no-install-recommends ca-certificates curl libxml2-dev libpython3.6 python3 -y \ + && apt install --no-install-recommends ca-certificates curl libxml2-dev python3 -y \ && /usr/bin/common_cleanup.sh COPY --from=build_pulsar /pulsar /pulsar diff --git a/compose/tests/docker-compose.test.bioblend.yml b/compose/tests/docker-compose.test.bioblend.yml index 9ebcaf7b8..6b77fb405 100644 --- a/compose/tests/docker-compose.test.bioblend.yml +++ b/compose/tests/docker-compose.test.bioblend.yml @@ -4,7 +4,7 @@ services: image: ${DOCKER_REGISTRY:-quay.io}/${DOCKER_REGISTRY_USERNAME:-bgruening}/galaxy-bioblend-test:${IMAGE_TAG:-latest} build: tests/galaxy-bioblend-test environment: - - GALAXY_VERSION=${GALAXY_VERSION:-release_20.05} # TODO: Change to GALAXY_RELEASE + - GALAXY_VERSION=${GALAXY_VERSION:-release_20.09} # TODO: Change to GALAXY_RELEASE - GALAXY_URL=http://nginx${GALAXY_PROXY_PREFIX:-}/ networks: - galaxy diff --git a/compose/tests/galaxy-selenium-test/Dockerfile b/compose/tests/galaxy-selenium-test/Dockerfile index f081a029e..370264852 100644 --- a/compose/tests/galaxy-selenium-test/Dockerfile +++ b/compose/tests/galaxy-selenium-test/Dockerfile @@ -1,18 +1,19 @@ FROM selenium/standalone-chrome:3.141.59 -ARG GALAXY_RELEASE=release_20.05 +ARG GALAXY_RELEASE=release_20.09 ARG GALAXY_REPO=https://github.com/galaxyproject/galaxy ENV GALAXY_ROOT=/galaxy -ENV GALAXY_PYTHON=/usr/bin/python +ENV GALAXY_PYTHON=/usr/bin/python3 USER root -RUN apt update && apt install --no-install-recommends python-dev python-pip -y && rm -rf /var/lib/apt/lists/* \ +RUN apt update && apt install --no-install-recommends python3-dev python3-pip -y && rm -rf /var/lib/apt/lists/* \ && mkdir "${GALAXY_ROOT}" \ && chown seluser "${GALAXY_ROOT}" USER seluser -RUN curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ +RUN mkdir -p $GALAXY_ROOT && \ + curl -L -s $GALAXY_REPO/archive/$GALAXY_RELEASE.tar.gz | tar xzf - --strip-components=1 -C $GALAXY_ROOT \ && cd "${GALAXY_ROOT}" \ && ./scripts/common_startup.sh --skip-client-build --dev-wheels diff --git a/compose/tests/galaxy-workflow-test/Dockerfile b/compose/tests/galaxy-workflow-test/Dockerfile index b392babbb..1b9385da9 100644 --- a/compose/tests/galaxy-workflow-test/Dockerfile +++ b/compose/tests/galaxy-workflow-test/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.11 -ENV TEST_REPO=${TEST_REPO:-https://github.com/usegalaxy-eu/workflow-testing} \ - TEST_RELEASE=${TEST_RELEASE:-master} +ENV TEST_REPO=${TEST_REPO:-https://github.com/jonas27/workflow-testing} \ + TEST_RELEASE=${TEST_RELEASE:-20.09-comment-filetype} RUN apk add --no-cache bash python3 curl \ && apk add --no-cache --virtual build-dep gcc libxml2-dev libxslt-dev musl-dev linux-headers python3-dev \ @@ -10,6 +10,9 @@ RUN apk add --no-cache bash python3 curl \ && curl -L -s $TEST_REPO/archive/$TEST_RELEASE.tar.gz | tar xzf - --strip-components=1 \ && apk del build-dep +# Make Python3 standard +RUN ln /usr/bin/python3 /usr/bin/python && ln /usr/bin/python3 /usr/bin/python2 + ADD ./run.sh /usr/bin/run.sh WORKDIR /src diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 2b46ec66c..ffe96856e 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -16,60 +16,60 @@ MAINTAINER Björn A. Grüning, bjoern.gruening@gmail.com ARG GALAXY_RELEASE ARG GALAXY_REPO -ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.05} \ -GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ -GALAXY_ROOT=/galaxy-central \ -GALAXY_CONFIG_DIR=/etc/galaxy \ -EXPORT_DIR=/export \ -DEBIAN_FRONTEND=noninteractive \ -PG_VERSION=11 - +ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_20.09} \ + GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \ + GALAXY_ROOT=/galaxy-central \ + GALAXY_CONFIG_DIR=/etc/galaxy \ + EXPORT_DIR=/export \ + DEBIAN_FRONTEND=noninteractive \ + PG_VERSION=11 ENV GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml \ -GALAXY_CONFIG_JOB_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_conf.xml \ -GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_metrics_conf.xml \ -GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml \ -GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \ -GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=$EXPORT_DIR/tool_deps \ -GALAXY_CONFIG_TOOL_PATH=$EXPORT_DIR/galaxy-central/tools \ -GALAXY_VIRTUAL_ENV=/galaxy_venv \ -GALAXY_USER=galaxy \ -GALAXY_UID=1450 \ -GALAXY_GID=1450 \ -GALAXY_POSTGRES_UID=1550 \ -GALAXY_POSTGRES_GID=1550 \ -GALAXY_HOME=/home/galaxy \ -GALAXY_LOGS_DIR=/home/galaxy/logs \ -GALAXY_DEFAULT_ADMIN_USER=admin \ -GALAXY_DEFAULT_ADMIN_EMAIL=admin@galaxy.org \ -GALAXY_DEFAULT_ADMIN_PASSWORD=password \ -GALAXY_DEFAULT_ADMIN_KEY=fakekey \ -GALAXY_DESTINATIONS_DEFAULT=slurm_cluster \ -GALAXY_RUNNERS_ENABLE_SLURM=True \ -GALAXY_RUNNERS_ENABLE_CONDOR=False \ -GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:galaxy@localhost:5432/galaxy?client_encoding=utf8 \ -GALAXY_CONFIG_ADMIN_USERS=admin@galaxy.org \ -GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3 \ -GALAXY_CONFIG_BRAND="Galaxy Docker Build" \ -GALAXY_CONFIG_STATIC_ENABLED=False \ -# Define the default postgresql database path -PG_DATA_DIR_DEFAULT=/var/lib/postgresql/$PG_VERSION/main/ \ -PG_CONF_DIR_DEFAULT=/etc/postgresql/$PG_VERSION/main/ \ -PG_DATA_DIR_HOST=$EXPORT_DIR/postgresql/$PG_VERSION/main/ \ -# The following 2 ENV vars can be used to set the number of uwsgi processes and threads -UWSGI_PROCESSES=2 \ -UWSGI_THREADS=4 \ -# Set HTTPS to use a self-signed certificate (or your own certificate in $EXPORT_DIR/{server.key,server.crt}) -USE_HTTPS=False \ -# Set USE_HTTPS_LENSENCRYPT and GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL to a domain that is reachable to get a letsencrypt certificate -USE_HTTPS_LETSENCRYPT=False \ -GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost \ -# Set the number of Galaxy handlers -GALAXY_HANDLER_NUMPROCS=2 \ -# Setting a standard encoding. This can get important for things like the unix sort tool. -LC_ALL=en_US.UTF-8 \ -LANG=en_US.UTF-8 \ -GALAXY_CONDA_PREFIX=/tool_deps/_conda + GALAXY_CONFIG_JOB_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_conf.xml \ + GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE=$GALAXY_CONFIG_DIR/job_metrics_conf.xml \ + GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH=/etc/galaxy/tool_data_table_conf.xml \ + GALAXY_CONFIG_WATCH_TOOL_DATA_DIR=True \ + GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=$EXPORT_DIR/tool_deps \ + GALAXY_CONFIG_TOOL_PATH=$EXPORT_DIR/galaxy-central/tools \ + GALAXY_VIRTUAL_ENV=/galaxy_venv \ + GALAXY_USER=galaxy \ + GALAXY_UID=1450 \ + GALAXY_GID=1450 \ + GALAXY_POSTGRES_UID=1550 \ + GALAXY_POSTGRES_GID=1550 \ + GALAXY_HOME=/home/galaxy \ + GALAXY_LOGS_DIR=/home/galaxy/logs \ + GALAXY_DEFAULT_ADMIN_USER=admin \ + GALAXY_DEFAULT_ADMIN_EMAIL=admin@galaxy.org \ + GALAXY_DEFAULT_ADMIN_PASSWORD=password \ + GALAXY_DEFAULT_ADMIN_KEY=fakekey \ + GALAXY_DESTINATIONS_DEFAULT=slurm_cluster \ + GALAXY_RUNNERS_ENABLE_SLURM=True \ + GALAXY_RUNNERS_ENABLE_CONDOR=False \ + GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:galaxy@localhost:5432/galaxy?client_encoding=utf8 \ + GALAXY_CONFIG_ADMIN_USERS=admin@galaxy.org \ + GALAXY_CONFIG_MASTER_API_KEY=HSNiugRFvgT574F43jZ7N9F3 \ + GALAXY_CONFIG_BRAND="Galaxy Docker Build" \ + GALAXY_CONFIG_STATIC_ENABLED=False \ + # Define the default postgresql database path + PG_DATA_DIR_DEFAULT=/var/lib/postgresql/$PG_VERSION/main/ \ + PG_CONF_DIR_DEFAULT=/etc/postgresql/$PG_VERSION/main/ \ + PG_DATA_DIR_HOST=$EXPORT_DIR/postgresql/$PG_VERSION/main/ \ + # The following 2 ENV vars can be used to set the number of uwsgi processes and threads + UWSGI_PROCESSES=2 \ + UWSGI_THREADS=4 \ + # Set HTTPS to use a self-signed certificate (or your own certificate in $EXPORT_DIR/{server.key,server.crt}) + USE_HTTPS=False \ + # Set USE_HTTPS_LENSENCRYPT and GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL to a domain that is reachable to get a letsencrypt certificate + USE_HTTPS_LETSENCRYPT=False \ + GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost \ + # Set the number of Galaxy handlers + GALAXY_HANDLER_NUMPROCS=2 \ + # Setting a standard encoding. This can get important for things like the unix sort tool. + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + NODE_OPTIONS=--max-old-space-size=4096 \ + GALAXY_CONDA_PREFIX=/tool_deps/_conda # 16MB RUN echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup \ @@ -118,7 +118,7 @@ ADD ./bashrc $GALAXY_HOME/.bashrc # Install miniconda, then virtualenv from conda and then # download latest stable release of Galaxy. -RUN curl -s -L https://repo.continuum.io/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh > ~/miniconda.sh \ +RUN curl -s -L https://repo.anaconda.com/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh > ~/miniconda.sh \ && /bin/bash ~/miniconda.sh -b -p $GALAXY_CONDA_PREFIX/ \ && rm ~/miniconda.sh \ && ln -s $GALAXY_CONDA_PREFIX/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ @@ -188,30 +188,35 @@ RUN mkdir -p /shed_tools $EXPORT_DIR/ftp/ \ && chown $GALAXY_USER:$GALAXY_USER $EXPORT_DIR/tool_deps \ && apt update -qq && apt install --no-install-recommends -y ansible \ && ansible-playbook /ansible/provision.yml \ - --extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \ - --extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \ - --extra-vars galaxy_user_name=$GALAXY_USER \ - --extra-vars galaxy_config_file=$GALAXY_CONFIG_FILE \ - --extra-vars galaxy_config_dir=$GALAXY_CONFIG_DIR \ - --extra-vars galaxy_job_conf_path=$GALAXY_CONFIG_JOB_CONFIG_FILE \ - --extra-vars galaxy_job_metrics_conf_path=$GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE \ - --extra-vars supervisor_manage_slurm="" \ - --extra-vars galaxy_extras_config_condor=True \ - --extra-vars galaxy_extras_config_condor_docker=True \ - --extra-vars galaxy_extras_config_rabbitmq=True \ - --extra-vars galaxy_extras_config_cvmfs=True \ - --extra-vars galaxy_extras_config_uwsgi=False \ - --extra-vars proftpd_db_connection=galaxy@galaxy \ - --extra-vars proftpd_files_dir=$EXPORT_DIR/ftp \ - --extra-vars proftpd_use_sftp=True \ - --extra-vars galaxy_extras_docker_legacy=False \ - --extra-vars galaxy_minimum_version=19.01 \ - --extra-vars supervisor_postgres_config_path=$PG_CONF_DIR_DEFAULT/postgresql.conf \ - --extra-vars supervisor_postgres_autostart=false \ - --extra-vars nginx_use_remote_header=True \ - --tags=galaxyextras,cvmfs -c local \ + --extra-vars galaxy_venv_dir=$GALAXY_VIRTUAL_ENV \ + --extra-vars galaxy_log_dir=$GALAXY_LOGS_DIR \ + --extra-vars galaxy_user_name=$GALAXY_USER \ + --extra-vars galaxy_config_file=$GALAXY_CONFIG_FILE \ + --extra-vars galaxy_config_dir=$GALAXY_CONFIG_DIR \ + --extra-vars galaxy_job_conf_path=$GALAXY_CONFIG_JOB_CONFIG_FILE \ + --extra-vars galaxy_job_metrics_conf_path=$GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE \ + --extra-vars supervisor_manage_slurm="" \ + --extra-vars galaxy_extras_config_condor=True \ + --extra-vars galaxy_extras_config_condor_docker=True \ + --extra-vars galaxy_extras_config_rabbitmq=True \ + --extra-vars galaxy_extras_config_cvmfs=True \ + --extra-vars galaxy_extras_config_uwsgi=False \ + --extra-vars proftpd_db_connection=galaxy@galaxy \ + --extra-vars proftpd_files_dir=$EXPORT_DIR/ftp \ + --extra-vars proftpd_use_sftp=True \ + --extra-vars galaxy_extras_docker_legacy=False \ + --extra-vars galaxy_minimum_version=19.01 \ + --extra-vars supervisor_postgres_config_path=$PG_CONF_DIR_DEFAULT/postgresql.conf \ + --extra-vars supervisor_postgres_autostart=false \ + --extra-vars nginx_use_remote_header=True \ + --tags=galaxyextras,cvmfs -c local \ + && . $GALAXY_VIRTUAL_ENV/bin/activate \ + && pip install WeasyPrint \ + && deactivate \ # TODO: no clue why this is needed here again && cd $GALAXY_ROOT && ./scripts/common_startup.sh \ + && cd config && find . -name 'node_modules' -type d -prune -exec rm -rf '{}' + \ + && find . -name '.cache' -type d -prune -exec rm -rf '{}' + \ && cd / \ && rm $PG_DATA_DIR_DEFAULT -rf \ && python /usr/local/bin/setup_postgresql.py --dbuser galaxy --dbpassword galaxy --db-name galaxy --dbpath $PG_DATA_DIR_DEFAULT --dbversion $PG_VERSION \ @@ -238,28 +243,28 @@ WORKDIR $GALAXY_ROOT #RUN export GALAXY=$GALAXY_ROOT && sh ./cron/updateucsc.sh.sample ENV GALAXY_CONFIG_JOB_WORKING_DIRECTORY=$EXPORT_DIR/galaxy-central/database/job_working_directory \ -GALAXY_CONFIG_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \ -GALAXY_CONFIG_NEW_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \ -GALAXY_CONFIG_TEMPLATE_CACHE_PATH=$EXPORT_DIR/galaxy-central/database/compiled_templates \ -GALAXY_CONFIG_CITATION_CACHE_DATA_DIR=$EXPORT_DIR/galaxy-central/database/citations/data \ -GALAXY_CONFIG_CLUSTER_FILES_DIRECTORY=$EXPORT_DIR/galaxy-central/database/pbs \ -GALAXY_CONFIG_FTP_UPLOAD_DIR=$EXPORT_DIR/ftp \ -GALAXY_CONFIG_FTP_UPLOAD_SITE=galaxy.docker.org \ -GALAXY_CONFIG_USE_PBKDF2=False \ -GALAXY_CONFIG_NGINX_X_ACCEL_REDIRECT_BASE=/_x_accel_redirect \ -GALAXY_CONFIG_NGINX_X_ARCHIVE_FILES_BASE=/_x_accel_redirect \ -GALAXY_CONFIG_DYNAMIC_PROXY_MANAGE=False \ -GALAXY_CONFIG_VISUALIZATION_PLUGINS_DIRECTORY=config/plugins/visualizations \ -GALAXY_CONFIG_TRUST_IPYTHON_NOTEBOOK_CONVERSION=True \ -GALAXY_CONFIG_TOOLFORM_UPGRADE=True \ -GALAXY_CONFIG_SANITIZE_ALL_HTML=False \ -GALAXY_CONFIG_TOOLFORM_UPGRADE=True \ -GALAXY_CONFIG_WELCOME_URL=$GALAXY_CONFIG_DIR/web/welcome.html \ -GALAXY_CONFIG_OVERRIDE_DEBUG=False \ -GALAXY_CONFIG_ENABLE_QUOTAS=True \ -# We need to set $HOME for some Tool Shed tools (e.g Perl libs with $HOME/.cpan) -HOME=$GALAXY_HOME \ -GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda + GALAXY_CONFIG_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \ + GALAXY_CONFIG_NEW_FILE_PATH=$EXPORT_DIR/galaxy-central/database/files \ + GALAXY_CONFIG_TEMPLATE_CACHE_PATH=$EXPORT_DIR/galaxy-central/database/compiled_templates \ + GALAXY_CONFIG_CITATION_CACHE_DATA_DIR=$EXPORT_DIR/galaxy-central/database/citations/data \ + GALAXY_CONFIG_CLUSTER_FILES_DIRECTORY=$EXPORT_DIR/galaxy-central/database/pbs \ + GALAXY_CONFIG_FTP_UPLOAD_DIR=$EXPORT_DIR/ftp \ + GALAXY_CONFIG_FTP_UPLOAD_SITE=galaxy.docker.org \ + GALAXY_CONFIG_USE_PBKDF2=False \ + GALAXY_CONFIG_NGINX_X_ACCEL_REDIRECT_BASE=/_x_accel_redirect \ + GALAXY_CONFIG_NGINX_X_ARCHIVE_FILES_BASE=/_x_accel_redirect \ + GALAXY_CONFIG_DYNAMIC_PROXY_MANAGE=False \ + GALAXY_CONFIG_VISUALIZATION_PLUGINS_DIRECTORY=config/plugins/visualizations \ + GALAXY_CONFIG_TRUST_IPYTHON_NOTEBOOK_CONVERSION=True \ + GALAXY_CONFIG_TOOLFORM_UPGRADE=True \ + GALAXY_CONFIG_SANITIZE_ALL_HTML=False \ + GALAXY_CONFIG_TOOLFORM_UPGRADE=True \ + GALAXY_CONFIG_WELCOME_URL=$GALAXY_CONFIG_DIR/web/welcome.html \ + GALAXY_CONFIG_OVERRIDE_DEBUG=False \ + GALAXY_CONFIG_ENABLE_QUOTAS=True \ + # We need to set $HOME for some Tool Shed tools (e.g Perl libs with $HOME/.cpan) + HOME=$GALAXY_HOME \ + GALAXY_CONDA_PREFIX=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda # Container Style ADD GalaxyDocker.png $GALAXY_CONFIG_DIR/web/welcome_image.png @@ -276,10 +281,10 @@ ADD welcome.html $GALAXY_CONFIG_DIR/web/welcome.html # && find $GALAXY_VIRTUAL_ENV -name '*.pyc' -delete \ # && rm -rf /tmp/* $GALAXY_ROOT/client/node_modules/ $GALAXY_VIRTUAL_ENV/src/ /home/galaxy/.cache/ /home/galaxy/.npm - # Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments - #cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \ - #npm install && \ - #rm -rf ~/.cache/ $GALAXY_ROOT/client/node_modules/ +# Install all required Node dependencies. This is required to get proxy support to work for Interactive Environments +#cd $GALAXY_ROOT/lib/galaxy/web/proxy/js && \ +#npm install && \ +#rm -rf ~/.cache/ $GALAXY_ROOT/client/node_modules/ # Switch back to User root USER root @@ -304,6 +309,10 @@ RUN echo "DISCARD_SESSION_KEYRING_ON_STARTUP=False" > /etc/condor/condor_config. ADD https://github.com/krallin/tini/releases/download/v0.18.0/tini /sbin/tini RUN chmod +x /sbin/tini +# https://stackoverflow.com/questions/62250160/uwsgi-runtimeerror-cannot-release-un-acquired-lock +ADD run.sh $GALAXY_ROOT/run.sh +RUN chmod +x $GALAXY_ROOT/run.sh && sed -i 's/py-call-osafterfork.*//g' /etc/galaxy/galaxy.yml + # This needs to happen here and not above, otherwise the Galaxy start # (without running the startup.sh script) will crash because integrated_tool_panel.xml could not be found. ENV GALAXY_CONFIG_INTEGRATED_TOOL_PANEL_CONFIG $EXPORT_DIR/galaxy-central/integrated_tool_panel.xml diff --git a/galaxy/roles/galaxy-postgresql b/galaxy/roles/galaxy-postgresql index f363e42e3..1507fbe5e 160000 --- a/galaxy/roles/galaxy-postgresql +++ b/galaxy/roles/galaxy-postgresql @@ -1 +1 @@ -Subproject commit f363e42e373dce38e8dceb75acb07e770f71b83c +Subproject commit 1507fbe5eda946fc9f4c9aabef4e73a9b11a8315 diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index b9e47789f..82da900da 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit b9e47789f9b2be2a18d3db3d9585893ddefca062 +Subproject commit 82da900da5667a4d1e06ff976487d30ce460a59d diff --git a/galaxy/run.sh b/galaxy/run.sh new file mode 100755 index 000000000..fa54e0f25 --- /dev/null +++ b/galaxy/run.sh @@ -0,0 +1,98 @@ +#!/bin/sh + + +# Usage: ./run.sh +# +# +# Description: This script can be used to start or stop the galaxy +# web application. + +cd "$(dirname "$0")" + +. ./scripts/common_startup_functions.sh + + +# If there is a file that defines a shell environment specific to this +# instance of Galaxy, source the file. +if [ -z "$GALAXY_LOCAL_ENV_FILE" ]; +then + GALAXY_LOCAL_ENV_FILE='./config/local_env.sh' +fi + +if [ -f "$GALAXY_LOCAL_ENV_FILE" ]; +then + . "$GALAXY_LOCAL_ENV_FILE" +fi + +GALAXY_PID=${GALAXY_PID:-galaxy.pid} +GALAXY_LOG=${GALAXY_LOG:-galaxy.log} +PID_FILE=$GALAXY_PID +LOG_FILE=$GALAXY_LOG + +parse_common_args $@ + +run_common_start_up + +setup_python + + +if [ ! -z "$GALAXY_RUN_WITH_TEST_TOOLS" ]; +then + export GALAXY_CONFIG_OVERRIDE_TOOL_CONFIG_FILE="$(pwd)/test/functional/tools/samples_tool_conf.xml" + export GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES="true" + export GALAXY_CONFIG_OVERRIDE_ENABLE_BETA_TOOL_FORMATS="true" + export GALAXY_CONFIG_INTERACTIVETOOLS_ENABLE="true" + export GALAXY_CONFIG_OVERRIDE_WEBHOOKS_DIR="test/functional/webhooks" +fi + +set_galaxy_config_file_var + + +if [ "$INITIALIZE_TOOL_DEPENDENCIES" -eq 1 ]; then + # Install Conda environment if needed. + python ./scripts/manage_tool_dependencies.py init_if_needed +fi + +[ -n "$GALAXY_UWSGI" ] && APP_WEBSERVER='uwsgi' +find_server "${GALAXY_CONFIG_FILE:-none}" galaxy +server_args=`echo $server_args | sed 's/--py-call-osafterfork//g'` + + +if [ "$run_server" = "python" -a -n "$GALAXY_RUN_ALL" ]; then + servers=$(sed -n 's/^\[server:\(.*\)\]/\1/ p' "$GALAXY_CONFIG_FILE" | xargs echo) + if [ -z "$stop_daemon_arg_set" -a -z "$daemon_or_restart_arg_set" ]; then + echo "ERROR: \$GALAXY_RUN_ALL cannot be used without the '--daemon', '--stop-daemon', 'restart', 'start' or 'stop' arguments to run.sh" + exit 1 + fi + for server in $servers; do + echo "Executing: python $server_args --server-name=\"$server\" --pid-file=\"$server.pid\" --log-file=\"$server.log\"" + eval python $server_args --server-name="$server" --pid-file="$server.pid" --log-file="$server.log" + if [ -n "$wait_arg_set" -a -n "$daemon_or_restart_arg_set" ]; then + while true; do + sleep 1 + # Grab the current pid from the pid file and remove any trailing space + if ! current_pid_in_file=$(sed -e 's/[[:space:]]*$//' "$server.pid"); then + echo "A Galaxy process died, interrupting" >&2 + exit 1 + fi + if [ -n "$current_pid_in_file" ]; then + echo "Found PID $current_pid_in_file in '$server.pid', monitoring '$server.log'" + else + echo "No PID found in '$server.pid' yet" + continue + fi + # Search for all pids in the logs and tail for the last one + latest_pid=$(grep '^Starting server in PID [0-9]\+\.$' "$server.log" | sed 's/^Starting server in PID \([0-9]\{1,\}\).$/\1/' | tail -n 1) + # If they're equivalent, then the current pid file agrees with our logs + # and we've succesfully started + [ -n "$latest_pid" ] && [ "$latest_pid" -eq "$current_pid_in_file" ] && break + done + echo + fi + done +else + + echo "Executing: $run_server $server_args" + # args are properly quoted so use eval + eval $run_server $server_args +fi diff --git a/test/slurm/Dockerfile b/test/slurm/Dockerfile index 76495a16d..bf74cf193 100644 --- a/test/slurm/Dockerfile +++ b/test/slurm/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 ENV DEBIAN_FRONTEND noninteractive