20.09

.github/workflows/compose.yml

@@ -2,7 +2,7 @@ name: build-and-test
 on: [push]
 jobs:
   build_container_base:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -120,7 +120,7 @@ jobs:
             --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \
             --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \
             --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \
-            --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy \
+            --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy \
             ${{ matrix.image.subdir }}${{ matrix.image.name }} && break || echo "Fail.. Retrying"
           done;
         shell: bash
@@ -164,6 +164,7 @@ jobs:
             files: -f docker-compose.yml -f docker-compose.k8s.yml
             exclude_test:
               - bioblend
+              - workflow_example1
               - workflow_ard
               - workflow_mapping_by_sequencing
               - selenium
@@ -232,13 +233,13 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Set image tag in env
-        run: echo "::set-env name=IMAGE_TAG::${GITHUB_REF#refs/heads/}"
+        run: echo "IMAGE_TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
       - name: Master branch - Set image to to 'latest'
         if: github.ref == 'refs/heads/master'
-        run: echo "::set-env name=IMAGE_TAG::latest"
+        run: echo "IMAGE_TAG=latest" >> $GITHUB_ENV
       - name: Set WORKFLOWS env for worfklows-test
         if: matrix.test.workflow
-        run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}"
+        run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV
       - name: Run tests for the first time
         if: steps.run_check.outputs.run
         run: |

.github/workflows/lint.yml

@@ -16,7 +16,7 @@ jobs:
         level: warning
         pattern: "*.sh"
     - name: Run hadolint with reviewdog
-      uses: reviewdog/action-hadolint@v1
+      uses: reviewdog/action-hadolint@v1.16.0
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         reporter: github-check

.github/workflows/pull-request.yml

@@ -2,7 +2,7 @@ name: pr-test
 on: pull_request
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         infrastructure:
@@ -30,6 +30,7 @@ jobs:
             exclude_test:
               - workflow_example1
               - workflow_mapping_by_sequencing
+              - workflow_ard
           # - name: galaxy-pulsar-mq
           #   files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml
           #   exclude_test:
@@ -41,6 +42,7 @@ jobs:
               - bioblend
               - workflow_ard
               - workflow_mapping_by_sequencing
+              - workflow_example1
               - selenium
           - name: galaxy-singularity
             files: -f docker-compose.yml -f docker-compose.singularity.yml
@@ -108,7 +110,7 @@ jobs:
         uses: actions/checkout@v2
       - name: Set WORKFLOWS env for worfklows-test
         if: matrix.test.workflow
-        run: echo "::set-env name=WORKFLOWS::${{ matrix.test.workflow }}"
+        run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV
       - name: Build galaxy-container-base
         env:
           image_name: galaxy-container-base
@@ -142,7 +144,7 @@ jobs:
             echo "Removing export directory if existent";
             sudo rm -rf export
             set +e
-            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy
+            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} build --build-arg IMAGE_TAG=ci-testing --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy
             docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }}
             test_exit_code=$?
             error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1)

.github/workflows/single.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+set -ex
+
+export GALAXY_HOME=/home/galaxy
+export GALAXY_USER=admin@galaxy.org
+export GALAXY_USER_EMAIL=admin@galaxy.org
+export GALAXY_USER_PASSWD=password
+export BIOBLEND_GALAXY_API_KEY=fakekey
+export BIOBLEND_GALAXY_URL=http://localhost:8080
+
+sudo apt-get update -qq
+#sudo apt-get install docker-ce --no-install-recommends -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew"
+sudo apt-get install sshpass --no-install-recommends -y
+
+pip3 install ephemeris
+
+docker --version
+docker info
+
+# start building this repo
+git submodule update --init --recursive
+sudo chown 1450 /tmp && sudo chmod a=rwx /tmp
+
+## define a container size check function, first parameter is the container name, second the max allowed size in MB
+container_size_check () {
+
+    # check that the image size is not growing too much between releases
+    # the 19.05 monolithic image was around 1.500 MB
+    size="${docker image inspect $1 --format='{{.Size}}'}"
+    size_in_mb=$(($size/(1024*1024)))
+    if [[ $size_in_mb -ge $2 ]]
+    then
+        echo "The new compiled image ($1) is larger than allowed. $size_in_mb vs. $2"
+        sleep 2
+        #exit
+    fi
+}
+
+export WORKING_DIR=${GITHUB_WORKSPACE:-$PWD}
+
+export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy"
+SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml
+cd "$WORKING_DIR"
+docker build -t quay.io/bgruening/galaxy galaxy/
+#container_size_check   quay.io/bgruening/galaxy  1500
+
+mkdir local_folder
+docker run -d -p 8080:80 -p 8021:21 -p 8022:22 \
+    --name galaxy \
+    --privileged=true \
+    -v "$(pwd)/local_folder:/export/" \
+    -e GALAXY_CONFIG_ALLOW_USER_DATASET_PURGE=True \
+    -e GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE=True \
+    -e GALAXY_CONFIG_ENABLE_USER_DELETION=True \
+    -e GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True \
+    -v /tmp/:/tmp/ \
+    quay.io/bgruening/galaxy
+
+sleep 30
+docker logs galaxy
+# Define start functions
+docker_exec() {
+      cd "$WORKING_DIR"
+      docker exec galaxy "$@"
+}
+docker_exec_run() {
+   cd "$WORKING_DIR"
+   docker run quay.io/bgruening/galaxy "$@"
+}
+docker_run() {
+   cd "$WORKING_DIR"
+   docker run "$@"
+}
+
+docker ps
+
+# Test submitting jobs to an external slurm cluster
+cd "${WORKING_DIR}/test/slurm/" && bash test.sh && cd "$WORKING_DIR"
+
+# Test submitting jobs to an external gridengine cluster
+# TODO 19.05, need to enable this again!
+# - cd $WORKING_DIR/test/gridengine/ && bash test.sh && cd $WORKING_DIR
+
+echo 'Waiting for Galaxy to come up.'
+galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 300
+
+curl -v --fail $BIOBLEND_GALAXY_URL/api/version
+
+# Test self-signed HTTPS
+docker_run -d --name httpstest -p 443:443 -e "USE_HTTPS=True" $DOCKER_RUN_CONTAINER
+# TODO 19.05
+# - sleep 90s && curl -v -k --fail https://127.0.0.1:443/api/version
+#- echo | openssl s_client -connect 127.0.0.1:443 2>/dev/null | openssl x509 -issuer -noout| grep selfsigned
+
+docker logs httpstest && docker stop httpstest && docker rm httpstest
+
+# Test FTP Server upload
+date > time.txt
+# FIXME passive mode does not work, it would require the container to run with --net=host
+#curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true
+# Test FTP Server get
+#curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD
+
+# Test CVMFS
+docker_exec bash -c "service autofs start"
+docker_exec bash -c "cvmfs_config chksetup"
+docker_exec bash -c "ls /cvmfs/data.galaxyproject.org/byhand"
+
+# Test SFTP Server
+sshpass -p $GALAXY_USER_PASSWD sftp -v -P 8022 -o User=$GALAXY_USER -o "StrictHostKeyChecking no" localhost <<< $'put time.txt'
+
+# Run a ton of BioBlend test against our servers.
+cd "$WORKING_DIR/test/bioblend/" && . ./test.sh && cd "$WORKING_DIR/"
+
+# not working anymore in 18.01
+# executing: /galaxy_venv/bin/uwsgi --yaml /etc/galaxy/galaxy.yml --master --daemonize2 galaxy.log --pidfile2 galaxy.pid  --log-file=galaxy_install.log --pid-file=galaxy_install.pid
+# [uWSGI] getting YAML configuration from /etc/galaxy/galaxy.yml
+# /galaxy_venv/bin/python: unrecognized option '--log-file=galaxy_install.log'
+# getopt_long() error
+# cat: galaxy_install.pid: No such file or directory
+# tail: cannot open ‘galaxy_install.log’ for reading: No such file or directory
+#- |
+#  if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ]
+#  then
+#    # Test without install-repository wrapper
+#      sleep 10
+#      docker_exec_run bash -c 'cd $GALAXY_ROOT && python ./scripts/api/install_tool_shed_repositories.py --api admin -l http://localhost:80 --url https://toolshed.g2.bx.psu.edu -o devteam --name cut_columns --panel-section-name BEDTools'
+#  fi
+
+
+# Test the 'new' tool installation script
+docker_exec install-tools "$SAMPLE_TOOLS"
+# Test the Conda installation
+docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/bin/:$PATH && conda --version && conda install samtools -c bioconda --yes'
+
+
+docker stop galaxy
+docker rm -f galaxy
+docker rmi -f $DOCKER_RUN_CONTAINER

.github/workflows/single_container.yml

@@ -0,0 +1,16 @@
+name: Single Container Test
+on: [push]
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Build and Test
+      run: bash .github/workflows/single.sh

.travis.yml

@@ -40,6 +40,11 @@ before_install:
   # start building this repo
   - git submodule update --init --recursive
   - sudo chown 1450 /tmp && sudo chmod a=rwx /tmp
+  - export WORKING_DIR="$TRAVIS_BUILD_DIR"
+  - export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy"
+  - export INSTALL_REPO_ARG=""
+  - export SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml
+  - travis_wait 30 cd "$WORKING_DIR" && docker build -t quay.io/bgruening/galaxy galaxy/
   - |
     ## define a container size check function, first parameter is the container name, second the max allowed size in MB
     container_size_check () {
@@ -55,14 +60,6 @@ before_install:
             #exit
         fi
     }
-
-    export WORKING_DIR="$TRAVIS_BUILD_DIR"
-    export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy"
-    INSTALL_REPO_ARG=""
-    SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml
-    cd "$WORKING_DIR"
-    docker build -t quay.io/bgruening/galaxy galaxy/
-
     container_size_check   quay.io/bgruening/galaxy  1500
 
     mkdir local_folder

README.md

@@ -993,6 +993,8 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro
    - Featuring Galaxy 20.05
    - Completely reworked compose setup
    - The default admin password and apikey (`GALAXY_DEFAULT_ADMIN_PASSWORD` and `GALAXY_DEFAULT_ADMIN_KEY`) have changed: the password is now `password` (instead of `admin`) and the apikey `fakekey` (instead of `admin`).
+- 20.09:
+   - Featuring Galaxy 20.09
 
 # Support & Bug Reports <a name="Support-Bug-Reports" /> [[toc]](#toc)
 

compose/README.md

@@ -151,9 +151,9 @@ All options are discussed under [configuration reference](#configuration-referen
 
 ### Use specific Galaxy version or Docker images
 The `IMAGE_TAG` environment variable allows to use specific versions of the
-setup. Say, you want to stay with Galaxy v20.05 for now:
+setup. Say, you want to stay with Galaxy v20.09 for now:
 
-> export IMAGE_TAG=20.05
+> export IMAGE_TAG=20.09
 > docker-compose up
 
 Without setting this variable, you will always get updated to the newest
@@ -378,7 +378,7 @@ The following are settings specific to this docker-compose setup:
 | `GALAXY_K8S_DOCKER_REPO_DEFAULT` | The Docker Repo/Registry to use if the resolver could not resolve the proper image for a job. Defaults to `docker.io`. |
 | `GALAXY_K8S_DOCKER_OWNER_DEFAULT` | The Owner/Username to use if the resolver could not resolve the proper image for a job. Is not set by default. |
 | `GALAXY_K8S_DOCKER_IMAGE_DEFAULT` | The Image to use if the resolver could not resolve the proper image for a job. Defaults to `ubuntu`. |
-| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `18.04`. |
+| `GALAXY_K8S_DOCKER_TAG_DEFAULT` | The Image Tag to use if the resolver could not resolve the proper image for a job. Defaults to `20.04`. |
 
 ### HTCondor
 | Variable                    | Description                                                                                                        |
@@ -393,3 +393,21 @@ The following are settings specific to this docker-compose setup:
 | `SLURM_NODE_CPUS`         | Number of CPUs per node. Defaults to 1. |
 | `SLURM_NODE_MEMORY`       | Amount of memory per node. Defaults to 1024. |
 | `SLURM_NODE_HOSTNAME`     | Docker Compose adds a prefix in front of the container names by default. Change this value to the name of your setup and `_slurm_node` (e.g. `compose_slurm_node`) to ensure a correct mapping of the Slurm nodes. |
+
+### Github Workflow Tests (Branch 20.09)
+| Setup                  | bioblend           | workflow ard       | workflow mapping_by_sequencing | workflow wf3-shed-tools (example1) | selenium           |
+|------------------------|--------------------|--------------------|--------------------------------|------------------------------------|--------------------|
+| Galaxy Base            | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:             | :x:                                | :heavy_check_mark: |
+| Galaxy Proxy Prefix    | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:             | :x:                                | :heavy_check_mark: |
+| HTCondor               | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:             | :x:                                | :heavy_check_mark: |
+| Slurm                  | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:             | :x:                                | :heavy_check_mark: |
+| Pulsar                 | :heavy_check_mark: | :x:                | :x:                            | :x:                                | :heavy_check_mark: |
+| k8s                    | :x:                | :x:                | :x:                            | :x:                                | :x:                |
+| Singularity            | :x:                | :x:                | :x:                            | :heavy_check_mark:                 | :x:                |
+| Slurm + Singularity    | :x:                | :x:                | :x:                            | :heavy_check_mark:                 | :x:                |
+| HTCondor + Singularity | :x:                | :x:                | :x:                            | :heavy_check_mark:                 | :x:                |
+
+
+Implemented: :heavy_check_mark:   
+Not Implemented: :x:
+

compose/base-images/galaxy-cluster-base/Dockerfile

@@ -20,11 +20,11 @@ ENV MUNGER_USER=munge \
     MUNGE_GID=1200
 RUN groupadd -r $MUNGER_USER -g $MUNGE_GID \
     && useradd -u $MUNGE_UID -r -g $MUNGER_USER $MUNGER_USER \
-    && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \
-    && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu bionic main" >> /etc/apt/sources.list \
+    && echo "deb http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \
+    && echo "deb-src http://ppa.launchpad.net/natefoo/slurm-drmaa/ubuntu focal main" >> /etc/apt/sources.list \
     && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8DE68488997C5C6BA19021136F2CC56412788738 \
     && apt update \
-    && apt install --no-install-recommends slurm-client slurmd slurmctld slurm-drmaa1 -y \
+    && apt install --no-install-recommends python3-distutils slurm-client slurmd slurmctld slurm-drmaa1 -y \
     && apt --no-install-recommends install munge libmunge-dev -y \
     && ln -s /usr/lib/slurm-drmaa/lib/libdrmaa.so.1 /usr/lib/slurm-drmaa/lib/libdrmaa.so \
     && /usr/bin/common_cleanup.sh

compose/base-images/galaxy-container-base/Dockerfile

@@ -1,4 +1,4 @@
-FROM buildpack-deps:18.04  as build_singularity
+FROM buildpack-deps:20.04  as build_singularity
 
 COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
 
@@ -21,12 +21,12 @@ RUN wget https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_
 
 
 # --- Final image ---
-FROM ubuntu:18.04 as final
+FROM ubuntu:20.04 as final
 
 COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
 
 # Base dependencies
-RUN apt update && apt install --no-install-recommends ca-certificates squashfs-tools -y \
+RUN apt update && apt install --no-install-recommends ca-certificates python3-distutils squashfs-tools -y \
     && /usr/bin/common_cleanup.sh
 
 # Install Docker

compose/base_config.yml

@@ -9,11 +9,10 @@ galaxy_uwsgi:
   master: false
   virtualenv: .venv
   pythonpath: lib
-  thunder-lock: false
+  thunder-lock: true
   die-on-term: true
   hook-master-start: unix_signal:2 gracefully_kill_them_all
   hook-master-start: unix_signal:15 gracefully_kill_them_all
-  py-call-osafterfork: true
   enable-threads: true
 
 galaxy:

compose/docker-compose.htcondor.yml

@@ -28,6 +28,7 @@ services:
     volumes:
       - ${EXPORT_DIR:-./export}/htcondor:/config
       - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database
+      - ${EXPORT_DIR:-./export}/galaxy/lib/galaxy/tools:/galaxy/lib/galaxy/tools:ro
       - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro
       - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data
       - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv

compose/docker-compose.slurm.yml

@@ -40,6 +40,7 @@ services:
     volumes:
       - ${EXPORT_DIR:-./export}/galaxy/database:/galaxy/database
       - ${EXPORT_DIR:-./export}/galaxy/tools:/galaxy/tools:ro
+      - ${EXPORT_DIR:-./export}/galaxy/lib/galaxy/tools:/galaxy/lib/galaxy/tools:ro
       - ${EXPORT_DIR:-./export}/galaxy/tool-data:/galaxy/tool-data
       - ${EXPORT_DIR:-./export}/galaxy/.venv:/galaxy/.venv
       - ${EXPORT_DIR:-./export}/tool_deps:/tool_deps