From 1986d990ef2598885d6673b83842362fac4b692a Mon Sep 17 00:00:00 2001 From: ggabernet Date: Mon, 5 Jul 2021 22:37:11 +0200 Subject: [PATCH 1/7] aws fulltest via tower --- .../.github/workflows/awsfulltest.yml | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index c033c7756c..47e72fec3e 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -10,11 +10,9 @@ on: {% raw %} env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }} - AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }} - AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }} + TOWER_WORKSPACE_ID: ${{ secrets.TOWER_WORKSPACE_ID }} + TOWER_BEARER_TOKEN: ${{ secrets.TOWER_BEARER_TOKEN }} + TOWER_COMPUTE_ENV: ${{ secrets.TOWER_COMPUTE_ENV }} AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} {% endraw %} @@ -24,22 +22,26 @@ jobs: if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: 3.7 - - name: Install awscli - run: conda install -c conda-forge awscli - - name: Start AWS batch job + - name: Launch workflow via tower # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters - # Then specify `-profile test_full` instead of `-profile test` on the AWS batch command run: | - aws batch submit-job \ - --region eu-west-1 \ - --job-name nf-core-{{ short_name }} \ - --job-queue $AWS_JOB_QUEUE \ - --job-definition $AWS_JOB_DEFINITION \ - --container-overrides '{"command": ["{{ name }}", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' + curl -X POST "https://api.tower.nf/workflow/launch?workspaceId=${TOWER_WORKSPACE_ID}" \ + -H "Accept: application/json" \ + -H "Authorization: Bearer ${TOWER_BEARER_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{ + "launch": { + "id" : "string", + "computeEnvId" : "'"${TOWER_COMPUTE_ENV}"'", + "pipeline" : "{{ name }}", + "workDir" : "s3://'"${AWS_S3_BUCKET}"'/work/{{ short_name }}/commit-'"${GITHUB_SHA}"'", + "paramsText" : "{\"outdir\":\"s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/results-'"${GITHUB_SHA}"'\"}", + "revision" : "'"${GITHUB_SHA}"'", + "configProfiles" : ["test_full","aws_tower"], + "resume" : false, + "dateCreated" : "1970-01-01T00:00:00.000Z" + } + }' \ + --fail From 1f9d1ab343119c97169dcaae27dfae887b5602e8 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Mon, 5 Jul 2021 22:40:42 +0200 Subject: [PATCH 2/7] aws test via tower --- .../.github/workflows/awstest.yml | 53 ++++++++++--------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 38cb57d086..5edb0614af 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -1,18 +1,16 @@ -name: nf-core AWS test -# This workflow is triggered on push to the master branch. +name: nf-core AWS tests +# This workflow is triggered only on manual dispatch. # It can be additionally triggered manually with GitHub actions workflow dispatch. -# It runs the -profile 'test' on AWS batch. +# It runs the -profile 'test' on AWS batch on: workflow_dispatch: {% raw %} env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }} - AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }} - AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }} + TOWER_WORKSPACE_ID: ${{ secrets.TOWER_WORKSPACE_ID }} + TOWER_BEARER_TOKEN: ${{ secrets.TOWER_BEARER_TOKEN }} + TOWER_COMPUTE_ENV: ${{ secrets.TOWER_COMPUTE_ENV }} AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} {% endraw %} @@ -22,21 +20,26 @@ jobs: if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: 3.7 - - name: Install awscli - run: conda install -c conda-forge awscli - - name: Start AWS batch job - # TODO nf-core: You can customise CI pipeline run tests as required - # For example: adding multiple test runs with different parameters - # Remember that you can parallelise this by using strategy.matrix + - name: Launch workflow via tower + # TODO nf-core: You can customise AWS full pipeline tests as required + # Add full size test data (but still relatively small datasets for few samples) + # on the `test_full.config` test runs with only one set of parameters run: | - aws batch submit-job \ - --region eu-west-1 \ - --job-name nf-core-{{ short_name }} \ - --job-queue $AWS_JOB_QUEUE \ - --job-definition $AWS_JOB_DEFINITION \ - --container-overrides '{"command": ["{{ name }}", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}' + curl -X POST "https://api.tower.nf/workflow/launch?workspaceId=${TOWER_WORKSPACE_ID}" \ + -H "Accept: application/json" \ + -H "Authorization: Bearer ${TOWER_BEARER_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{ + "launch": { + "id" : "string", + "computeEnvId" : "'"${TOWER_COMPUTE_ENV}"'", + "pipeline" : "{{ name }}", + "workDir" : "s3://'"${AWS_S3_BUCKET}"'/work/{{ short_name }}/commit-'"${GITHUB_SHA}"'", + "paramsText" : "{\"outdir\":\"s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/results-'"${GITHUB_SHA}"'\"}", + "revision" : "'"${GITHUB_SHA}"'", + "configProfiles" : ["test","aws_tower"], + "resume" : false, + "dateCreated" : "1970-01-01T00:00:00.000Z" + } + }' \ + --fail From ed88861d130bd193e6a74e61caaf07fe135a6be1 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Mon, 5 Jul 2021 22:50:23 +0200 Subject: [PATCH 3/7] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9f055c477..7f00571332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template * Move TODO item of `contains:` map in a YAML string [[#1082](https://github.com/nf-core/tools/issues/1082)] +* Trigger AWS tests via Tower API [[#1160](https://github.com/nf-core/tools/pull/1160)] ### General From 20efacc410d513f720c03fca5445df4b347a4c49 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Tue, 6 Jul 2021 09:37:26 +0200 Subject: [PATCH 4/7] remove unnecessary fields --- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 47e72fec3e..399a20b9db 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -33,15 +33,13 @@ jobs: -H "Content-Type: application/json" \ -d '{ "launch": { - "id" : "string", "computeEnvId" : "'"${TOWER_COMPUTE_ENV}"'", "pipeline" : "{{ name }}", "workDir" : "s3://'"${AWS_S3_BUCKET}"'/work/{{ short_name }}/commit-'"${GITHUB_SHA}"'", "paramsText" : "{\"outdir\":\"s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/results-'"${GITHUB_SHA}"'\"}", "revision" : "'"${GITHUB_SHA}"'", "configProfiles" : ["test_full","aws_tower"], - "resume" : false, - "dateCreated" : "1970-01-01T00:00:00.000Z" + "resume" : false } }' \ --fail From efc134dcbe254171ac7792c24ac3bb16bcd64c61 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Tue, 6 Jul 2021 09:38:09 +0200 Subject: [PATCH 5/7] remove unnecessary fields --- nf_core/pipeline-template/.github/workflows/awstest.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 5edb0614af..c032e3a962 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -31,15 +31,13 @@ jobs: -H "Content-Type: application/json" \ -d '{ "launch": { - "id" : "string", "computeEnvId" : "'"${TOWER_COMPUTE_ENV}"'", "pipeline" : "{{ name }}", "workDir" : "s3://'"${AWS_S3_BUCKET}"'/work/{{ short_name }}/commit-'"${GITHUB_SHA}"'", "paramsText" : "{\"outdir\":\"s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/results-'"${GITHUB_SHA}"'\"}", "revision" : "'"${GITHUB_SHA}"'", "configProfiles" : ["test","aws_tower"], - "resume" : false, - "dateCreated" : "1970-01-01T00:00:00.000Z" + "resume" : false } }' \ --fail From 86e2db09dd6554b80fa4bb1ba4733cbd558ab463 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Thu, 8 Jul 2021 14:21:40 +0200 Subject: [PATCH 6/7] AWS test use tower action --- .../.github/workflows/awsfulltest.yml | 45 ++++++---------- .../.github/workflows/awstest.yml | 53 +++++++------------ 2 files changed, 36 insertions(+), 62 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 399a20b9db..9ec98b29f9 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -1,45 +1,34 @@ name: nf-core AWS full size tests # This workflow is triggered on published releases. -# It can be additionally triggered manually with GitHub actions workflow dispatch. +# It can be additionally triggered manually with GitHub actions workflow dispatch button. # It runs the -profile 'test_full' on AWS batch on: release: types: [published] workflow_dispatch: - -{% raw %} -env: - TOWER_WORKSPACE_ID: ${{ secrets.TOWER_WORKSPACE_ID }} - TOWER_BEARER_TOKEN: ${{ secrets.TOWER_BEARER_TOKEN }} - TOWER_COMPUTE_ENV: ${{ secrets.TOWER_COMPUTE_ENV }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} -{% endraw %} - jobs: - run-awstest: + run-tower: name: Run AWS full tests if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: - name: Launch workflow via tower + uses: nf-core/tower-action@master # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters - run: | - curl -X POST "https://api.tower.nf/workflow/launch?workspaceId=${TOWER_WORKSPACE_ID}" \ - -H "Accept: application/json" \ - -H "Authorization: Bearer ${TOWER_BEARER_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{ - "launch": { - "computeEnvId" : "'"${TOWER_COMPUTE_ENV}"'", - "pipeline" : "{{ name }}", - "workDir" : "s3://'"${AWS_S3_BUCKET}"'/work/{{ short_name }}/commit-'"${GITHUB_SHA}"'", - "paramsText" : "{\"outdir\":\"s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/results-'"${GITHUB_SHA}"'\"}", - "revision" : "'"${GITHUB_SHA}"'", - "configProfiles" : ["test_full","aws_tower"], - "resume" : false - } - }' \ - --fail + {% raw %} + with: + workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} + bearer_token: ${{ secrets.TOWER_BEARER_TOKEN }} + compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + pipeline: ${{ github.repository }} + revision: ${{ github.sha }} + workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} + parameters: | + { + "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" + } + profiles: '[ "test_full", "aws_tower" ]' + diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index c032e3a962..2921e792e2 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -1,43 +1,28 @@ -name: nf-core AWS tests -# This workflow is triggered only on manual dispatch. -# It can be additionally triggered manually with GitHub actions workflow dispatch. +name: nf-core AWS test +# This workflow can be triggered manually with the GitHub actions workflow dispatch button. # It runs the -profile 'test' on AWS batch on: workflow_dispatch: - -{% raw %} -env: - TOWER_WORKSPACE_ID: ${{ secrets.TOWER_WORKSPACE_ID }} - TOWER_BEARER_TOKEN: ${{ secrets.TOWER_BEARER_TOKEN }} - TOWER_COMPUTE_ENV: ${{ secrets.TOWER_COMPUTE_ENV }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} -{% endraw %} - jobs: - run-awstest: - name: Run AWS tests + run-tower: + name: Run AWS full tests if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: - name: Launch workflow via tower - # TODO nf-core: You can customise AWS full pipeline tests as required - # Add full size test data (but still relatively small datasets for few samples) - # on the `test_full.config` test runs with only one set of parameters - run: | - curl -X POST "https://api.tower.nf/workflow/launch?workspaceId=${TOWER_WORKSPACE_ID}" \ - -H "Accept: application/json" \ - -H "Authorization: Bearer ${TOWER_BEARER_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{ - "launch": { - "computeEnvId" : "'"${TOWER_COMPUTE_ENV}"'", - "pipeline" : "{{ name }}", - "workDir" : "s3://'"${AWS_S3_BUCKET}"'/work/{{ short_name }}/commit-'"${GITHUB_SHA}"'", - "paramsText" : "{\"outdir\":\"s3://'"${AWS_S3_BUCKET}"'/{{ short_name }}/results-'"${GITHUB_SHA}"'\"}", - "revision" : "'"${GITHUB_SHA}"'", - "configProfiles" : ["test","aws_tower"], - "resume" : false - } - }' \ - --fail + uses: nf-core/tower-action@master + {% raw %} + with: + workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} + bearer_token: ${{ secrets.TOWER_BEARER_TOKEN }} + compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + pipeline: ${{ github.repository }} + revision: ${{ github.sha }} + workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} + parameters: | + { + "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" + } + profiles: '[ "test", "aws_tower" ]' + From 46a121c03d4b7b6102816acf3ec06ad0a10e2841 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Fri, 9 Jul 2021 08:47:24 +0200 Subject: [PATCH 7/7] fix AWS wf name --- nf_core/pipeline-template/.github/workflows/awstest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 2921e792e2..fff75db64c 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: run-tower: - name: Run AWS full tests + name: Run AWS tests if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: