From cc3aa80f13d9ef4919652b8a163ae7f8c321d12e Mon Sep 17 00:00:00 2001 From: "Huska, Matthew" Date: Tue, 21 May 2024 14:36:21 +0200 Subject: [PATCH 1/9] Set default branch to 'main' instead of the Nextflow default 'master' --- nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nextflow.config b/nextflow.config index d410891..694092f 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,6 +1,7 @@ manifest { mainScript = 'clean.nf' nextflowVersion = '>=21.04.0' + defaultBranch = 'main' } // default parameters From 6e2c0183c524a1e7e6b1501843a95d53ff685549 Mon Sep 17 00:00:00 2001 From: "Huska, Matthew" Date: Wed, 22 May 2024 11:35:37 +0200 Subject: [PATCH 2/9] Bump action versions for node 16 -> 20 change, tweak conda clean command based on forums, bump singularity action. Closes #91 --- .github/workflows/dryrun.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dryrun.yml b/.github/workflows/dryrun.yml index 681521f..53c7279 100644 --- a/.github/workflows/dryrun.yml +++ b/.github/workflows/dryrun.yml @@ -30,9 +30,9 @@ jobs: - "3.9" steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: /usr/local/bin/nextflow key: ${{ runner.os }} @@ -40,7 +40,7 @@ jobs: ${{ runner.os }}-nextflow- - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: "${{ matrix.NXF_VER }}" @@ -51,7 +51,7 @@ jobs: - name: Set up Singularity if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-singularity@v5 + uses: eWaterCycle/setup-singularity@v7 with: singularity-version: 3.7.1 @@ -66,7 +66,7 @@ jobs: - name: Conda clean if: matrix.profile == 'conda' - run: conda clean -a + run: conda clean --all -y -f - name: Run nf-test run: nf-test test --profile=${{ matrix.profile }} tests/${{ matrix.inputtype }}/*.nf.test --tap=test.tap From a2cf50812bd7d64b42894277069989e821d850e4 Mon Sep 17 00:00:00 2001 From: "Huska, Matthew" Date: Fri, 7 Jun 2024 13:05:58 +0200 Subject: [PATCH 3/9] Remove 'conda clean' from GitHub action to avoid random crashes --- .github/workflows/dryrun.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/dryrun.yml b/.github/workflows/dryrun.yml index 681521f..d23ad24 100644 --- a/.github/workflows/dryrun.yml +++ b/.github/workflows/dryrun.yml @@ -64,10 +64,6 @@ jobs: channels: conda-forge,bioconda,defaults python-version: ${{ matrix.python-version }} - - name: Conda clean - if: matrix.profile == 'conda' - run: conda clean -a - - name: Run nf-test run: nf-test test --profile=${{ matrix.profile }} tests/${{ matrix.inputtype }}/*.nf.test --tap=test.tap From 8bff893e42e03a580016fed3bddd11cd7cab1efe Mon Sep 17 00:00:00 2001 From: hoelzer Date: Tue, 6 Aug 2024 17:17:32 +0200 Subject: [PATCH 4/9] add T2T v2.0 human genome as download option --- README.md | 3 ++- clean.nf | 3 ++- modules/prepare_contamination.nf | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee45c64..eae5272 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Currently supported are: |flag | species | source| |-----|---------|-------| |hsa | _Homo sapiens_ | [Ensembl: Homo_sapiens.GRCh38.dna.primary_assembly] | +|t2t | _Homo sapiens_ | [[T2T Consortium](https://sites.google.com/ucsc.edu/t2tworkinggroup/): T2T-CHM13v2.0 (file: chm13v2.0.fa.gz), datasets released along the v2.0 and the T2T-Y chromosome, see [paper](https://doi.org/10.1101/2022.12.01.518724)] | |mmu | _Mus musculus_ | [Ensembl: Mus_musculus.GRCm38.dna.primary_assembly] | |csa | _Chlorocebus sabeus_ | [NCBI: GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic] | |gga | _Gallus gallus_ | [NCBI: Gallus_gallus.GRCg6a.dna.toplevel] | @@ -101,7 +102,7 @@ Included in this repository are: | eno | ONT RNA-Seq reads |yeast ENO2 Enolase II of strain S288C, YHR174W| https://raw.githubusercontent.com/rki-mf1/clean/master/controls/S288C_YHR174W_ENO2_coding.fsa | | phix| Illumina reads |enterobacteria_phage_phix174_sensu_lato_uid14015, NC_001422| ftp://ftp.ncbi.nlm.nih.gov/genomes/Viruses/enterobacteria_phage_phix174_sensu_lato_uid14015/NC_001422.fna | -... for reasons. More can be easily added! Just write me, add an issue or make a pull request. +... for reasons. More can be easily added! Just write us, add an issue or make a pull request. ## Workflow diff --git a/clean.nf b/clean.nf index 8a38258..0b5ac4c 100755 --- a/clean.nf +++ b/clean.nf @@ -79,7 +79,7 @@ if ( workflow.profile.contains('singularity') ) { } Set controls = ['phix', 'dcs', 'eno'] -Set hosts = ['hsa', 'mmu', 'cli', 'csa', 'gga', 'eco', 'sc2'] +Set hosts = ['hsa', 'mmu', 'cli', 'csa', 'gga', 'eco', 'sc2', 't2t'] Set input_types = ['nano', 'illumina', 'illumina_single_end', 'fasta'] if ( params.profile ) { exit 1, "--profile is wrong, use -profile" } @@ -254,6 +254,7 @@ def helpMSG() { ${c_green}--host${c_reset} Comma separated list of reference genomes for decontamination, downloaded based on this parameter [default: $params.host] ${c_dim}Currently supported are: - hsa [Ensembl: Homo_sapiens.GRCh38.dna.primary_assembly] + - t2t ["Telomere-to-Telomere" (T2T) Consortium: T2T-CHM13v2.0, datasets released along the v2.0 and the T2T-Y chromosome] - mmu [Ensembl: Mus_musculus.GRCm38.dna.primary_assembly] - csa [NCBI: GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic] - gga [NCBI: Gallus_gallus.GRCg6a.dna.toplevel] diff --git a/modules/prepare_contamination.nf b/modules/prepare_contamination.nf index 678635e..fdbae89 100644 --- a/modules/prepare_contamination.nf +++ b/modules/prepare_contamination.nf @@ -39,6 +39,9 @@ process download_host { wget "https://www.ebi.ac.uk/ena/browser/api/fasta/MN908947.3?download=true" -O host-temp.fa gzip host-temp.fa ;; + t2t) + wget https://s3-us-west-2.amazonaws.com/human-pangenomics/T2T/CHM13/assemblies/analysis_set/chm13v2.0.fa.gz -O host-temp.fa.gz + ;; *) echo "Unknown host ($host)." ;; From 8254386268915041e1dfcd91410d35fff1d754e6 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Tue, 6 Aug 2024 17:24:03 +0200 Subject: [PATCH 5/9] add T2T v2.0 human genome as download option --- README.md | 2 +- clean.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index eae5272..76f51e3 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Currently supported are: |flag | species | source| |-----|---------|-------| |hsa | _Homo sapiens_ | [Ensembl: Homo_sapiens.GRCh38.dna.primary_assembly] | -|t2t | _Homo sapiens_ | [[T2T Consortium](https://sites.google.com/ucsc.edu/t2tworkinggroup/): T2T-CHM13v2.0 (file: chm13v2.0.fa.gz), datasets released along the v2.0 and the T2T-Y chromosome, see [paper](https://doi.org/10.1101/2022.12.01.518724)] | +|t2t | _Homo sapiens_ | [[T2T Consortium](https://sites.google.com/ucsc.edu/t2tworkinggroup/): T2T-CHM13v2.0 (T2T-CHM13+Y, file name: chm13v2.0.fa.gz), datasets released along the v2.0 (T2T-CHM13) and the T2T-Y chromosome, see [paper](https://www.nature.com/articles/s41586-023-06457-y)] | |mmu | _Mus musculus_ | [Ensembl: Mus_musculus.GRCm38.dna.primary_assembly] | |csa | _Chlorocebus sabeus_ | [NCBI: GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic] | |gga | _Gallus gallus_ | [NCBI: Gallus_gallus.GRCg6a.dna.toplevel] | diff --git a/clean.nf b/clean.nf index 0b5ac4c..71f0aa6 100755 --- a/clean.nf +++ b/clean.nf @@ -254,7 +254,7 @@ def helpMSG() { ${c_green}--host${c_reset} Comma separated list of reference genomes for decontamination, downloaded based on this parameter [default: $params.host] ${c_dim}Currently supported are: - hsa [Ensembl: Homo_sapiens.GRCh38.dna.primary_assembly] - - t2t ["Telomere-to-Telomere" (T2T) Consortium: T2T-CHM13v2.0, datasets released along the v2.0 and the T2T-Y chromosome] + - t2t ["Telomere-to-Telomere" (T2T) Consortium: T2T-CHM13v2.0 (T2T-CHM13+Y) human genome with additional 200 Mbp, closed gaps, and more complete Y] - mmu [Ensembl: Mus_musculus.GRCm38.dna.primary_assembly] - csa [NCBI: GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic] - gga [NCBI: Gallus_gallus.GRCg6a.dna.toplevel] From 87a4f6f6559d96331c23cb43c42f9b5d346c93af Mon Sep 17 00:00:00 2001 From: hoelzer Date: Tue, 6 Aug 2024 17:27:27 +0200 Subject: [PATCH 6/9] add T2T v2.0 human genome as download option --- clean.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clean.nf b/clean.nf index 71f0aa6..0076b7f 100755 --- a/clean.nf +++ b/clean.nf @@ -254,7 +254,7 @@ def helpMSG() { ${c_green}--host${c_reset} Comma separated list of reference genomes for decontamination, downloaded based on this parameter [default: $params.host] ${c_dim}Currently supported are: - hsa [Ensembl: Homo_sapiens.GRCh38.dna.primary_assembly] - - t2t ["Telomere-to-Telomere" (T2T) Consortium: T2T-CHM13v2.0 (T2T-CHM13+Y) human genome with additional 200 Mbp, closed gaps, and more complete Y] + - t2t [T2T Consortium: human genome w/ additional 200 Mbp, closed gaps, and more complete Y (T2T-CHM13+Yv2.0)] - mmu [Ensembl: Mus_musculus.GRCm38.dna.primary_assembly] - csa [NCBI: GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic] - gga [NCBI: Gallus_gallus.GRCg6a.dna.toplevel] From bb19fc15cfba29bb022e57c72d8c66fc87078c58 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Wed, 7 Aug 2024 11:25:54 +0200 Subject: [PATCH 7/9] error strategy for fastqc --- configs/node.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/node.config b/configs/node.config index 37ed5c4..f1bdb73 100644 --- a/configs/node.config +++ b/configs/node.config @@ -3,8 +3,9 @@ process { withLabel: bbmap { cpus = 24; memory = 24.GB } withLabel: smallTask { cpus = 1; memory = 2.GB } withLabel: pysam { cpus = 2; memory = 4.GB } - withLabel: fastqc { cpus = 2; memory = 4.GB } + withLabel: fastqc { cpus = {2 * task.attempt}; memory = {4.GB * task.attempt } ; maxRetries = 3 ; errorStrategy = { task.exitStatus in 130..140 ? 'retry' : 'terminate' } } withLabel: multiqc { cpus = 4; memory = 4.GB } withLabel: nanoplot{ cpus = 8; memory = 8.GB } withLabel: quast{ cpus = 8; memory = 8.GB } } + From 610ac75af5bb70f9cabb775a6926406498709227 Mon Sep 17 00:00:00 2001 From: hoelzer Date: Thu, 8 Aug 2024 13:07:26 +0200 Subject: [PATCH 8/9] switch from AWS to NCBI GenBank download, checked all genomes for mtDNA (yes all have) --- README.md | 12 ++++++------ clean.nf | 12 ++++++------ modules/prepare_contamination.nf | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 76f51e3..8cb1eba 100644 --- a/README.md +++ b/README.md @@ -85,12 +85,12 @@ Currently supported are: |flag | species | source| |-----|---------|-------| -|hsa | _Homo sapiens_ | [Ensembl: Homo_sapiens.GRCh38.dna.primary_assembly] | -|t2t | _Homo sapiens_ | [[T2T Consortium](https://sites.google.com/ucsc.edu/t2tworkinggroup/): T2T-CHM13v2.0 (T2T-CHM13+Y, file name: chm13v2.0.fa.gz), datasets released along the v2.0 (T2T-CHM13) and the T2T-Y chromosome, see [paper](https://www.nature.com/articles/s41586-023-06457-y)] | -|mmu | _Mus musculus_ | [Ensembl: Mus_musculus.GRCm38.dna.primary_assembly] | -|csa | _Chlorocebus sabeus_ | [NCBI: GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic] | -|gga | _Gallus gallus_ | [NCBI: Gallus_gallus.GRCg6a.dna.toplevel] | -|cli | _Columba livia_ | [NCBI: GCF_000337935.1_Cliv_1.0_genomic] | +|hsa | _Homo sapiens_ | [Ensembl: Homo_sapiens.GRCh38.dna.primary_assembly, incl. mtDNA] | +|t2t | _Homo sapiens_ | [[T2T Consortium](https://sites.google.com/ucsc.edu/t2tworkinggroup/): T2T-CHM13v2.0 (T2T-CHM13+Y, file name: GCA_009914755.4_T2T-CHM13v2.0_genomic), datasets released along the v2.0 (T2T-CHM13) and the T2T-Y chromosome, see [paper](https://www.nature.com/articles/s41586-023-06457-y), incl. mtDNA] | +|mmu | _Mus musculus_ | [Ensembl: Mus_musculus.GRCm38.dna.primary_assembly, incl. mtDNA] | +|csa | _Chlorocebus sabeus_ | [NCBI: GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic, incl. mtDNA] | +|gga | _Gallus gallus_ | [NCBI: Gallus_gallus.GRCg6a.dna.toplevel, incl. mtDNA] | +|cli | _Columba livia_ | [NCBI: GCF_000337935.1_Cliv_1.0_genomic, incl. mtDNA] | |eco | _Escherichia coli_ | [Ensembl: Escherichia_coli_k_12.ASM80076v1.dna.toplevel] | |sc2 | _SARS-CoV-2_ | [ENA Sequence: MN908947.3 (Wuhan-Hu-1 complete genome) [web](https://www.ebi.ac.uk/ena/browser/view/MN908947.3) [fasta](https://www.ebi.ac.uk/ena/browser/api/fasta/MN908947.3?download=true)] | diff --git a/clean.nf b/clean.nf index 0076b7f..395b29e 100755 --- a/clean.nf +++ b/clean.nf @@ -253,12 +253,12 @@ def helpMSG() { ${c_yellow}Decontamination options:${c_reset} ${c_green}--host${c_reset} Comma separated list of reference genomes for decontamination, downloaded based on this parameter [default: $params.host] ${c_dim}Currently supported are: - - hsa [Ensembl: Homo_sapiens.GRCh38.dna.primary_assembly] - - t2t [T2T Consortium: human genome w/ additional 200 Mbp, closed gaps, and more complete Y (T2T-CHM13+Yv2.0)] - - mmu [Ensembl: Mus_musculus.GRCm38.dna.primary_assembly] - - csa [NCBI: GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic] - - gga [NCBI: Gallus_gallus.GRCg6a.dna.toplevel] - - cli [NCBI: GCF_000337935.1_Cliv_1.0_genomic] + - hsa [Ensembl: Homo_sapiens.GRCh38.dna.primary_assembly, incl. mtDNA] + - t2t [T2T Consortium: human genome w/ additional 200 Mbp, closed gaps, and more complete Y (T2T-CHM13+Yv2.0), incl. mtDNA] + - mmu [Ensembl: Mus_musculus.GRCm38.dna.primary_assembly, incl. mtDNA] + - csa [NCBI: GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic, incl. mtDNA] + - gga [NCBI: Gallus_gallus.GRCg6a.dna.toplevel, incl. mtDNA] + - cli [NCBI: GCF_000337935.1_Cliv_1.0_genomic, incl. mtDNA] - eco [Ensembl: Escherichia_coli_k_12.ASM80076v1.dna.toplevel] - sc2 [ENA: MN908947.3 (Wuhan-Hu-1 complete genome)]${c_reset} ${c_green}--control${c_reset} Comma separated list of common controls used in Illumina or Nanopore sequencing [default: $params.control] diff --git a/modules/prepare_contamination.nf b/modules/prepare_contamination.nf index fdbae89..f74c17e 100644 --- a/modules/prepare_contamination.nf +++ b/modules/prepare_contamination.nf @@ -40,7 +40,7 @@ process download_host { gzip host-temp.fa ;; t2t) - wget https://s3-us-west-2.amazonaws.com/human-pangenomics/T2T/CHM13/assemblies/analysis_set/chm13v2.0.fa.gz -O host-temp.fa.gz + wget https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/009/914/755/GCA_009914755.4_T2T-CHM13v2.0/GCA_009914755.4_T2T-CHM13v2.0_genomic.fna.gz -O host-temp.fa.gz ;; *) echo "Unknown host ($host)." From 9a9193b9baaf220de96ef53dcaa5202627c7f2bf Mon Sep 17 00:00:00 2001 From: "Huska, Matthew" Date: Thu, 8 Aug 2024 14:12:41 +0200 Subject: [PATCH 9/9] Quote all urls. Good to avoid problems with special characters being interpreted by the shell --- modules/prepare_contamination.nf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/prepare_contamination.nf b/modules/prepare_contamination.nf index f74c17e..f9d8cd5 100644 --- a/modules/prepare_contamination.nf +++ b/modules/prepare_contamination.nf @@ -18,29 +18,29 @@ process download_host { """ case $host in hsa) - wget ftp://ftp.ensembl.org/pub/release-99/fasta/homo_sapiens/dna/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz -O host-temp.fa.gz + wget 'ftp://ftp.ensembl.org/pub/release-99/fasta/homo_sapiens/dna/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz' -O host-temp.fa.gz ;; mmu) - wget ftp://ftp.ensembl.org/pub/release-99/fasta/mus_musculus/dna/Mus_musculus.GRCm38.dna.primary_assembly.fa.gz -O host-temp.fa.gz + wget 'ftp://ftp.ensembl.org/pub/release-99/fasta/mus_musculus/dna/Mus_musculus.GRCm38.dna.primary_assembly.fa.gz' -O host-temp.fa.gz ;; cli) - wget ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/337/935/GCF_000337935.1_Cliv_1.0/GCF_000337935.1_Cliv_1.0_genomic.fna.gz -O host-temp.fa.gz + wget 'ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/337/935/GCF_000337935.1_Cliv_1.0/GCF_000337935.1_Cliv_1.0_genomic.fna.gz' -O host-temp.fa.gz ;; csa) - wget ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/409/795/GCF_000409795.2_Chlorocebus_sabeus_1.1/GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic.fna.gz -O host-temp.fa.gz + wget 'ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/409/795/GCF_000409795.2_Chlorocebus_sabeus_1.1/GCF_000409795.2_Chlorocebus_sabeus_1.1_genomic.fna.gz' -O host-temp.fa.gz ;; gga) - wget ftp://ftp.ensembl.org/pub/release-99/fasta/gallus_gallus/dna/Gallus_gallus.GRCg6a.dna.toplevel.fa.gz -O host-temp.fa.gz + wget 'ftp://ftp.ensembl.org/pub/release-99/fasta/gallus_gallus/dna/Gallus_gallus.GRCg6a.dna.toplevel.fa.gz' -O host-temp.fa.gz ;; eco) - wget ftp://ftp.ensemblgenomes.org/pub/release-45/bacteria//fasta/bacteria_90_collection/escherichia_coli_k_12/dna/Escherichia_coli_k_12.ASM80076v1.dna.toplevel.fa.gz -O host-temp.fa.gz + wget 'ftp://ftp.ensemblgenomes.org/pub/release-45/bacteria//fasta/bacteria_90_collection/escherichia_coli_k_12/dna/Escherichia_coli_k_12.ASM80076v1.dna.toplevel.fa.gz' -O host-temp.fa.gz ;; sc2) - wget "https://www.ebi.ac.uk/ena/browser/api/fasta/MN908947.3?download=true" -O host-temp.fa + wget 'https://www.ebi.ac.uk/ena/browser/api/fasta/MN908947.3?download=true' -O host-temp.fa gzip host-temp.fa ;; t2t) - wget https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/009/914/755/GCA_009914755.4_T2T-CHM13v2.0/GCA_009914755.4_T2T-CHM13v2.0_genomic.fna.gz -O host-temp.fa.gz + wget 'https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/009/914/755/GCA_009914755.4_T2T-CHM13v2.0/GCA_009914755.4_T2T-CHM13v2.0_genomic.fna.gz' -O host-temp.fa.gz ;; *) echo "Unknown host ($host)."