Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove old references to CUSTOMDUMPSOFTWAREVERSIONS #2897

Merged
merged 13 commits into from
Apr 3, 2024
Merged
Prev Previous commit
Next Next commit
combine file check and add tests
  • Loading branch information
mashehu committed Apr 2, 2024
commit 6c717dcb2189353a7e25e63cbfd9740c2274407e
67 changes: 34 additions & 33 deletions nf_core/lint/configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
from pathlib import Path
from typing import Dict, List

from nf_core.lint_utils import ignore_file

log = logging.getLogger(__name__)


Expand All @@ -14,44 +16,43 @@ def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]):
def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]:
"""Lint a file and add the result to the passed or failed list."""

fn = Path(self.wf_path, file_path)
passed: List[str] = []
failed: List[str] = []
ignored: List[str] = []
ignore_configs: List[str] = []

fn = Path(self.wf_path, file_path)

ignore_configs = self.lint_config.get(lint_name, [])

# Return a failed status if we can't find the file
if not fn.is_file():
if ignore_configs:
return {"ignored": [f"`{file_path}` not found, but it is ignored."]}
else:
return {"failed": [f"`${file_path}` not found"]}

try:
with open(fn) as fh:
config = fh.read()
except Exception as e:
return {"failed": [f"Could not parse file: {fn}, {e}"]}

# find sections with a withName: prefix
sections = re.findall(r"['\"](.*)['\"]", config)

# find all .nf files in the workflow directory
nf_files = list(Path(self.wf_path).rglob("*.nf"))
log.debug(f"found nf_files: {nf_files}")

# check if withName sections are present in config, but not in workflow files
for section in sections:
if section not in ignore_configs or section.lower() not in ignore_configs:
if not any(section in nf_file.read_text() for nf_file in nf_files):
failed.append(
f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the following workflow files: `{nf_files}`."
)
passed, failed, ignored, ignore_configs = ignore_file(lint_name, file_path, Path(self.wf_path))

error_message = f"`{file_path}` not found"
# check for partial match in failed or ignored
if not any(f.startswith(error_message) for f in (failed + ignored)):
try:
with open(fn) as fh:
config = fh.read()
except Exception as e:
return {"failed": [f"Could not parse file: {fn}, {e}"]}

# find sections with a withName: prefix
sections = re.findall(r"withName:\s*['\"]?(\w+)['\"]?", config)
log.debug(f"found sections: {sections}")

# find all .nf files in the workflow directory
nf_files = list(Path(self.wf_path).rglob("*.nf"))
log.debug(f"found nf_files: {[str(f) for f in nf_files]}")

# check if withName sections are present in config, but not in workflow files
for section in sections:
if section not in ignore_configs or section.lower() not in ignore_configs:
if not any(section in nf_file.read_text() for nf_file in nf_files):
failed.append(
f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the Nextflow scripts."
)
else:
passed.append(f"`{section}` found in `{file_path}` and Nextflow scripts.")
else:
passed.append(f"both `{file_path}` and `{[str(f) for f in nf_files]} contain `{section}`.")
else:
ignored.append(f"``{section}` is ignored")
ignored.append(f"``{section}` is ignored.")

return {"passed": passed, "failed": failed, "ignored": ignored}

Expand Down
196 changes: 104 additions & 92 deletions nf_core/lint/multiqc_config.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import os
from pathlib import Path
from typing import Dict, List

import yaml

from nf_core.lint_utils import ignore_file


def multiqc_config(self) -> Dict[str, List[str]]:
"""Make sure basic multiQC plugins are installed and plots are exported
Expand All @@ -21,100 +23,110 @@ def multiqc_config(self) -> Dict[str, List[str]]:
order: -1001
export_plots: true

"""

passed: List[str] = []
failed: List[str] = []

# Remove field that should be ignored according to the linting config
ignore_configs = self.lint_config.get("multiqc_config", [])
.. note:: You can choose to ignore this lint tests by editing the file called
``.nf-core.yml`` in the root of your pipeline and setting the test to false:

fn = os.path.join(self.wf_path, "assets", "multiqc_config.yml")
.. code-block:: yaml

# Return a failed status if we can't find the file
if not os.path.isfile(fn):
return {"ignored": ["'assets/multiqc_config.yml' not found"]}
lint:
multiqc_config: False

try:
with open(fn) as fh:
mqc_yml = yaml.safe_load(fh)
except Exception as e:
return {"failed": [f"Could not parse yaml file: {fn}, {e}"]}

# check if requried sections are present
required_sections = ["report_section_order", "export_plots", "report_comment"]
for section in required_sections:
if section not in mqc_yml and section not in ignore_configs:
failed.append(f"'assets/multiqc_config.yml' does not contain `{section}`")
return {"passed": passed, "failed": failed}
else:
passed.append(f"'assets/multiqc_config.yml' contains `{section}`")

try:
orders = {}
summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary"
min_plugins = ["software_versions", summary_plugin_name]
for plugin in min_plugins:
if plugin not in mqc_yml["report_section_order"]:
raise AssertionError(f"Section {plugin} missing in report_section_order")
if "order" not in mqc_yml["report_section_order"][plugin]:
raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0")
plugin_order = mqc_yml["report_section_order"][plugin]["order"]
if plugin_order >= 0:
raise AssertionError(f"Section {plugin} 'order' must be < 0")

for plugin in mqc_yml["report_section_order"]:
if "order" in mqc_yml["report_section_order"][plugin]:
orders[plugin] = mqc_yml["report_section_order"][plugin]["order"]

if orders[summary_plugin_name] != min(orders.values()):
raise AssertionError(f"Section {summary_plugin_name} should have the lowest order")
orders.pop(summary_plugin_name)
if orders["software_versions"] != min(orders.values()):
raise AssertionError("Section software_versions should have the second lowest order")
except (AssertionError, KeyError, TypeError) as e:
failed.append(f"'assets/multiqc_config.yml' does not meet requirements: {e}")
else:
passed.append("'assets/multiqc_config.yml' follows the ordering scheme of the minimally required plugins.")

if "report_comment" not in ignore_configs:
# Check that the minimum plugins exist and are coming first in the summary
version = self.nf_config.get("manifest.version", "").strip(" '\"")
if "dev" in version:
version = "dev"
report_comments = (
f'This report has been generated by the <a href="https://github.com/nf-core/{self.pipeline_name}/tree/dev" target="_blank">nf-core/{self.pipeline_name}</a>'
f" analysis pipeline. For information about how to interpret these results, please see the "
f'<a href="https://nf-co.re/{self.pipeline_name}/dev/docs/output" target="_blank">documentation</a>.'
)
"""

passed: List[str] = []
failed: List[str] = []
ignored: List[str] = []

fn = Path(self.wf_path, "assets", "multiqc_config.yml")
file_path = fn.relative_to(self.wf_path)
passed, failed, ignored, ignore_configs = ignore_file("multiqc_config", file_path, self.wf_path)
print(f"passed: {passed}")
print(f"failed: {failed}")
print(f"ignored: {ignored}")
# skip other tests if the file is not found
error_message = f"`{file_path}` not found"
# check for partial match in failed or ignored
if not any(f.startswith(error_message) for f in (failed + ignored)):
try:
with open(fn) as fh:
mqc_yml = yaml.safe_load(fh)
except Exception as e:
return {"failed": [f"Could not parse yaml file: {fn}, {e}"]}

# check if required sections are present
required_sections = ["report_section_order", "export_plots", "report_comment"]
for section in required_sections:
if section not in mqc_yml and section not in ignore_configs:
failed.append(f"`assets/multiqc_config.yml` does not contain `{section}`")
return {"passed": passed, "failed": failed}
else:
passed.append(f"`assets/multiqc_config.yml` contains `{section}`")

try:
orders = {}
summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary"
min_plugins = ["software_versions", summary_plugin_name]
for plugin in min_plugins:
if plugin not in mqc_yml["report_section_order"]:
raise AssertionError(f"Section {plugin} missing in report_section_order")
if "order" not in mqc_yml["report_section_order"][plugin]:
raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0")
plugin_order = mqc_yml["report_section_order"][plugin]["order"]
if plugin_order >= 0:
raise AssertionError(f"Section {plugin} 'order' must be < 0")

for plugin in mqc_yml["report_section_order"]:
if "order" in mqc_yml["report_section_order"][plugin]:
orders[plugin] = mqc_yml["report_section_order"][plugin]["order"]

if orders[summary_plugin_name] != min(orders.values()):
raise AssertionError(f"Section {summary_plugin_name} should have the lowest order")
orders.pop(summary_plugin_name)
if orders["software_versions"] != min(orders.values()):
raise AssertionError("Section software_versions should have the second lowest order")
except (AssertionError, KeyError, TypeError) as e:
failed.append(f"`assets/multiqc_config.yml` does not meet requirements: {e}")
else:
report_comments = (
f'This report has been generated by the <a href="https://github.com/nf-core/{self.pipeline_name}/releases/tag/{version}" target="_blank">nf-core/{self.pipeline_name}</a>'
f" analysis pipeline. For information about how to interpret these results, please see the "
f'<a href="https://nf-co.re/{self.pipeline_name}/{version}/docs/output" target="_blank">documentation</a>.'
)

if mqc_yml["report_comment"].strip() != report_comments:
# find where the report_comment is wrong and give it as a hint
hint = report_comments
failed.append(
f"'assets/multiqc_config.yml' does not contain a matching 'report_comment'. \n"
f"The expected comment is: \n"
f"```{hint}``` \n"
f"The current comment is: \n"
f"```{ mqc_yml['report_comment'].strip()}```"
)
passed.append("`assets/multiqc_config.yml` follows the ordering scheme of the minimally required plugins.")

if "report_comment" not in ignore_configs:
# Check that the minimum plugins exist and are coming first in the summary
version = self.nf_config.get("manifest.version", "").strip(" '\"")
if "dev" in version:
version = "dev"
report_comments = (
f'This report has been generated by the <a href="https://github.com/nf-core/{self.pipeline_name}/tree/dev" target="_blank">nf-core/{self.pipeline_name}</a>'
f" analysis pipeline. For information about how to interpret these results, please see the "
f'<a href="https://nf-co.re/{self.pipeline_name}/dev/docs/output" target="_blank">documentation</a>.'
)

else:
report_comments = (
f'This report has been generated by the <a href="https://github.com/nf-core/{self.pipeline_name}/releases/tag/{version}" target="_blank">nf-core/{self.pipeline_name}</a>'
f" analysis pipeline. For information about how to interpret these results, please see the "
f'<a href="https://nf-co.re/{self.pipeline_name}/{version}/docs/output" target="_blank">documentation</a>.'
)

if mqc_yml["report_comment"].strip() != report_comments:
# find where the report_comment is wrong and give it as a hint
hint = report_comments
failed.append(
f"`assets/multiqc_config.yml` does not contain a matching 'report_comment'. \n"
f"The expected comment is: \n"
f"```{hint}``` \n"
f"The current comment is: \n"
f"```{ mqc_yml['report_comment'].strip()}```"
)
else:
passed.append("`assets/multiqc_config.yml` contains a matching 'report_comment'.")

# Check that export_plots is activated
try:
if not mqc_yml["export_plots"]:
raise AssertionError()
except (AssertionError, KeyError, TypeError):
failed.append("`assets/multiqc_config.yml` does not contain 'export_plots: true'.")
else:
passed.append("'assets/multiqc_config.yml' contains a matching 'report_comment'.")

# Check that export_plots is activated
try:
if not mqc_yml["export_plots"]:
raise AssertionError()
except (AssertionError, KeyError, TypeError):
failed.append("'assets/multiqc_config.yml' does not contain 'export_plots: true'.")
else:
passed.append("'assets/multiqc_config.yml' contains 'export_plots: true'.")

return {"passed": passed, "failed": failed}
passed.append("`assets/multiqc_config.yml` contains 'export_plots: true'.")

return {"passed": passed, "failed": failed, "ignored": ignored}
52 changes: 25 additions & 27 deletions nf_core/lint_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import logging
import subprocess
from pathlib import Path
from typing import Tuple
from typing import List

import rich
from rich.console import Console
Expand Down Expand Up @@ -104,32 +104,30 @@ def dump_json_with_prettier(file_name, file_content):
run_prettier_on_file(file_name)


def parse_config_file(self, lint_name: str, file_path: Path) -> Tuple[dict, dict]:
"""Parse different kind of config files and return a dict."""

# Remove field that should be ignored according to the linting config
ignore_configs = self.lint_config.get(lint_name, [])

fn = Path(self.wf_path, file_path)
def ignore_file(lint_name: str, file_path: Path, dir_path: Path) -> List[List[str]]:
"""Ignore a file and add the result to the ignored list. Return the passed, failed, ignored and ignore_configs lists."""

passed: List[str] = []
failed: List[str] = []
ignored: List[str] = []
_, lint_conf = nf_core.utils.load_tools_config(dir_path)
lint_conf = lint_conf.get("lint", {})
print(f"lint_conf: {lint_conf}")
ignore_entry: List[str] | bool = lint_conf.get(lint_name, [])
print(f"ignore_entry: {ignore_entry}")
full_path = dir_path / file_path
# Return a failed status if we can't find the file
if not fn.is_file():
return {"ignored": [f"`${file_path}` not found"]}, ignore_configs

try:
if fn.suffix == ".json":
import json

with open(fn) as fh:
config = json.load(fh)
return config, ignore_configs
elif fn.suffix == ".yml" or fn.suffix == ".yaml":
import yaml

with open(fn) as fh:
config = yaml.safe_load(fh)
return config, ignore_configs
if not full_path.is_file():
if isinstance(ignore_entry, bool) and not ignore_entry:
ignored.append(f"`{file_path}` not found, but it is ignored.")
ignore_entry = []
else:
return {"failed": [f"Could not parse file: {fn}, unknown file type"]}, ignore_configs
except Exception as e:
return {"failed": [f"Could not parse file: {fn}, {e}"]}, ignore_configs
failed.append(f"`{file_path}` not found.")
else:
passed.append(f"`{file_path}` found and not ignored.")

# we handled the only case where ignore_entry should be a bool, convert it to a list, to make downstream code easier
if isinstance(ignore_entry, bool):
ignore_entry = []

return [passed, failed, ignored, ignore_entry]
9 changes: 6 additions & 3 deletions nf_core/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,11 @@ def is_pipeline_directory(wf_path):
for fn in ["main.nf", "nextflow.config"]:
path = os.path.join(wf_path, fn)
if not os.path.isfile(path):
raise UserWarning(f"'{wf_path}' is not a pipeline - '{fn}' is missing")
if wf_path == ".":
warning = f"Current directory is not a pipeline - '{fn}' is missing."
else:
warning = f"'{wf_path}' is not a pipeline - '{fn}' is missing."
raise UserWarning(warning)


def fetch_wf_config(wf_path: str, cache_config: bool = True) -> dict:
Expand Down Expand Up @@ -1013,7 +1017,7 @@ def get_repo_releases_branches(pipeline, wfs):
DEPRECATED_CONFIG_PATHS = [".nf-core-lint.yml", ".nf-core-lint.yaml"]


def load_tools_config(directory: Union[str, Path] = "."):
def load_tools_config(directory: Union[str, Path] = ".") -> Tuple[Path, dict]:
"""
Parse the nf-core.yml configuration file

Expand Down Expand Up @@ -1041,7 +1045,6 @@ def load_tools_config(directory: Union[str, Path] = "."):

with open(config_fn) as fh:
tools_config = yaml.safe_load(fh)

# If the file is empty
tools_config = tools_config or {}

Expand Down
Loading