Merge pull request #902 from ewels/bump-v-1.13

Bump version to 1.13

.github/markdownlint.yml

@@ -10,6 +10,7 @@ no-inline-html:
         - kbd
         - details
         - summary
+        - kbd
 # tools only - the {{ jinja variables }} break URLs and cause this to error
 no-bare-urls: false
 # tools only - suppresses error messages for usage of $ in main README

CHANGELOG.md

@@ -1,6 +1,6 @@
 # nf-core/tools: Changelog
 
-## [v1.13 - Copper Crocodile](https://github.com/nf-core/tools/releases/tag/1.13) - [2021-03-16]
+## [v1.13 - Copper Crocodile](https://github.com/nf-core/tools/releases/tag/1.13) - [2021-03-18]
 
 ### Template
 
@@ -27,12 +27,16 @@
 
 ### Modules
 
-* added `nf-core modules lint` command to enable linting of nf-core and local modules
-* added `nf-core modules remove` command to uninstall modules
-* added `nf-core modules create-test-yml` command which runs the test for a new module and automatically
-  creates the `test.yml` for with md5 sums, tags, commands and names added
-* added `nf-core modules create` command to generate a new module from the module template
-* added questionary autocomplete functionality to `nf-core modules install`
+Initial addition of a number of new helper commands for working with DSL2 modules:
+
+* `modules list` - List available modules
+* `modules install` - Install a module from nf-core/modules
+* `modules remove` - Remove a module from a pipeline
+* `modules create` - Create a module from the template
+* `modules create-test-yml` - Create the `test.yml` file for a module with md5 sums, tags, commands and names added
+* `modules lint` - Check a module against nf-core guidelines
+
+You can read more about each of these commands in the main tools documentation (see `README.md` or <https://nf-co.re/tools>)
 
 ### Tools helper code
 
@@ -47,6 +51,10 @@
 
 ### Linting
 
+* Major refactor and rewrite of pipieline linting code
+  * Much better code organisation and maintainability
+  * New automatically generated documentation using Sphinx
+  * Numerous new tests and functions, removal of some unnecessary tests
 * Added lint check for merge markers [[#321]](https://github.com/nf-core/tools/issues/321)
 * Added new option `--fix` to automatically correct some problems detected by linting
 * Added validation of default params to `nf-core schema lint` [[#823](https://github.com/nf-core/tools/issues/823)]

nf_core/create.py

@@ -83,7 +83,7 @@ def render_template(self):
             loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True
         )
         template_dir = os.path.join(os.path.dirname(__file__), "pipeline-template")
-        copy_ftypes = ["image", "application/java-archive"]
+        binary_ftypes = ["image", "application/java-archive"]
         object_attrs = vars(self)
         object_attrs["nf_core_version"] = nf_core.__version__
 
@@ -108,7 +108,7 @@ def render_template(self):
 
             # Just copy binary files
             (ftype, encoding) = mimetypes.guess_type(template_fn_path)
-            if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in copy_ftypes])):
+            if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])):
                 log.debug(f"Copying binary file: '{output_path}'")
                 shutil.copy(template_fn_path, output_path)
                 continue

nf_core/lint/__init__.py

@@ -275,7 +275,7 @@ def format_result(test_results, table):
             string for the terminal with appropriate ASCII colours.
             """
             for eid, msg in test_results:
-                table.add_row(Markdown("[{0}](https://nf-co.re/errors#{0}): {1}".format(eid, msg)))
+                table.add_row(Markdown("[{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html): {1}".format(eid, msg)))
             return table
 
         def _s(some_list):
@@ -332,7 +332,7 @@ def _s(some_list):
         if len(self.could_fix):
             fix_cmd = "nf-core lint {} --fix {}".format(self.wf_path, " --fix ".join(self.could_fix))
             console.print(
-                f"\nTip: Some of these linting errors can automatically resolved with the following command:\n\n[blue]    {fix_cmd}\n"
+                f"\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n[blue]    {fix_cmd}\n"
             )
         if len(self.fix):
             console.print(
@@ -359,7 +359,9 @@ def _get_results_md(self):
             test_failures = "### :x: Test failures:\n\n{}\n\n".format(
                 "\n".join(
                     [
-                        "* [{0}](https://nf-co.re/errors#{0}) - {1}".format(eid, self._strip_ansi_codes(msg, "`"))
+                        "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format(
+                            eid, self._strip_ansi_codes(msg, "`")
+                        )
                         for eid, msg in self.failed
                     ]
                 )
@@ -372,7 +374,9 @@ def _get_results_md(self):
             test_ignored = "### :grey_question: Tests ignored:\n\n{}\n\n".format(
                 "\n".join(
                     [
-                        "* [{0}](https://nf-co.re/errors#{0}) - {1}".format(eid, self._strip_ansi_codes(msg, "`"))
+                        "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format(
+                            eid, self._strip_ansi_codes(msg, "`")
+                        )
                         for eid, msg in self.ignored
                     ]
                 )
@@ -385,7 +389,9 @@ def _get_results_md(self):
             test_fixed = "### :grey_question: Tests fixed:\n\n{}\n\n".format(
                 "\n".join(
                     [
-                        "* [{0}](https://nf-co.re/errors#{0}) - {1}".format(eid, self._strip_ansi_codes(msg, "`"))
+                        "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format(
+                            eid, self._strip_ansi_codes(msg, "`")
+                        )
                         for eid, msg in self.fixed
                     ]
                 )
@@ -398,7 +404,9 @@ def _get_results_md(self):
             test_warnings = "### :heavy_exclamation_mark: Test warnings:\n\n{}\n\n".format(
                 "\n".join(
                     [
-                        "* [{0}](https://nf-co.re/errors#{0}) - {1}".format(eid, self._strip_ansi_codes(msg, "`"))
+                        "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format(
+                            eid, self._strip_ansi_codes(msg, "`")
+                        )
                         for eid, msg in self.warned
                     ]
                 )
@@ -411,7 +419,9 @@ def _get_results_md(self):
             test_passes = "### :white_check_mark: Tests passed:\n\n{}\n\n".format(
                 "\n".join(
                     [
-                        "* [{0}](https://nf-co.re/errors#{0}) - {1}".format(eid, self._strip_ansi_codes(msg, "`"))
+                        "* [{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html) - {1}".format(
+                            eid, self._strip_ansi_codes(msg, "`")
+                        )
                         for eid, msg in self.passed
                     ]
                 )

nf_core/lint/merge_markers.py

@@ -24,21 +24,23 @@ def merge_markers(self):
         with io.open(os.path.join(self.wf_path, ".gitignore"), "rt", encoding="latin1") as fh:
             for l in fh:
                 ignore.append(os.path.basename(l.strip().rstrip("/")))
-    for root, dirs, files in os.walk(self.wf_path):
+    for root, dirs, files in os.walk(self.wf_path, topdown=True):
         # Ignore files
-        for i in ignore:
-            dirs = [d for d in dirs if not fnmatch.fnmatch(os.path.join(root, d), i)]
-            files = [f for f in files if not fnmatch.fnmatch(os.path.join(root, f), i)]
+        for i_base in ignore:
+            i = os.path.join(root, i_base)
+            dirs[:] = [d for d in dirs if not fnmatch.fnmatch(os.path.join(root, d), i)]
+            files[:] = [f for f in files if not fnmatch.fnmatch(os.path.join(root, f), i)]
         for fname in files:
             try:
                 with io.open(os.path.join(root, fname), "rt", encoding="latin1") as fh:
                     for l in fh:
                         if ">>>>>>>" in l:
-                            failed.append(f"Merge marker in `{fname}`: {l}")
+                            failed.append(f"Merge marker '>>>>>>>' in `{os.path.join(root, fname)}`: {l}")
                         if "<<<<<<<" in l:
-                            failed.append(f"Merge marker in `{fname}`: {l}")
+                            failed.append(f"Merge marker '<<<<<<<' in `{os.path.join(root, fname)}`: {l}")
+                            print(root)
             except FileNotFoundError:
-                log.debug(f"Could not open file {fname} in merge_markers lint test")
+                log.debug(f"Could not open file {os.path.join(root, fname)} in merge_markers lint test")
     if len(failed) == 0:
         passed.append("No merge markers found in pipeline files")
     return {"passed": passed, "failed": failed}

nf_core/lint/pipeline_todos.py

@@ -42,11 +42,12 @@ def pipeline_todos(self):
         with io.open(os.path.join(self.wf_path, ".gitignore"), "rt", encoding="latin1") as fh:
             for l in fh:
                 ignore.append(os.path.basename(l.strip().rstrip("/")))
-    for root, dirs, files in os.walk(self.wf_path):
+    for root, dirs, files in os.walk(self.wf_path, topdown=True):
         # Ignore files
-        for i in ignore:
-            dirs = [d for d in dirs if not fnmatch.fnmatch(os.path.join(root, d), i)]
-            files = [f for f in files if not fnmatch.fnmatch(os.path.join(root, f), i)]
+        for i_base in ignore:
+            i = os.path.join(root, i_base)
+            dirs[:] = [d for d in dirs if not fnmatch.fnmatch(os.path.join(root, d), i)]
+            files[:] = [f for f in files if not fnmatch.fnmatch(os.path.join(root, f), i)]
         for fname in files:
             try:
                 with io.open(os.path.join(root, fname), "rt", encoding="latin1") as fh:

nf_core/lint/template_strings.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 import io
-import os
+import mimetypes
 import re
 
 
@@ -26,11 +26,18 @@ def template_strings(self):
     # Loop through files, searching for string
     num_matches = 0
     for fn in self.files:
+
+        # Skip binary files
+        binary_ftypes = ["image", "application/java-archive"]
+        (ftype, encoding) = mimetypes.guess_type(fn)
+        if encoding is not None or (ftype is not None and any([ftype.startswith(ft) for ft in binary_ftypes])):
+            continue
+
         with io.open(fn, "r", encoding="latin1") as fh:
             lnum = 0
             for l in fh:
                 lnum += 1
-                cc_matches = re.findall(r"[^$]{{[^}]*}}", l)
+                cc_matches = re.findall(r"[^$]{{[^:}]*}}", l)
                 if len(cc_matches) > 0:
                     for cc_match in cc_matches:
                         failed.append("Found a Jinja template string in `{}` L{}: {}".format(fn, lnum, cc_match))

nf_core/modules/lint.py

@@ -98,7 +98,10 @@ def lint(self, module=None, all_modules=False, print_results=True, show_passed=F
             if len(nfcore_modules) == 0:
                 raise ModuleLintException(f"Could not find the specified module: '{module}'")
 
-        log.info(f"Linting pipeline: [magenta]{self.dir}")
+        if self.repo_type == "modules":
+            log.info(f"Linting modules repo: [magenta]{self.dir}")
+        else:
+            log.info(f"Linting pipeline: [magenta]{self.dir}")
         if module:
             log.info(f"Linting module: [magenta]{module}")
 

nf_core/modules/pipeline_modules.py

@@ -161,6 +161,8 @@ def list_modules(self, print_json=False):
 
         # No pipeline given - show all remote
         if self.pipeline_dir is None:
+            log.info(f"Modules available from {self.modules_repo.name} ({self.modules_repo.branch}):\n")
+
             # Get the list of available modules
             self.modules_repo.get_modules_file_tree()
             modules = self.modules_repo.modules_avail_module_names
@@ -171,6 +173,8 @@ def list_modules(self, print_json=False):
 
         # We have a pipeline - list what's installed
         else:
+            log.info(f"Modules installed in '{self.pipeline_dir}':\n")
+
             # Check whether pipelines is valid
             try:
                 self.has_valid_pipeline()
@@ -185,7 +189,6 @@ def list_modules(self, print_json=False):
                 log.info(f"No nf-core modules found in '{self.pipeline_dir}'")
                 return ""
 
-        log.info("Modules available from {} ({}):\n".format(self.modules_repo.name, self.modules_repo.branch))
         for mod in sorted(modules):
             table.add_row(mod)
         if print_json:

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 import sys
 
-version = "1.13dev"
+version = "1.13"
 
 with open("README.md") as f:
     readme = f.read()

tests/lint/merge_markers.py

@@ -21,4 +21,4 @@ def test_merge_markers_found(self):
     results = lint_obj.merge_markers()
     assert len(results["failed"]) > 0
     assert len(results["passed"]) == 0
-    assert "Merge marker in `main.nf`: >>>>>>>\n" == results["failed"][0]
+    assert "Merge marker '>>>>>>>' in " in results["failed"][0]