From bc669b8d2fc3e416b089409690e3dde938b83e37 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 15:22:48 +0200 Subject: [PATCH 01/13] fix refresh rate propagation --- nf_core/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 9321ff9629..b6f17cc51c 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -339,7 +339,7 @@ def setup_requests_cachedir(): return config -def wait_cli_function(poll_func, poll_every=20): +def wait_cli_function(poll_func, refresh_per_second=20): """ Display a command-line spinner while calling a function repeatedly. @@ -347,14 +347,14 @@ def wait_cli_function(poll_func, poll_every=20): Arguments: poll_func (function): Function to call - poll_every (int): How many tenths of a second to wait between function calls. Default: 20. + refresh_per_second (int): Refresh this many times per second. Default: 20. Returns: None. Just sits in an infite loop until the function returns True. """ try: spinner = Spinner("dots2", "Use ctrl+c to stop waiting and force exit.") - with Live(spinner, refresh_per_second=20): + with Live(spinner, refresh_per_second=refresh_per_second): while True: if poll_func(): break From 4aa1be470a853a9c1d6f31169dd75d7da1bb5070 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:21:58 +0200 Subject: [PATCH 02/13] clean up unused argument module_lint_obj --- nf_core/lint/__init__.py | 2 +- nf_core/lint_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 7e61de3ac1..90e770f37b 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -124,7 +124,7 @@ def run_linting( lint_obj._print_results(show_passed) module_lint_obj._print_results(show_passed) nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj) - nf_core.lint_utils.print_fixes(lint_obj, module_lint_obj) + nf_core.lint_utils.print_fixes(lint_obj) # Save results to Markdown file if md_fn is not None: diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index ffb3bdf7b3..956dfea208 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -33,7 +33,7 @@ def print_joint_summary(lint_obj, module_lint_obj): console.print(table) -def print_fixes(lint_obj, module_lint_obj): +def print_fixes(lint_obj): """Prints available and applied fixes""" if len(lint_obj.could_fix): From 3ab94b27240437052da7195979d4dedb860b510e Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:23:13 +0200 Subject: [PATCH 03/13] fix line-to-long --- nf_core/lint_utils.py | 6 ++++-- nf_core/modules/test_yml_builder.py | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 956dfea208..b831644166 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -41,9 +41,11 @@ def print_fixes(lint_obj): "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}", " --fix ".join(lint_obj.could_fix) ) console.print( - f"\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n[blue] {fix_cmd}\n" + "\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n" + f"[blue] {fix_cmd}\n" ) if len(lint_obj.fix): console.print( - "Automatic fixes applied. Please check with 'git diff' and revert any changes you do not want with 'git checkout '." + "Automatic fixes applied. " + "Please check with 'git diff' and revert any changes you do not want with 'git checkout '." ) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index a47f7c352a..79a502604a 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -54,7 +54,8 @@ def run(self): """Run build steps""" if not self.no_prompts: log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" + "[yellow]Press enter to use default values " + "[cyan bold](shown in brackets) [yellow]or type your own responses" ) self.check_inputs() self.scrape_workflow_entry_points() @@ -166,7 +167,10 @@ def build_single_test(self, entry_point): while ep_test["command"] == "": # Don't think we need the last `-c` flag, but keeping to avoid having to update 100s modules. # See https://github.com/nf-core/tools/issues/1562 - default_val = f"nextflow run ./tests/modules/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config -c ./tests/modules/{self.module_name}/nextflow.config" + default_val = ( + f"nextflow run ./tests/modules/{self.module_name} -entry {entry_point} " + f"-c ./tests/config/nextflow.config -c ./tests/modules/{self.module_name}/nextflow.config" + ) if self.no_prompts: ep_test["command"] = default_val else: From a2333cb8d840dbd9838bcf91f78328da7d86c086 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:23:45 +0200 Subject: [PATCH 04/13] use f-string --- nf_core/lint_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index b831644166..5495628e64 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -36,10 +36,10 @@ def print_joint_summary(lint_obj, module_lint_obj): def print_fixes(lint_obj): """Prints available and applied fixes""" - if len(lint_obj.could_fix): - fix_cmd = "nf-core lint {} --fix {}".format( - "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}", " --fix ".join(lint_obj.could_fix) - ) + if lint_obj.could_fix: + fix_flags = "".join([f" --fix {fix}" for fix in lint_obj.could_fix]) + wf_dir = "" if lint_obj.wf_path == "." else f"--dir {lint_obj.wf_path}" + fix_cmd = f"nf-core lint {wf_dir} {fix_flags}" console.print( "\nTip: Some of these linting errors can automatically be resolved with the following command:\n\n" f"[blue] {fix_cmd}\n" From accaaec2c1310225dbe20b68021eb283d8397bf6 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:25:13 +0200 Subject: [PATCH 05/13] clean up unused argument entry_point --- nf_core/modules/test_yml_builder.py | 4 ++-- tests/modules/create_test_yml.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 79a502604a..8254b3f761 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -192,7 +192,7 @@ def build_single_test(self, entry_point): ).strip() ep_test["tags"] = [t.strip() for t in prompt_tags.split(",")] - ep_test["files"] = self.get_md5_sums(entry_point, ep_test["command"]) + ep_test["files"] = self.get_md5_sums(ep_test["command"]) return ep_test @@ -254,7 +254,7 @@ def create_test_file_dict(self, results_dir, is_repeat=False): return test_files - def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repeat=None): + def get_md5_sums(self, command, results_dir=None, results_dir_repeat=None): """ Recursively go through directories and subdirectories and generate tuples of (, ) diff --git a/tests/modules/create_test_yml.py b/tests/modules/create_test_yml.py index dfb5fb5c6c..2f88b066a8 100644 --- a/tests/modules/create_test_yml.py +++ b/tests/modules/create_test_yml.py @@ -35,9 +35,7 @@ def test_modules_create_test_yml_get_md5(self, test_file_dir): meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", False, "./", False, True) with open(os.path.join(test_file_dir, "test_file.txt"), "w") as fh: fh.write("this line is just for testing") - test_files = meta_builder.get_md5_sums( - entry_point="dummy", command="dummy", results_dir=test_file_dir, results_dir_repeat=test_file_dir - ) + test_files = meta_builder.get_md5_sums(command="dummy", results_dir=test_file_dir, results_dir_repeat=test_file_dir) assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" From 7ee61f12c5a91b710cbcad643c3103eed38c3468 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:30:56 +0200 Subject: [PATCH 06/13] clean up unused argument hide_progress --- nf_core/modules/lint/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index dd1fd5b7d2..a35e10fde6 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -134,7 +134,6 @@ def lint( module=None, key=(), all_modules=False, - hide_progress=False, print_results=True, show_passed=False, local=False, From 750130431c688f8849774956a4fe3e6ec73a9e2f Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:42:39 +0200 Subject: [PATCH 07/13] fix line-to-long --- nf_core/modules/lint/main_nf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 4b5327020f..1fb4a17a66 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -432,7 +432,8 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, "https://" + new_url if not new_url.startswith("https://") else new_url, stream=True ) log.debug( - f"Connected to URL: {'https://' + new_url if not new_url.startswith('https://') else new_url}, status_code: {response_new_container.status_code}" + f"Connected to URL: {'https://' + new_url if not new_url.startswith('https://') else new_url}, " + f"status_code: {response_new_container.status_code}" ) except (requests.exceptions.RequestException, sqlite3.InterfaceError) as e: log.debug(f"Unable to connect to url '{new_url}' due to error: {e}") From d136124e021ac713fe997e93dfaff140da755fa3 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:43:34 +0200 Subject: [PATCH 08/13] clean up unused argument self --- nf_core/modules/lint/main_nf.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 1fb4a17a66..0c449834c8 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -100,18 +100,18 @@ def main_nf(module_lint_object, module, fix_version, progress_bar): continue # Perform state-specific linting checks - if state == "process" and not _is_empty(module, l): + if state == "process" and not _is_empty(l): process_lines.append(l) - if state == "input" and not _is_empty(module, l): + if state == "input" and not _is_empty(l): inputs.extend(_parse_input(module, l)) - if state == "output" and not _is_empty(module, l): + if state == "output" and not _is_empty(l): outputs += _parse_output(module, l) outputs = list(set(outputs)) # remove duplicate 'meta's - if state == "when" and not _is_empty(module, l): + if state == "when" and not _is_empty(l): when_lines.append(l) - if state == "script" and not _is_empty(module, l): + if state == "script" and not _is_empty(l): script_lines.append(l) - if state == "shell" and not _is_empty(module, l): + if state == "shell" and not _is_empty(l): shell_lines.append(l) # Check that we have required sections @@ -391,7 +391,7 @@ def _parse_output(self, line): return output -def _is_empty(self, line): +def _is_empty(line): """Check whether a line is empty or a comment""" empty = False if line.strip().startswith("//"): From a7bc6de42b7a3fb4f7050d454ede67da1fdbe2b1 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 16:44:12 +0200 Subject: [PATCH 09/13] simplified condition --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 0c449834c8..75d584a203 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -422,7 +422,7 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, build_type = _container_type(l) if build_type == "bioconda": new_lines.append(re.sub(rf"{current_version}", f"{latest_version}", line)) - elif build_type == "singularity" or build_type == "docker": + elif build_type in ("singularity", "docker"): # Check that the new url is valid new_url = re.search( "(?:['\"])(.+)(?:['\"])", re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line) From 49164ae51c405f7a28c1915f21b5edcd3aca8728 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Wed, 7 Sep 2022 17:29:56 +0200 Subject: [PATCH 10/13] indicate unused self --- nf_core/modules/lint/module_deprecations.py | 2 +- nf_core/modules/lint/module_tests.py | 2 +- nf_core/modules/lint/module_todos.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/module_deprecations.py b/nf_core/modules/lint/module_deprecations.py index f7e8761c75..13ab2a583b 100644 --- a/nf_core/modules/lint/module_deprecations.py +++ b/nf_core/modules/lint/module_deprecations.py @@ -5,7 +5,7 @@ log = logging.getLogger(__name__) -def module_deprecations(module_lint_object, module): +def module_deprecations(_, module): """ Check that the modules are up to the latest nf-core standard """ diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index b0c9fa0ee2..0b76acb944 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -9,7 +9,7 @@ log = logging.getLogger(__name__) -def module_tests(module_lint_object, module): +def module_tests(_, module): """ Lint the tests of a module in ``nf-core/modules`` diff --git a/nf_core/modules/lint/module_todos.py b/nf_core/modules/lint/module_todos.py index 90af44987e..82e7295eda 100644 --- a/nf_core/modules/lint/module_todos.py +++ b/nf_core/modules/lint/module_todos.py @@ -6,7 +6,7 @@ log = logging.getLogger(__name__) -def module_todos(module_lint_object, module): +def module_todos(_, module): """ Look for TODO statements in the module files From df2d4a08c9b3804386025b77d61d220b9d489baf Mon Sep 17 00:00:00 2001 From: fabianegli Date: Fri, 21 Oct 2022 09:09:06 +0200 Subject: [PATCH 11/13] remove unused argument since --- nf_core/modules/modules_repo.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 85e4a5b573..bd16e55000 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -443,17 +443,14 @@ def get_module_git_log(self, module_name, depth=None, since="2021-07-07T00:00:00 commits = iter(commits_new + commits_old) return commits - def get_subworkflow_git_log(self, subworkflow_name, depth=None, since="2021-07-07T00:00:00Z"): + def get_subworkflow_git_log(self, subworkflow_name, depth=None): """ - Fetches the commit history the of requested subworkflow since a given date. The default value is - not arbitrary - it is the last time the structure of the nf-core/subworkflow repository was had an - update breaking backwards compatibility. + Fetches the commit history the of requested subworkflow. Args: subworkflow_name (str): Name of subworkflow modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question per_page (int): Number of commits per page returned by API page_nbr (int): Page number of the retrieved commits - since (str): Only show commits later than this timestamp. Time should be given in ISO-8601 format: YYYY-MM-DDTHH:MM:SSZ. Returns: From 237e628be7d28db71c0e96be007399452e1232cd Mon Sep 17 00:00:00 2001 From: fabianegli Date: Fri, 21 Oct 2022 09:09:51 +0200 Subject: [PATCH 12/13] update docstring of get_subworkflow_git_log --- nf_core/modules/modules_repo.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index bd16e55000..72c794d85e 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -448,10 +448,7 @@ def get_subworkflow_git_log(self, subworkflow_name, depth=None): Fetches the commit history the of requested subworkflow. Args: subworkflow_name (str): Name of subworkflow - modules_repo (ModulesRepo): A ModulesRepo object configured for the repository in question - per_page (int): Number of commits per page returned by API - page_nbr (int): Page number of the retrieved commits - Time should be given in ISO-8601 format: YYYY-MM-DDTHH:MM:SSZ. + depth (int): Maximal number of commits Returns: ( dict ): Iterator of commit SHAs and associated (truncated) message From f3160d096368ae2b3118562276b5248011f59907 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Fri, 21 Oct 2022 09:15:05 +0200 Subject: [PATCH 13/13] code maintenance --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7e1afb421..bfa422e8c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,8 @@ - Update subworkflows install so it installs also imported modules and subworkflows ([#1904](https://github.com/nf-core/tools/pull/1904)) - Improve test coverage of sync.py - `check_up_to_date()` function from `modules_json` also checks for subworkflows. -- The default branch can now be specified when creating a new pipeline repo [#1959](https://github.com/nf-core/tools/pull/1959). +- The default branch can now be specified when creating a new pipeline repo ([#1959](https://github.com/nf-core/tools/pull/1959)) +- Code maintenance ([#1818](https://github.com/nf-core/tools/pull/1818)) ### Modules @@ -37,7 +38,7 @@ - Update MultiQC module, update supplying MultiQC default and custom config and logo files to module - Add a 'recommend' methods description text to MultiQC to help pipeline users report pipeline usage in publications ([#1749](https://github.com/nf-core/tools/pull/1749)) - Fix template spacing modified by JINJA ([#1830](https://github.com/nf-core/tools/pull/1830)) -- Fix MultiQC execution on template [#1855](https://github.com/nf-core/tools/pull/1855) +- Fix MultiQC execution on template ([#1855](https://github.com/nf-core/tools/pull/1855)) - Don't skip including `base.config` when skipping nf-core/configs ### Linting