From fb1f0bcf97f2753ae13cba6807d1fe452a7e6573 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 18 Apr 2023 13:00:35 +0200 Subject: [PATCH 01/13] remove remove_template_modules --- .github/workflows/create-lint-wf.yml | 16 +++------------- tests/modules/install.py | 3 --- tests/modules/patch.py | 16 +--------------- tests/modules/remove.py | 5 ----- tests/modules/update.py | 10 ---------- tests/subworkflows/install.py | 2 -- tests/subworkflows/list.py | 3 +-- tests/utils.py | 16 ---------------- 8 files changed, 5 insertions(+), 66 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index a5b69650bf..d09bdfd822 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -107,25 +107,15 @@ jobs: - name: nf-core modules install run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force + - name: nf-core modules install gitlab + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ + - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ - name: nf-core modules list remote run: nf-core --log-file log.txt modules list remote - - name: nf-core modules remove - run: | - find nf-core-testpipeline/workflows/ -type f -exec sed -i '/^include /d' {} \; - nf-core --log-file log.txt modules remove fastqc --dir nf-core-testpipeline/ - nf-core --log-file log.txt modules remove multiqc --dir nf-core-testpipeline/ - nf-core --log-file log.txt modules remove custom/dumpsoftwareversions --dir nf-core-testpipeline/ - - - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ - - - name: nf-core modules list local gitlab - run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ - - name: nf-core modules list remote gitlab run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote diff --git a/tests/modules/install.py b/tests/modules/install.py index fa588daf00..01f58b6fd2 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -9,7 +9,6 @@ GITLAB_BRANCH_TEST_BRANCH, GITLAB_REPO, GITLAB_URL, - remove_template_modules, with_temporary_folder, ) @@ -50,7 +49,6 @@ def test_modules_install_trimgalore_twice(self): def test_modules_install_from_gitlab(self): """Test installing a module from GitLab""" - remove_template_modules(self) assert self.mods_install_gitlab.install("fastqc") is True @@ -63,7 +61,6 @@ def test_modules_install_different_branch_fail(self): def test_modules_install_different_branch_succeed(self): """Test installing a module from a different branch""" - remove_template_modules(self) install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) # The fastp module does exists in the branch-test branch assert install_obj.install("fastp") is True diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 26ca5805a5..aba87e1857 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -7,7 +7,7 @@ import nf_core.components.components_command import nf_core.modules -from ..utils import GITLAB_URL, remove_template_modules +from ..utils import GITLAB_URL """ Test the 'nf-core modules patch' command @@ -69,9 +69,6 @@ def modify_main_nf(path): def test_create_patch_no_change(self): """Test creating a patch when there is no change to the module""" - # Remove modules that may cause org_path conflict - remove_template_modules(self) - setup_patch(self.pipeline_dir, False, self.pipeline_name) # Try creating a patch file @@ -91,9 +88,6 @@ def test_create_patch_no_change(self): def test_create_patch_change(self): """Test creating a patch when there is a change to the module""" - # Remove modules that may cause org_path conflict - remove_template_modules(self) - setup_patch(self.pipeline_dir, True, self.pipeline_name) # Try creating a patch file @@ -127,8 +121,6 @@ def test_create_patch_try_apply_successful(self): """ Test creating a patch file and applying it to a new version of the the files """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) setup_patch(self.pipeline_dir, True, self.pipeline_name) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) @@ -196,8 +188,6 @@ def test_create_patch_try_apply_failed(self): """ Test creating a patch file and applying it to a new version of the the files """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) setup_patch(self.pipeline_dir, True, self.pipeline_name) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) @@ -237,8 +227,6 @@ def test_create_patch_update_success(self): Should have the same effect as 'test_create_patch_try_apply_successful' but uses higher level api """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) setup_patch(self.pipeline_dir, True, self.pipeline_name) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) @@ -301,8 +289,6 @@ def test_create_patch_update_fail(self): """ Test creating a patch file and updating a module when there is a diff conflict """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) setup_patch(self.pipeline_dir, True, self.pipeline_name) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) diff --git a/tests/modules/remove.py b/tests/modules/remove.py index 9a8953a7ca..b343a02da7 100644 --- a/tests/modules/remove.py +++ b/tests/modules/remove.py @@ -1,7 +1,5 @@ import os -from ..utils import remove_template_modules - def test_modules_remove_trimgalore(self): """Test removing TrimGalore! module after installing it""" @@ -18,9 +16,6 @@ def test_modules_remove_trimgalore_uninstalled(self): def test_modules_remove_multiqc_from_gitlab(self): """Test removing multiqc module after installing it from an alternative source""" - # Remove modules that may cause org_path conflict - remove_template_modules(self) - self.mods_install_gitlab.install("multiqc") module_path = os.path.join(self.mods_install_gitlab.dir, "modules", "nf-core", "multiqc") assert self.mods_remove_gitlab.remove("multiqc", force=True) diff --git a/tests/modules/update.py b/tests/modules/update.py index e28f058760..fcfd92fc39 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -24,7 +24,6 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, - remove_template_modules, ) @@ -45,7 +44,6 @@ def test_install_and_update(self): def test_install_at_hash_and_update(self): """Installs an old version of a module in the pipeline and updates it""" - remove_template_modules(self) assert self.mods_install_old.install("trimgalore") update_obj = ModuleUpdate( self.pipeline_dir, show_diff=False, update_deps=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH @@ -71,7 +69,6 @@ def test_install_at_hash_and_update(self): def test_install_at_hash_and_update_and_save_diff_to_file(self): """Installs an old version of a module in the pipeline and updates it""" - remove_template_modules(self) self.mods_install_old.install("trimgalore") patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch") update_obj = ModuleUpdate( @@ -112,7 +109,6 @@ def test_update_all(self): def test_update_with_config_fixed_version(self): """Try updating when there are entries in the .nf-core.yml""" - remove_template_modules(self) # Install trimgalore at the latest version assert self.mods_install_trimgalore.install("trimgalore") @@ -138,7 +134,6 @@ def test_update_with_config_fixed_version(self): def test_update_with_config_dont_update(self): """Try updating when module is to be ignored""" - remove_template_modules(self) # Install an old version of trimgalore self.mods_install_old.install("trimgalore") @@ -169,7 +164,6 @@ def test_update_with_config_dont_update(self): def test_update_with_config_fix_all(self): """Fix the version of all nf-core modules""" - remove_template_modules(self) self.mods_install_trimgalore.install("trimgalore") # Fix the version of all nf-core modules in the .nf-core.yml to an old version @@ -193,7 +187,6 @@ def test_update_with_config_fix_all(self): def test_update_with_config_no_updates(self): """Don't update any nf-core modules""" - remove_template_modules(self) self.mods_install_old.install("trimgalore") old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() @@ -227,7 +220,6 @@ def test_update_with_config_no_updates(self): def test_update_different_branch_single_module(self): """Try updating a module in a specific branch""" - remove_template_modules(self) install_obj = ModuleInstall( self.pipeline_dir, prompt=False, @@ -254,7 +246,6 @@ def test_update_different_branch_single_module(self): def test_update_different_branch_mixed_modules_main(self): """Try updating all modules where MultiQC is installed from main branch""" - remove_template_modules(self) # Install fastp assert self.mods_install_gitlab_old.install("fastp") @@ -281,7 +272,6 @@ def test_update_different_branch_mixed_modules_main(self): def test_update_different_branch_mix_modules_branch_test(self): """Try updating all modules where MultiQC is installed from branch-test branch""" - remove_template_modules(self) # Install multiqc from the branch-test branch assert self.mods_install_gitlab_old.install( "multiqc" diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 1c451d8964..c7fe5e18e2 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -10,7 +10,6 @@ GITLAB_REPO, GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, - remove_template_modules, with_temporary_folder, ) @@ -63,7 +62,6 @@ def test_subworkflows_install_bam_sort_stats_samtools_twice(self): def test_subworkflows_install_from_gitlab(self): """Test installing a subworkflow from GitLab""" - remove_template_modules(self) assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is True # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py index cde6808bb8..c65999d42c 100644 --- a/tests/subworkflows/list.py +++ b/tests/subworkflows/list.py @@ -2,7 +2,7 @@ import nf_core.subworkflows -from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, remove_template_modules +from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL def test_subworkflows_list_remote(self): @@ -40,7 +40,6 @@ def test_subworkflows_install_and_list_subworkflows(self): def test_subworkflows_install_gitlab_and_list_subworkflows(self): """Test listing locally installed subworkflows""" - remove_template_modules(self) self.subworkflow_install_gitlab.install("bam_sort_stats_samtools") subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_subworkflows = subworkflows_list.list_components() diff --git a/tests/utils.py b/tests/utils.py index f33af2a7af..3fd265dd0c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -106,19 +106,3 @@ def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module, version): ], } rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) - - -def remove_template_modules(self): - # Remove modules that may cause org_path conflict - workflow_path = Path(self.pipeline_dir, "workflows", self.pipeline_name + ".nf") - with open(workflow_path, "r") as fh: - lines = fh.readlines() - with open(workflow_path, "w") as fh: - for line in lines: - if not line.startswith("include {"): - fh.write(line) - - remove_obj = nf_core.modules.ModuleRemove(self.pipeline_dir, no_pull=False) - - for i in ["multiqc", "fastqc", "custom/dumpsoftwareversions"]: - remove_obj.remove(i) From 1ddaad8e3e5e76bad126776c04f5445e73b55d33 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 18 Apr 2023 14:52:25 +0200 Subject: [PATCH 02/13] remove the steps which were removing pre-installed modules from gitlab nf-core --- tests/modules/install.py | 3 ++- tests/modules/lint.py | 5 +---- tests/subworkflows/install.py | 2 +- tests/test_subworkflows.py | 14 +++++++++++++- tests/utils.py | 4 +++- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/modules/install.py b/tests/modules/install.py index 01f58b6fd2..7f7f06de39 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -6,6 +6,7 @@ from nf_core.modules.modules_json import ModulesJson from ..utils import ( + GITLAB_BRANCH_ORG_PATH_BRANCH, GITLAB_BRANCH_TEST_BRANCH, GITLAB_REPO, GITLAB_URL, @@ -87,6 +88,6 @@ def test_modules_install_tracking(self): def test_modules_install_alternate_remote(self): """Test installing a module from a different remote with the same organization path""" - install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) + install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_ORG_PATH_BRANCH) # The fastp module does exists in the branch-test branch assert install_obj.install("fastp") is False diff --git a/tests/modules/lint.py b/tests/modules/lint.py index cc38e0ac34..9bab9eddeb 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -5,7 +5,7 @@ import nf_core.modules -from ..utils import GITLAB_URL, remove_template_modules, set_wd +from ..utils import GITLAB_URL, set_wd from .patch import BISMARK_ALIGN, CORRECT_SHA, PATCH_BRANCH, REPO_NAME, modify_main_nf @@ -62,7 +62,6 @@ def test_modules_lint_no_gitlab(self): def test_modules_lint_gitlab_modules(self): """Lint modules from a different remote""" - remove_template_modules(self) self.mods_install_gitlab.install("fastqc") self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) @@ -74,7 +73,6 @@ def test_modules_lint_gitlab_modules(self): def test_modules_lint_multiple_remotes(self): """Lint modules from a different remote""" - remove_template_modules(self) self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) module_lint.lint(print_results=False, all_modules=True) @@ -87,7 +85,6 @@ def test_modules_lint_patched_modules(self): """ Test creating a patch file and applying it to a new version of the the files """ - remove_template_modules(self) setup_patch(self.pipeline_dir, True) # Create a patch file diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index c7fe5e18e2..653de93a09 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -145,4 +145,4 @@ def test_subworkflows_install_tracking_added_super_subworkflow(self): def test_subworkflows_install_alternate_remote(self): """Test installing a subworkflow from a different remote with the same organization path""" self.subworkflow_install.install("bam_sort_stats_samtools") - assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is False + assert self.subworkflow_install_gitlab_same_org_path.install("bam_stats_samtools") is False diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 27373d2478..1c290cb882 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -12,7 +12,12 @@ import nf_core.modules import nf_core.subworkflows -from .utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, OLD_SUBWORKFLOWS_SHA +from .utils import ( + GITLAB_SUBWORKFLOWS_BRANCH, + GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, + GITLAB_URL, + OLD_SUBWORKFLOWS_SHA, +) def create_modules_repo_dummy(tmp_dir): @@ -62,6 +67,13 @@ def setUp(self): self.subworkflow_install_gitlab = nf_core.subworkflows.SubworkflowInstall( self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH ) + self.subworkflow_install_gitlab_same_org_path = nf_core.subworkflows.SubworkflowInstall( + self.pipeline_dir, + prompt=False, + force=False, + remote_url=GITLAB_URL, + branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, + ) self.subworkflow_install_old = nf_core.subworkflows.SubworkflowInstall( self.pipeline_dir, prompt=False, diff --git a/tests/utils.py b/tests/utils.py index 3fd265dd0c..a49de974fc 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -15,12 +15,14 @@ OLD_TRIMGALORE_SHA = "06348dffce2a732fc9e656bdc5c64c3e02d302cb" OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" -GITLAB_REPO = "nf-core" +GITLAB_REPO = "nf-core-test" GITLAB_DEFAULT_BRANCH = "main" GITLAB_SUBWORKFLOWS_BRANCH = "subworkflows" +GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH = "subworkflows-org-path" OLD_SUBWORKFLOWS_SHA = "f3c078809a2513f1c95de14f6633fe1f03572fdb" # Branch test stuff GITLAB_BRANCH_TEST_BRANCH = "branch-tester" +GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" GITLAB_BRANCH_TEST_OLD_SHA = "bce3f17980b8d1beae5e917cfd3c65c0c69e04b5" GITLAB_BRANCH_TEST_NEW_SHA = "2f5f180f6e705bb81d6e7742dc2f24bf4a0c721e" From 29c0defbb3058dc87fb33e4d6eabd1f387073546 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 09:12:13 +0200 Subject: [PATCH 03/13] try to fix some tests --- .github/workflows/create-lint-wf.yml | 2 +- tests/modules/install.py | 14 +++++++++++--- tests/modules/update.py | 2 +- tests/test_modules.py | 6 +++--- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index d09bdfd822..391a8cb9c0 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -108,7 +108,7 @@ jobs: run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --dir nf-core-testpipeline/ - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ diff --git a/tests/modules/install.py b/tests/modules/install.py index 7f7f06de39..0a072159d3 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -86,8 +86,16 @@ def test_modules_install_tracking(self): ] == ["modules"] -def test_modules_install_alternate_remote(self): +@pytest.fixture +def test_modules_install_alternate_remote(self, caplog): """Test installing a module from a different remote with the same organization path""" install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_ORG_PATH_BRANCH) - # The fastp module does exists in the branch-test branch - assert install_obj.install("fastp") is False + # Install fastqc from GitLab which is also installed from GitHub with the same org_path + install_obj.install("fastqc") + assert ( + caplog.msg + == "You are trying to install modules from different repositories with the same organization name 'nf-core'" + ) + # with pytest.raises(UserWarning) as excinfo: + # install_obj.install("fastqc")) + # assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) diff --git a/tests/modules/update.py b/tests/modules/update.py index fcfd92fc39..399e9cc12c 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -187,7 +187,7 @@ def test_update_with_config_fix_all(self): def test_update_with_config_no_updates(self): """Don't update any nf-core modules""" - self.mods_install_old.install("trimgalore") + assert self.mods_install_old.install("trimgalore") old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Fix the version of all nf-core modules in the .nf-core.yml to an old version diff --git a/tests/test_modules.py b/tests/test_modules.py index 057e77737c..74795c8080 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -78,21 +78,21 @@ def setUp(self): self.mods_install_trimgalore = nf_core.modules.ModuleInstall( self.pipeline_dir, prompt=False, - force=True, + force=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH, ) self.mods_install_gitlab = nf_core.modules.ModuleInstall( self.pipeline_dir, prompt=False, - force=True, + force=False, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH, ) self.mods_install_gitlab_old = nf_core.modules.ModuleInstall( self.pipeline_dir, prompt=False, - force=True, + force=False, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA, From 3ddfb83471555f29e18ade596218af01a2b831d2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 09:27:54 +0200 Subject: [PATCH 04/13] fix test_modules_install_alternate_remote and remove test_subworkflows_install_alternate_remote --- tests/modules/install.py | 14 ++++---------- tests/subworkflows/install.py | 6 ------ 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/tests/modules/install.py b/tests/modules/install.py index 0a072159d3..deca31204e 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -86,16 +86,10 @@ def test_modules_install_tracking(self): ] == ["modules"] -@pytest.fixture -def test_modules_install_alternate_remote(self, caplog): +def test_modules_install_alternate_remote(self): """Test installing a module from a different remote with the same organization path""" install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_ORG_PATH_BRANCH) # Install fastqc from GitLab which is also installed from GitHub with the same org_path - install_obj.install("fastqc") - assert ( - caplog.msg - == "You are trying to install modules from different repositories with the same organization name 'nf-core'" - ) - # with pytest.raises(UserWarning) as excinfo: - # install_obj.install("fastqc")) - # assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) + with pytest.raises(Exception) as excinfo: + install_obj.install("fastqc") + assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 653de93a09..6c04c9ad22 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -140,9 +140,3 @@ def test_subworkflows_install_tracking_added_super_subworkflow(self): "installed_by" ] ) == sorted(["subworkflows", "bam_sort_stats_samtools"]) - - -def test_subworkflows_install_alternate_remote(self): - """Test installing a subworkflow from a different remote with the same organization path""" - self.subworkflow_install.install("bam_sort_stats_samtools") - assert self.subworkflow_install_gitlab_same_org_path.install("bam_stats_samtools") is False From 359fa4ff463b6606360d1ecc57cef36714fdda5c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 09:51:27 +0200 Subject: [PATCH 05/13] remove unused modifications --- tests/modules/patch.py | 23 +++++++---------------- tests/test_modules.py | 1 - 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index aba87e1857..989aee8b8b 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -26,7 +26,7 @@ REPO_URL = "https://gitlab.com/nf-core/modules-test.git" -def setup_patch(pipeline_dir, modify_module, pipeline_name): +def setup_patch(pipeline_dir, modify_module): install_obj = nf_core.modules.ModuleInstall( pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH, sha=ORG_SHA ) @@ -40,15 +40,6 @@ def setup_patch(pipeline_dir, modify_module, pipeline_name): modify_main_nf(module_path / "main.nf") -def modify_workflow_nf(path): - with open(path, "r") as fh: - lines = fh.readlines() - with open(path, "w") as fh: - for line in lines: - if not line.startswith("include {"): - fh.write(line) - - def modify_main_nf(path): """Modify a file to test patch creation""" with open(path, "r") as fh: @@ -69,7 +60,7 @@ def modify_main_nf(path): def test_create_patch_no_change(self): """Test creating a patch when there is no change to the module""" - setup_patch(self.pipeline_dir, False, self.pipeline_name) + setup_patch(self.pipeline_dir, False) # Try creating a patch file patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) @@ -88,7 +79,7 @@ def test_create_patch_no_change(self): def test_create_patch_change(self): """Test creating a patch when there is a change to the module""" - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) # Try creating a patch file patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) @@ -122,7 +113,7 @@ def test_create_patch_try_apply_successful(self): Test creating a patch file and applying it to a new version of the the files """ - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) module_path = Path(self.pipeline_dir, module_relpath) @@ -189,7 +180,7 @@ def test_create_patch_try_apply_failed(self): Test creating a patch file and applying it to a new version of the the files """ - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) module_path = Path(self.pipeline_dir, module_relpath) @@ -228,7 +219,7 @@ def test_create_patch_update_success(self): but uses higher level api """ - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) # Try creating a patch file @@ -290,7 +281,7 @@ def test_create_patch_update_fail(self): Test creating a patch file and updating a module when there is a diff conflict """ - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) # Try creating a patch file diff --git a/tests/test_modules.py b/tests/test_modules.py index 74795c8080..1c24f9be7e 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -66,7 +66,6 @@ def setUp(self): ).init_pipeline() # Set up install objects self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) - self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=True, force=True) self.mods_install_old = nf_core.modules.ModuleInstall( self.pipeline_dir, prompt=False, From 66092a8122e52c69190a84f6d7cda88ac13d2ee6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 12:58:57 +0200 Subject: [PATCH 06/13] update sha for tests using gitlab --- tests/modules/patch.py | 10 +++++----- tests/utils.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 989aee8b8b..49c1a9255b 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -16,12 +16,12 @@ testing if the update commands works correctly with patch files """ -ORG_SHA = "775fcd090fb776a0be695044f8ab1af8896c8452" -CORRECT_SHA = "335cd32405568ca3b6d4c05ab1e8a98c21e18a4d" -SUCCEED_SHA = "f1566140c752e9c68fffc189fbe8cb9ee942b3ca" -FAIL_SHA = "1fc8b0f953d915d66ee40d28bc337ff0998d05bd" +ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" +CORRECT_SHA = "63fd3cdb1be733041db74c15542a7b5b8f4095ed" +SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" +FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align" -REPO_NAME = "nf-core" +REPO_NAME = "nf-core-test" PATCH_BRANCH = "patch-tester" REPO_URL = "https://gitlab.com/nf-core/modules-test.git" diff --git a/tests/utils.py b/tests/utils.py index a49de974fc..ea6eac7315 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -12,7 +12,7 @@ import nf_core.modules -OLD_TRIMGALORE_SHA = "06348dffce2a732fc9e656bdc5c64c3e02d302cb" +OLD_TRIMGALORE_SHA = "9b7a3bdefeaad5d42324aa7dd50f87bea1b04386" OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" GITLAB_REPO = "nf-core-test" @@ -23,8 +23,8 @@ # Branch test stuff GITLAB_BRANCH_TEST_BRANCH = "branch-tester" GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" -GITLAB_BRANCH_TEST_OLD_SHA = "bce3f17980b8d1beae5e917cfd3c65c0c69e04b5" -GITLAB_BRANCH_TEST_NEW_SHA = "2f5f180f6e705bb81d6e7742dc2f24bf4a0c721e" +GITLAB_BRANCH_TEST_OLD_SHA = "e772abc22c1ff26afdf377845c323172fb3c19ca" +GITLAB_BRANCH_TEST_NEW_SHA = "f5a3da1aea979ba7fa49cc51d2b7b6cd3f25239e" def with_temporary_folder(func): From 82ac98b18bd0c6224dd37d8e4dd30d70ec430b0e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 13:00:46 +0200 Subject: [PATCH 07/13] fix path for test_modules_remove_multiqc_from_gitlab --- tests/modules/remove.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/remove.py b/tests/modules/remove.py index b343a02da7..e412fd35a8 100644 --- a/tests/modules/remove.py +++ b/tests/modules/remove.py @@ -17,6 +17,6 @@ def test_modules_remove_trimgalore_uninstalled(self): def test_modules_remove_multiqc_from_gitlab(self): """Test removing multiqc module after installing it from an alternative source""" self.mods_install_gitlab.install("multiqc") - module_path = os.path.join(self.mods_install_gitlab.dir, "modules", "nf-core", "multiqc") + module_path = os.path.join(self.mods_install_gitlab.dir, "modules", "nf-core-test", "multiqc") assert self.mods_remove_gitlab.remove("multiqc", force=True) assert os.path.exists(module_path) is False From 4573a754c8e61e57883f2683f5825437f0f284ad Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 13:18:03 +0200 Subject: [PATCH 08/13] add modified test_subworkflows_install_alternate_remote --- tests/subworkflows/install.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 6c04c9ad22..dfe71686fb 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -9,6 +9,7 @@ GITLAB_BRANCH_TEST_BRANCH, GITLAB_REPO, GITLAB_SUBWORKFLOWS_BRANCH, + GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, GITLAB_URL, with_temporary_folder, ) @@ -140,3 +141,14 @@ def test_subworkflows_install_tracking_added_super_subworkflow(self): "installed_by" ] ) == sorted(["subworkflows", "bam_sort_stats_samtools"]) + + +def test_subworkflows_install_alternate_remote(self): + """Test installing a module from a different remote with the same organization path""" + install_obj = SubworkflowInstall( + self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH + ) + # Install a subworkflow from GitLab which is also installed from GitHub with the same org_path + with pytest.raises(Exception) as excinfo: + install_obj.install("fastqc") + assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) From d51f82c6acfeb94ec1bc959b1a6ef50d4f0095e8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 15:06:05 +0200 Subject: [PATCH 09/13] fix create-lint-wf.yml GHA --- .github/workflows/create-lint-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 391a8cb9c0..b936d9a74a 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -108,7 +108,7 @@ jobs: run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --dir nf-core-testpipeline/ + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastq --dir nf-core-testpipeline/ - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ From ecfa1019d297e8e90ee2a278bfe988c3b3f96998 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 15:15:03 +0200 Subject: [PATCH 10/13] fixing a typo --- .github/workflows/create-lint-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index b936d9a74a..f76187d6af 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -108,7 +108,7 @@ jobs: run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastq --dir nf-core-testpipeline/ + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastp --dir nf-core-testpipeline/ - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ From 68019595938e7da13a2db68d41a29d336c5fe237 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 16:30:17 +0200 Subject: [PATCH 11/13] hopefully this last sha update will fix all tests --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index ea6eac7315..86c07fd469 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -24,7 +24,7 @@ GITLAB_BRANCH_TEST_BRANCH = "branch-tester" GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" GITLAB_BRANCH_TEST_OLD_SHA = "e772abc22c1ff26afdf377845c323172fb3c19ca" -GITLAB_BRANCH_TEST_NEW_SHA = "f5a3da1aea979ba7fa49cc51d2b7b6cd3f25239e" +GITLAB_BRANCH_TEST_NEW_SHA = "7d73e21f30041297ea44367f2b4fd4e045c0b991" def with_temporary_folder(func): From 56a71edb5b89842f37790efc1bac2fb88766c93a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 20 Apr 2023 09:17:36 +0200 Subject: [PATCH 12/13] add review suggestions from @anoronh4 --- nf_core/modules/modules_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 431ad1d657..b9b64c56d2 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -427,8 +427,8 @@ def parse_dirs(self, dirs, missing_installation, component_type): git_url = None for repo in missing_installation: if component_type in missing_installation[repo]: - for dir_name in missing_installation[repo][component_type]: - if component in missing_installation[repo][component_type][dir_name]: + if install_dir in missing_installation[repo][component_type]: + if component in missing_installation[repo][component_type][install_dir]: component_in_file = True git_url = repo break From 73172ead2de0411ccf7c567b8cea5a2256774301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 20 Apr 2023 17:39:07 +0000 Subject: [PATCH 13/13] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e59d084a8..b41cd517b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,7 @@ - Add the Nextflow version to Gitpod container matching the minimal Nextflow version for nf-core (according to `nextflow.config`) ([#2196](https://github.com/nf-core/tools/pull/2196)) - Use `nfcore/gitpod:dev` container in the dev branch ([#2196](https://github.com/nf-core/tools/pull/2196)) - Replace requests_mock with responses in test mocks ([#2165](https://github.com/nf-core/tools/pull/2165)). -- Add warning when installing a module from an `org_path` that exists in multiple remotes in `modules.json` ([#2228](https://github.com/nf-core/tools/pull/2228)). +- Add warning when installing a module from an `org_path` that exists in multiple remotes in `modules.json` ([#2228](https://github.com/nf-core/tools/pull/2228) [#2239](https://github.com/nf-core/tools/pull/2239)). ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19]