From fc8c26ade15eb7279b9c9a280f212b794b0ec1f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borja=20Dom=C3=ADnguez?= Date: Thu, 18 Apr 2024 21:01:04 +0200 Subject: [PATCH] Allow merge lists with extends (#3469) * Allow merge lists with extends * [MegaLinter] Apply linters fixes * Add CONFIG_PROPERTIES_TO_APPEND * Merge conflicts * [MegaLinter] Apply linters fixes --------- Co-authored-by: bdovaz Co-authored-by: nvuillam Co-authored-by: nvuillam --- .../base.local.mega-linter.yml | 4 ++ .../local.mega-linter.yml | 6 +++ .../base.local.mega-linter.yml | 4 ++ .../local.mega-linter.yml | 4 ++ README.md | 1 + docs/config-variables.md | 1 + megalinter/config.py | 24 +++++++++- .../megalinter-configuration.jsonschema.json | 14 ++++++ .../tests/test_megalinter/config_test.py | 48 +++++++++++++++++++ 9 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 .automation/test/mega-linter-config-test/local_extends_list_merge_append/base.local.mega-linter.yml create mode 100644 .automation/test/mega-linter-config-test/local_extends_list_merge_append/local.mega-linter.yml create mode 100644 .automation/test/mega-linter-config-test/local_extends_list_merge_replace/base.local.mega-linter.yml create mode 100644 .automation/test/mega-linter-config-test/local_extends_list_merge_replace/local.mega-linter.yml diff --git a/.automation/test/mega-linter-config-test/local_extends_list_merge_append/base.local.mega-linter.yml b/.automation/test/mega-linter-config-test/local_extends_list_merge_append/base.local.mega-linter.yml new file mode 100644 index 00000000000..7217d7fe231 --- /dev/null +++ b/.automation/test/mega-linter-config-test/local_extends_list_merge_append/base.local.mega-linter.yml @@ -0,0 +1,4 @@ +ENABLE_LINTERS: + - LINTER_1 + - LINTER_3 +FILTER_REGEX_INCLUDE: "(base-local)" diff --git a/.automation/test/mega-linter-config-test/local_extends_list_merge_append/local.mega-linter.yml b/.automation/test/mega-linter-config-test/local_extends_list_merge_append/local.mega-linter.yml new file mode 100644 index 00000000000..a63e81232cd --- /dev/null +++ b/.automation/test/mega-linter-config-test/local_extends_list_merge_append/local.mega-linter.yml @@ -0,0 +1,6 @@ +EXTENDS: base.local.mega-linter.yml +CONFIG_PROPERTIES_TO_APPEND: + - ENABLE_LINTERS +ENABLE_LINTERS: + - LINTER_2 +FILTER_REGEX_INCLUDE: "(local)" diff --git a/.automation/test/mega-linter-config-test/local_extends_list_merge_replace/base.local.mega-linter.yml b/.automation/test/mega-linter-config-test/local_extends_list_merge_replace/base.local.mega-linter.yml new file mode 100644 index 00000000000..7217d7fe231 --- /dev/null +++ b/.automation/test/mega-linter-config-test/local_extends_list_merge_replace/base.local.mega-linter.yml @@ -0,0 +1,4 @@ +ENABLE_LINTERS: + - LINTER_1 + - LINTER_3 +FILTER_REGEX_INCLUDE: "(base-local)" diff --git a/.automation/test/mega-linter-config-test/local_extends_list_merge_replace/local.mega-linter.yml b/.automation/test/mega-linter-config-test/local_extends_list_merge_replace/local.mega-linter.yml new file mode 100644 index 00000000000..dc5cf99e8a6 --- /dev/null +++ b/.automation/test/mega-linter-config-test/local_extends_list_merge_replace/local.mega-linter.yml @@ -0,0 +1,4 @@ +EXTENDS: base.local.mega-linter.yml +ENABLE_LINTERS: + - LINTER_2 +FILTER_REGEX_INCLUDE: "(local)" diff --git a/README.md b/README.md index f2534bdaed8..6fdbe801d63 100644 --- a/README.md +++ b/README.md @@ -954,6 +954,7 @@ description: List of common variables that you can use to customize MegaLinter b | **ADDITIONAL_EXCLUDED_DIRECTORIES** | \[\] | List of additional excluded directory basenames. they're excluded at any nested level. | | [**APPLY_FIXES**](https://github.com/oxsecurity/megalinter/tree/main/docs/config-apply-fixes.md) | `none` | Activates formatting and autofix [(more info)](https://github.com/oxsecurity/megalinter/tree/main/docs/config-apply-fixes.md) | | **CLEAR_REPORT_FOLDER** | `false` | Flag to clear files from report folder (usually megalinter-reports) before starting the linting process | +| **CONFIG_PROPERTIES_TO_APPEND** | \[\] | List of configuration properties to append their values (instead of replacing them) in case of using EXTENDS. | | **DEFAULT_BRANCH** | `HEAD` | Deprecated: The name of the repository's default branch. | | **DEFAULT_WORKSPACE** | `/tmp/lint` | The location containing files to lint if you are running locally. | | **DISABLE_ERRORS** | `false` | Flag to have the linter complete with exit code 0 even if errors were detected. | diff --git a/docs/config-variables.md b/docs/config-variables.md index 2f65d98a2fc..46b3053c051 100644 --- a/docs/config-variables.md +++ b/docs/config-variables.md @@ -13,6 +13,7 @@ description: List of common variables that you can use to customize MegaLinter b | **ADDITIONAL_EXCLUDED_DIRECTORIES** | \[\] | List of additional excluded directory basenames. they're excluded at any nested level. | | [**APPLY_FIXES**](config-apply-fixes.md) | `none` | Activates formatting and autofix [(more info)](config-apply-fixes.md) | | **CLEAR_REPORT_FOLDER** | `false` | Flag to clear files from report folder (usually megalinter-reports) before starting the linting process | +| **CONFIG_PROPERTIES_TO_APPEND** | \[\] | List of configuration properties to append their values (instead of replacing them) in case of using EXTENDS. | | **DEFAULT_BRANCH** | `HEAD` | Deprecated: The name of the repository's default branch. | | **DEFAULT_WORKSPACE** | `/tmp/lint` | The location containing files to lint if you are running locally. | | **DISABLE_ERRORS** | `false` | Flag to have the linter complete with exit code 0 even if errors were detected. | diff --git a/megalinter/config.py b/megalinter/config.py index c4e0c86104b..040f48ba764 100644 --- a/megalinter/config.py +++ b/megalinter/config.py @@ -104,6 +104,11 @@ def init_config(request_id, workspace=None, params={}): def combine_config(workspace, config, combined_config, config_source): + config_properties_to_append = [] + + if "CONFIG_PROPERTIES_TO_APPEND" in config: + config_properties_to_append = config["CONFIG_PROPERTIES_TO_APPEND"] + extends = config["EXTENDS"] if isinstance(extends, str): extends = extends.split(",") @@ -126,7 +131,7 @@ def combine_config(workspace, config, combined_config, config_source): workspace + os.path.sep + extends_item, "r", encoding="utf-8" ) as f: extends_config_data = yaml.safe_load(f) - combined_config.update(extends_config_data) + merge_dicts(combined_config, extends_config_data, config_properties_to_append) config_source += f"\n[config] - extends from: {extends_item}" if "EXTENDS" in extends_config_data: combine_config( @@ -135,10 +140,25 @@ def combine_config(workspace, config, combined_config, config_source): combined_config, config_source, ) - combined_config.update(config) + merge_dicts(combined_config, config, config_properties_to_append) return config_source +def merge_dicts(first, second, config_properties_to_append): + for k, v in second.items(): + if k not in first: + first[k] = v + else: + if ( + isinstance(first[k], list) + and isinstance(v, list) + and k in config_properties_to_append + ): + first[k] = first[k] + v + else: + first[k] = v + + def is_initialized_for(request_id): global RUN_CONFIGS if request_id in RUN_CONFIGS: diff --git a/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json b/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json index 8e57b421229..ecf29e3aff5 100644 --- a/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json +++ b/megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json @@ -2388,6 +2388,20 @@ "title": "Pre commands for COFFEE descriptor", "type": "array" }, + "CONFIG_PROPERTIES_TO_APPEND": { + "$id": "#/properties/CONFIG_PROPERTIES_TO_APPEND", + "description": "List of configuration properties to append their values (instead of replacing them) in case of using EXTENDS.", + "examples": [ + [ + "ENABLE_LINTERS" + ] + ], + "items": { + "type": "string" + }, + "title": "List of configuration properties to append their values", + "type": "array" + }, "CONFIG_REPORTER": { "$id": "#/properties/CONFIG_REPORTER", "default": true, diff --git a/megalinter/tests/test_megalinter/config_test.py b/megalinter/tests/test_megalinter/config_test.py index 561273e9d55..83ca6560985 100644 --- a/megalinter/tests/test_megalinter/config_test.py +++ b/megalinter/tests/test_megalinter/config_test.py @@ -96,6 +96,54 @@ def test_local_config_extends_success(self): self.assertEqual("false", config.get(request_id, "SHOW_ELAPSED_TIME")) self.restore_branch_in_input_files(changed_files) + def test_local_config_extends_list_merge_replace_success(self): + changed_files = self.replace_branch_in_input_files() + local_config = "local.mega-linter.yml" + request_id = str(uuid.uuid1()) + config.init_config( + request_id, + REPO_HOME_DEFAULT + + os.path.sep + + ".automation" + + os.path.sep + + "test" + + os.path.sep + + "mega-linter-config-test" + + os.path.sep + + "local_extends_list_merge_replace", + {"MEGALINTER_CONFIG": local_config}, + ) + self.assertEqual( + ["LINTER_2"], + config.get(request_id, "ENABLE_LINTERS"), + ) + self.assertEqual("(local)", config.get(request_id, "FILTER_REGEX_INCLUDE")) + self.restore_branch_in_input_files(changed_files) + + def test_local_config_extends_list_merge_append_success(self): + changed_files = self.replace_branch_in_input_files() + local_config = "local.mega-linter.yml" + request_id = str(uuid.uuid1()) + config.init_config( + request_id, + REPO_HOME_DEFAULT + + os.path.sep + + ".automation" + + os.path.sep + + "test" + + os.path.sep + + "mega-linter-config-test" + + os.path.sep + + "local_extends_list_merge_append", + {"MEGALINTER_CONFIG": local_config}, + ) + self.assertEqual( + ["LINTER_1", "LINTER_3", "LINTER_2"], + config.get(request_id, "ENABLE_LINTERS"), + ) + self.assertEqual("(local)", config.get(request_id, "FILTER_REGEX_INCLUDE")) + self.restore_branch_in_input_files(changed_files) + def test_local_config_extends_recurse_success(self): changed_files = self.replace_branch_in_input_files() local_config = "recurse.mega-linter.yml"