Skip to content

Commit

Permalink
Allow merge lists with extends (#3469)
Browse files Browse the repository at this point in the history 
* Allow merge lists with extends

* [MegaLinter] Apply linters fixes

* Add CONFIG_PROPERTIES_TO_APPEND

* Merge conflicts

* [MegaLinter] Apply linters fixes

---------

Co-authored-by: bdovaz <bdovaz@users.noreply.github.com>
Co-authored-by: nvuillam <nicolas.vuillamy@gmail.com>
Co-authored-by: nvuillam <nvuillam@users.noreply.github.com>
  • Loading branch information
@bdovaz @nvuillam
4 people authored Apr 18, 2024
1 parent 5cec3c2 commit fc8c26a
Show file tree
Hide file tree
Showing 9 changed files with 104 additions and 2 deletions.
4 changes: 4 additions & 0 deletions ...n/test/mega-linter-config-test/local_extends_list_merge_append/base.local.mega-linter.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
ENABLE_LINTERS:
- LINTER_1
- LINTER_3
FILTER_REGEX_INCLUDE: "(base-local)"
6 changes: 6 additions & 0 deletions ...mation/test/mega-linter-config-test/local_extends_list_merge_append/local.mega-linter.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
EXTENDS: base.local.mega-linter.yml
CONFIG_PROPERTIES_TO_APPEND:
- ENABLE_LINTERS
ENABLE_LINTERS:
- LINTER_2
FILTER_REGEX_INCLUDE: "(local)"
4 changes: 4 additions & 0 deletions .../test/mega-linter-config-test/local_extends_list_merge_replace/base.local.mega-linter.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
ENABLE_LINTERS:
- LINTER_1
- LINTER_3
FILTER_REGEX_INCLUDE: "(base-local)"
4 changes: 4 additions & 0 deletions ...ation/test/mega-linter-config-test/local_extends_list_merge_replace/local.mega-linter.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
EXTENDS: base.local.mega-linter.yml
ENABLE_LINTERS:
- LINTER_2
FILTER_REGEX_INCLUDE: "(local)"
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -954,6 +954,7 @@ description: List of common variables that you can use to customize MegaLinter b
| **ADDITIONAL_EXCLUDED_DIRECTORIES** | \[\] | List of additional excluded directory basenames. they're excluded at any nested level. |
| [**APPLY_FIXES**](https://github.com/oxsecurity/megalinter/tree/main/docs/config-apply-fixes.md) | `none` | Activates formatting and autofix [(more info)](https://github.com/oxsecurity/megalinter/tree/main/docs/config-apply-fixes.md) |
| **CLEAR_REPORT_FOLDER** | `false` | Flag to clear files from report folder (usually megalinter-reports) before starting the linting process |
| **CONFIG_PROPERTIES_TO_APPEND** | \[\] | List of configuration properties to append their values (instead of replacing them) in case of using EXTENDS. |
| **DEFAULT_BRANCH** | `HEAD` | Deprecated: The name of the repository's default branch. |
| **DEFAULT_WORKSPACE** | `/tmp/lint` | The location containing files to lint if you are running locally. |
| **DISABLE_ERRORS** | `false` | Flag to have the linter complete with exit code 0 even if errors were detected. |
Expand Down
1 change: 1 addition & 0 deletions docs/config-variables.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ description: List of common variables that you can use to customize MegaLinter b
| **ADDITIONAL_EXCLUDED_DIRECTORIES** | \[\] | List of additional excluded directory basenames. they're excluded at any nested level. |
| [**APPLY_FIXES**](config-apply-fixes.md) | `none` | Activates formatting and autofix [(more info)](config-apply-fixes.md) |
| **CLEAR_REPORT_FOLDER** | `false` | Flag to clear files from report folder (usually megalinter-reports) before starting the linting process |
| **CONFIG_PROPERTIES_TO_APPEND** | \[\] | List of configuration properties to append their values (instead of replacing them) in case of using EXTENDS. |
| **DEFAULT_BRANCH** | `HEAD` | Deprecated: The name of the repository's default branch. |
| **DEFAULT_WORKSPACE** | `/tmp/lint` | The location containing files to lint if you are running locally. |
| **DISABLE_ERRORS** | `false` | Flag to have the linter complete with exit code 0 even if errors were detected. |
Expand Down
24 changes: 22 additions & 2 deletions megalinter/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,11 @@ def init_config(request_id, workspace=None, params={}):


def combine_config(workspace, config, combined_config, config_source):
config_properties_to_append = []

if "CONFIG_PROPERTIES_TO_APPEND" in config:
config_properties_to_append = config["CONFIG_PROPERTIES_TO_APPEND"]

extends = config["EXTENDS"]
if isinstance(extends, str):
extends = extends.split(",")
Expand All @@ -126,7 +131,7 @@ def combine_config(workspace, config, combined_config, config_source):
workspace + os.path.sep + extends_item, "r", encoding="utf-8"
) as f:
extends_config_data = yaml.safe_load(f)
combined_config.update(extends_config_data)
merge_dicts(combined_config, extends_config_data, config_properties_to_append)
config_source += f"\n[config] - extends from: {extends_item}"
if "EXTENDS" in extends_config_data:
combine_config(
Expand All @@ -135,10 +140,25 @@ def combine_config(workspace, config, combined_config, config_source):
combined_config,
config_source,
)
combined_config.update(config)
merge_dicts(combined_config, config, config_properties_to_append)
return config_source


def merge_dicts(first, second, config_properties_to_append):
for k, v in second.items():
if k not in first:
first[k] = v
else:
if (
isinstance(first[k], list)
and isinstance(v, list)
and k in config_properties_to_append
):
first[k] = first[k] + v
else:
first[k] = v


def is_initialized_for(request_id):
global RUN_CONFIGS
if request_id in RUN_CONFIGS:
Expand Down
14 changes: 14 additions & 0 deletions megalinter/descriptors/schemas/megalinter-configuration.jsonschema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2388,6 +2388,20 @@
"title": "Pre commands for COFFEE descriptor",
"type": "array"
},
"CONFIG_PROPERTIES_TO_APPEND": {
"$id": "#/properties/CONFIG_PROPERTIES_TO_APPEND",
"description": "List of configuration properties to append their values (instead of replacing them) in case of using EXTENDS.",
"examples": [
[
"ENABLE_LINTERS"
]
],
"items": {
"type": "string"
},
"title": "List of configuration properties to append their values",
"type": "array"
},
"CONFIG_REPORTER": {
"$id": "#/properties/CONFIG_REPORTER",
"default": true,
Expand Down
48 changes: 48 additions & 0 deletions megalinter/tests/test_megalinter/config_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,54 @@ def test_local_config_extends_success(self):
self.assertEqual("false", config.get(request_id, "SHOW_ELAPSED_TIME"))
self.restore_branch_in_input_files(changed_files)

def test_local_config_extends_list_merge_replace_success(self):
changed_files = self.replace_branch_in_input_files()
local_config = "local.mega-linter.yml"
request_id = str(uuid.uuid1())
config.init_config(
request_id,
REPO_HOME_DEFAULT
+ os.path.sep
+ ".automation"
+ os.path.sep
+ "test"
+ os.path.sep
+ "mega-linter-config-test"
+ os.path.sep
+ "local_extends_list_merge_replace",
{"MEGALINTER_CONFIG": local_config},
)
self.assertEqual(
["LINTER_2"],
config.get(request_id, "ENABLE_LINTERS"),
)
self.assertEqual("(local)", config.get(request_id, "FILTER_REGEX_INCLUDE"))
self.restore_branch_in_input_files(changed_files)

def test_local_config_extends_list_merge_append_success(self):
changed_files = self.replace_branch_in_input_files()
local_config = "local.mega-linter.yml"
request_id = str(uuid.uuid1())
config.init_config(
request_id,
REPO_HOME_DEFAULT
+ os.path.sep
+ ".automation"
+ os.path.sep
+ "test"
+ os.path.sep
+ "mega-linter-config-test"
+ os.path.sep
+ "local_extends_list_merge_append",
{"MEGALINTER_CONFIG": local_config},
)
self.assertEqual(
["LINTER_1", "LINTER_3", "LINTER_2"],
config.get(request_id, "ENABLE_LINTERS"),
)
self.assertEqual("(local)", config.get(request_id, "FILTER_REGEX_INCLUDE"))
self.restore_branch_in_input_files(changed_files)

def test_local_config_extends_recurse_success(self):
changed_files = self.replace_branch_in_input_files()
local_config = "recurse.mega-linter.yml"
Expand Down

0 comments on commit fc8c26a

Please sign in to comment.