From 3fad1a25f9dd437e563f7bfaef7a7946e4fb75a0 Mon Sep 17 00:00:00 2001 From: Artem Rys Date: Mon, 11 Dec 2023 11:17:42 +0100 Subject: [PATCH] feat: build commands produces detailed output of what happened (#927) This is a copy of the PR @MateuszMa created but due to how the repository was previously organized I needed to create another one myself. This PR introduces a new option `-v` to the `ucc-gen build` command which shows detailed information about created/copied/modified/conflict files after build is complete. --------- Co-authored-by: Mateusz Macalik --- docs/quickstart.md | 18 ++ splunk_add_on_ucc_framework/commands/build.py | 124 ++++++++++ splunk_add_on_ucc_framework/main.py | 11 + tests/smoke/test_ucc_build.py | 113 +++++++++ .../expected_log.json | 30 +++ .../package_files_conflict_test/README.md | 1 + .../globalConfig.json | 219 ++++++++++++++++++ .../package/LICENSE.txt | 0 .../package/README.txt | 0 .../package/README/inputs.conf.spec | 3 + .../README/test_addon_account.conf.spec | 2 + .../README/test_addon_settings.conf.spec | 2 + .../package/app.manifest | 57 +++++ .../package/bin/import_declare_test.py | 12 + .../package/bin/my_first_input.py | 110 +++++++++ .../package/bin/test_addon_rh_account.py | 41 ++++ .../package/lib/requirements.txt | 2 + .../globalConfig.json | 2 +- tests/unit/test_main.py | 13 +- 19 files changed, 758 insertions(+), 2 deletions(-) create mode 100644 tests/testdata/expected_addons/expected_files_conflict_test/expected_log.json create mode 100644 tests/testdata/test_addons/package_files_conflict_test/README.md create mode 100644 tests/testdata/test_addons/package_files_conflict_test/globalConfig.json create mode 100644 tests/testdata/test_addons/package_files_conflict_test/package/LICENSE.txt create mode 100644 tests/testdata/test_addons/package_files_conflict_test/package/README.txt create mode 100644 tests/testdata/test_addons/package_files_conflict_test/package/README/inputs.conf.spec create mode 100644 tests/testdata/test_addons/package_files_conflict_test/package/README/test_addon_account.conf.spec create mode 100644 tests/testdata/test_addons/package_files_conflict_test/package/README/test_addon_settings.conf.spec create mode 100644 tests/testdata/test_addons/package_files_conflict_test/package/app.manifest create mode 100644 tests/testdata/test_addons/package_files_conflict_test/package/bin/import_declare_test.py create mode 100644 tests/testdata/test_addons/package_files_conflict_test/package/bin/my_first_input.py create mode 100644 tests/testdata/test_addons/package_files_conflict_test/package/bin/test_addon_rh_account.py create mode 100644 tests/testdata/test_addons/package_files_conflict_test/package/lib/requirements.txt diff --git a/docs/quickstart.md b/docs/quickstart.md index 93d1b6996..96e637f3f 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -118,6 +118,24 @@ It takes the following parameters: Accepts absolute paths as well. * `--python-binary-name` - [optional] Python binary name to use when installing Python libraries. Default: `python3`. +* `-v` / `--verbose` - [optional] show detailed information about + created/copied/modified/conflict files after build is complete. + This option is in experimental mode. Default: `False`. + +#### Verbose mode + +Available from `v5.33.0`. + +Running `ucc-gen build -v` or `ucc-gen build --verbose` prints additional information about +what was exactly created / copied / modified / conflict after the build is complete. It does +not scan `lib` folder due to the nature of the folder. + +Below is the explanation on what exactly each state means: + +* `created` - file is not in the original package and was created during the build process +* `copied` - file is in the original package and was copied during the build process +* `modified` - file is in the original package and was modified during the build process +* `conflict` - file is in the original package and was copied during the build process but may be generated by UCC itself so incorrect usage can lead to not working add-on ### `ucc-gen init` diff --git a/splunk_add_on_ucc_framework/commands/build.py b/splunk_add_on_ucc_framework/commands/build.py index aacc96975..f56ff6e24 100644 --- a/splunk_add_on_ucc_framework/commands/build.py +++ b/splunk_add_on_ucc_framework/commands/build.py @@ -21,6 +21,9 @@ import sys from typing import Optional, List import subprocess +import colorama as c +import fnmatch +import filecmp from openapi3 import OpenAPI @@ -327,12 +330,126 @@ def _get_python_version_from_executable(python_binary_name: str) -> str: ) +def summary_report( + source: str, + ta_name: str, + output_directory: str, + verbose_file_summary_report: bool, +) -> None: + # initialising colorama to handle ASCII color in windows cmd + c.init() + color_palette = { + "copied": c.Fore.GREEN, + "conflict": c.Fore.RED, + "modified": c.Fore.YELLOW, + } + + # conflicting files from ucc-gen package folder + conflict_path = os.path.join(internal_root_dir, "package") + # conflict files generated through-out the process + conflict_static_list = frozenset( + [ + "import_declare_test.py", + f"{ta_name}_rh_*.py", + "app.conf", + "inputs.conf*", + "restmap.conf", + "server.conf", + f"{ta_name}_*.conf*", + "web.conf", + "default.xml", + "configuration.xml", + "dashboard.xml", + "inputs.xml", + "openapi.json", + ] + ) + + def line_print(print_path: str, mod_type: str) -> None: + if verbose_file_summary_report: + logger.info( + color_palette.get(mod_type, "") + + str(print_path).ljust(80) + + mod_type + + c.Style.RESET_ALL, + ) + summary[mod_type] += 1 + + def check_for_conflict(file: str, relative_file_path: str) -> bool: + conflict_path_file = os.path.join(conflict_path, relative_file_path) + if os.path.isfile(conflict_path_file): + return True + for pattern in conflict_static_list: + if fnmatch.fnmatch(file, pattern): + return True + return False + + def file_check( + file: str, output_directory: str, relative_file_path: str, source: str + ) -> None: + source_path = os.path.join(source, relative_file_path) + + if os.path.isfile(source_path): + # file is present in package + output_path = os.path.join(output_directory, relative_file_path) + + is_conflict = check_for_conflict(file, relative_file_path) + + if not is_conflict: + files_are_same = filecmp.cmp(source_path, output_path) + if not files_are_same: + # output file was modified + line_print(relative_file_path, "modified") + else: + # files are the same + line_print(relative_file_path, "copied") + else: + line_print(relative_file_path, "conflict") + else: + # file does not exist in package + line_print(relative_file_path, "created") + + summary = {"created": 0, "copied": 0, "modified": 0, "conflict": 0} + + path_len = len(output_directory) + 1 + + if verbose_file_summary_report: + logger.info("Detailed information about created/copied/modified/conflict files") + logger.info( + "Read more about it here: " + "https://splunk.github.io/addonfactory-ucc-generator/quickstart/#verbose-mode" + ) + + for path, dir, files in os.walk(output_directory): + relative_path = path[path_len:] + # skipping lib directory + if relative_path[:3] == "lib": + if relative_path == "lib": + line_print("lib", "created") + continue + + files = sorted(files, key=str.casefold) + + for file in files: + relative_file_path = os.path.join(relative_path, file) + file_check(file, output_directory, relative_file_path, source) + + summary_combined = ", ".join( + [ + f"{file_type}: {amount_of_files}" + for file_type, amount_of_files in summary.items() + ] + ) + logger.info(f"File creation summary: {summary_combined}") + + def generate( source: str, config_path: Optional[str] = None, addon_version: Optional[str] = None, output_directory: Optional[str] = None, python_binary_name: str = "python3", + verbose_file_summary_report: bool = False, ) -> None: logger.info(f"ucc-gen version {__version__} is used") logger.info(f"Python binary name to use: {python_binary_name}") @@ -583,3 +700,10 @@ def generate( logger.info(f"Creating {output_openapi_folder} folder") with open(output_openapi_path, "w") as openapi_file: json.dump(open_api.raw_element, openapi_file, indent=4) + + summary_report( + source, + ta_name, + os.path.join(output_directory, ta_name), + verbose_file_summary_report, + ) diff --git a/splunk_add_on_ucc_framework/main.py b/splunk_add_on_ucc_framework/main.py index 810075951..b54539412 100644 --- a/splunk_add_on_ucc_framework/main.py +++ b/splunk_add_on_ucc_framework/main.py @@ -103,6 +103,16 @@ def main(argv: Optional[Sequence[str]] = None) -> int: help="Python binary name to use to install requirements", default="python3", ) + build_parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help=( + "[experimental] show detailed information about " + "created/copied/modified/conflict files after build is complete" + ), + ) package_parser = subparsers.add_parser("package", description="Package an add-on") package_parser.add_argument( @@ -178,6 +188,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: addon_version=args.ta_version, output_directory=args.output, python_binary_name=args.python_binary_name, + verbose_file_summary_report=args.verbose, ) if args.command == "package": package.package(path_to_built_addon=args.path, output_directory=args.output) diff --git a/tests/smoke/test_ucc_build.py b/tests/smoke/test_ucc_build.py index 66cf54e64..ce17f08a6 100644 --- a/tests/smoke/test_ucc_build.py +++ b/tests/smoke/test_ucc_build.py @@ -2,6 +2,8 @@ import tempfile import sys import pytest +import logging +import json from os import path from tests.smoke import helpers @@ -279,3 +281,114 @@ def test_ucc_generate_openapi_with_configuration_files_only(): temp_dir, "Splunk_TA_UCCExample", "appserver", "static", "openapi.json" ) assert not path.exists(expected_file_path) + + +def test_ucc_build_verbose_mode(caplog): + """ + Tests results will test both no option and --verbose mode of build command. + No option provides a short summary of file created in manner: File creation summary: + --verbose shows each file specific case and short summary + """ + + caplog.set_level(logging.INFO, logger="ucc-gen") + + def extract_summary_logs(): + return_logs = [] + copy_logs = False + + message_to_start = ( + "Detailed information about created/copied/modified/conflict files" + ) + message_to_end = "File creation summary:" + + for record in caplog.records: + if record.message == message_to_start: + copy_logs = True + + if copy_logs: + return_logs.append(record) + + if record.message[:22] == message_to_end: + copy_logs = False + + return return_logs + + def generate_expected_log(): + def append_appserver_content(raw_expected_logs): + path_len = len(app_server_lib_path) + 1 + excluded_files = ["redirect_page.js", "redirect.html"] + + for full_path, dir, files in os.walk(app_server_lib_path): + if files: + relative_path = full_path[path_len:] + for file in files: + if file not in excluded_files: + relative_file_path = os.path.join(relative_path, file) + key_to_insert = ( + str(relative_file_path).ljust(80) + "created\u001b[0m" + ) + raw_expected_logs[key_to_insert] = "INFO" + + def summarize_types(raw_expected_logs): + summary_counter = {"created": 0, "copied": 0, "modified": 0, "conflict": 0} + + for log in raw_expected_logs: + end = log.find("\u001b[0m") + if end > 1: + string_end = end - 10 + operation_type = log[string_end:end].strip() + summary_counter[operation_type] += 1 + + summary_message = ( + f'File creation summary: created: {summary_counter.get("created")}, ' + f'copied: {summary_counter.get("copied")}, ' + f'modified: {summary_counter.get("modified")}, ' + f'conflict: {summary_counter.get("conflict")}' + ) + raw_expected_logs[summary_message] = "INFO" + + with open(expected_logs_path) as f: + raw_expected_logs = json.load(f) + + append_appserver_content(raw_expected_logs) + summarize_types(raw_expected_logs) + + return raw_expected_logs + + with tempfile.TemporaryDirectory() as temp_dir: + package_folder = path.join( + path.dirname(path.realpath(__file__)), + "..", + "testdata", + "test_addons", + "package_files_conflict_test", + "package", + ) + + expected_logs_path = path.join( + path.dirname(path.realpath(__file__)), + "..", + "testdata", + "expected_addons", + "expected_files_conflict_test", + "expected_log.json", + ) + + build.generate( + source=package_folder, + output_directory=temp_dir, + verbose_file_summary_report=True, + ) + + app_server_lib_path = os.path.join(build.internal_root_dir, "package") + + summary_logs = extract_summary_logs() + + expected_logs = generate_expected_log() + + assert len(summary_logs) == len(expected_logs) + + for log_line in summary_logs: + # summary messages must be the same but might come in different order + assert log_line.message in expected_logs.keys() + assert log_line.levelname == expected_logs[log_line.message] diff --git a/tests/testdata/expected_addons/expected_files_conflict_test/expected_log.json b/tests/testdata/expected_addons/expected_files_conflict_test/expected_log.json new file mode 100644 index 000000000..13101ce6e --- /dev/null +++ b/tests/testdata/expected_addons/expected_files_conflict_test/expected_log.json @@ -0,0 +1,30 @@ +{ + "Detailed information about created/copied/modified/conflict files": "INFO", + "Read more about it here: https://splunk.github.io/addonfactory-ucc-generator/quickstart/#verbose-mode": "INFO", + "\u001b[33mapp.manifest modified\u001b[0m": "INFO", + "\u001b[32mLICENSE.txt copied\u001b[0m": "INFO", + "\u001b[32mREADME.txt copied\u001b[0m": "INFO", + "VERSION created\u001b[0m": "INFO", + "\u001b[31mbin/import_declare_test.py conflict\u001b[0m": "INFO", + "\u001b[32mbin/my_first_input.py copied\u001b[0m": "INFO", + "\u001b[31mbin/test_addon_rh_account.py conflict\u001b[0m": "INFO", + "bin/test_addon_rh_my_first_input.py created\u001b[0m": "INFO", + "bin/test_addon_rh_settings.py created\u001b[0m": "INFO", + "default/app.conf created\u001b[0m": "INFO", + "default/inputs.conf created\u001b[0m": "INFO", + "default/restmap.conf created\u001b[0m": "INFO", + "default/server.conf created\u001b[0m": "INFO", + "default/test_addon_settings.conf created\u001b[0m": "INFO", + "default/web.conf created\u001b[0m": "INFO", + "default/data/ui/nav/default.xml created\u001b[0m": "INFO", + "default/data/ui/views/configuration.xml created\u001b[0m": "INFO", + "default/data/ui/views/dashboard.xml created\u001b[0m": "INFO", + "default/data/ui/views/inputs.xml created\u001b[0m": "INFO", + "\u001b[31mREADME/inputs.conf.spec conflict\u001b[0m": "INFO", + "\u001b[31mREADME/test_addon_account.conf.spec conflict\u001b[0m": "INFO", + "\u001b[31mREADME/test_addon_settings.conf.spec conflict\u001b[0m": "INFO", + "lib created\u001b[0m": "INFO", + "appserver/static/js/build/globalConfig.json created\u001b[0m": "INFO", + "appserver/static/openapi.json created\u001b[0m": "INFO", + "metadata/default.meta created\u001b[0m": "INFO" +} \ No newline at end of file diff --git a/tests/testdata/test_addons/package_files_conflict_test/README.md b/tests/testdata/test_addons/package_files_conflict_test/README.md new file mode 100644 index 000000000..d707ab1cd --- /dev/null +++ b/tests/testdata/test_addons/package_files_conflict_test/README.md @@ -0,0 +1 @@ +# test_addon diff --git a/tests/testdata/test_addons/package_files_conflict_test/globalConfig.json b/tests/testdata/test_addons/package_files_conflict_test/globalConfig.json new file mode 100644 index 000000000..102b67c94 --- /dev/null +++ b/tests/testdata/test_addons/package_files_conflict_test/globalConfig.json @@ -0,0 +1,219 @@ +{ + "pages": { + "configuration": { + "tabs": [ + { + "name": "account", + "table": { + "actions": [ + "edit", + "delete", + "clone" + ], + "header": [ + { + "label": "Name", + "field": "name" + } + ] + }, + "entity": [ + { + "type": "text", + "label": "Name", + "validators": [ + { + "type": "regex", + "errorMsg": "Account Name must begin with a letter and consist exclusively of alphanumeric characters and underscores.", + "pattern": "^[a-zA-Z]\\w*$" + }, + { + "type": "string", + "errorMsg": "Length of input name should be between 1 and 100", + "minLength": 1, + "maxLength": 100 + } + ], + "field": "name", + "help": "A unique name for the account.", + "required": true + }, + { + "type": "text", + "label": "API key", + "field": "api_key", + "help": "API key", + "required": true, + "encrypted": true + } + ], + "title": "Accounts" + }, + { + "name": "logging", + "entity": [ + { + "type": "singleSelect", + "label": "Log level", + "options": { + "disableSearch": true, + "autoCompleteFields": [ + { + "value": "DEBUG", + "label": "DEBUG" + }, + { + "value": "INFO", + "label": "INFO" + }, + { + "value": "WARN", + "label": "WARN" + }, + { + "value": "ERROR", + "label": "ERROR" + }, + { + "value": "CRITICAL", + "label": "CRITICAL" + } + ] + }, + "defaultValue": "INFO", + "field": "loglevel" + } + ], + "title": "Logging" + } + ], + "title": "Configuration", + "description": "Set up your add-on" + }, + "inputs": { + "services": [ + { + "name": "my_first_input", + "entity": [ + { + "type": "text", + "label": "Name", + "validators": [ + { + "type": "regex", + "errorMsg": "Input Name must begin with a letter and consist exclusively of alphanumeric characters and underscores.", + "pattern": "^[a-zA-Z]\\w*$" + }, + { + "type": "string", + "errorMsg": "Length of input name should be between 1 and 100", + "minLength": 1, + "maxLength": 100 + } + ], + "field": "name", + "help": "A unique name for the data input.", + "required": true + }, + { + "type": "text", + "label": "Interval", + "validators": [ + { + "type": "regex", + "errorMsg": "Interval must be an integer.", + "pattern": "^\\-[1-9]\\d*$|^\\d*$" + } + ], + "defaultValue": "300", + "field": "interval", + "help": "Time interval of the data input, in seconds.", + "required": true + }, + { + "type": "singleSelect", + "label": "Account to use", + "options": { + "referenceName": "account" + }, + "help": "Account to use for this input.", + "field": "account", + "required": true + } + ], + "title": "my_first_input" + } + ], + "title": "Inputs", + "description": "Manage your data inputs", + "table": { + "actions": [ + "edit", + "enable", + "delete", + "clone" + ], + "header": [ + { + "label": "Name", + "field": "name" + }, + { + "label": "Interval", + "field": "interval" + }, + { + "label": "Index", + "field": "index" + }, + { + "label": "Status", + "field": "disabled" + } + ], + "moreInfo": [ + { + "label": "Name", + "field": "name" + }, + { + "label": "Interval", + "field": "interval" + }, + { + "label": "Index", + "field": "index" + }, + { + "label": "Status", + "field": "disabled", + "mapping": { + "true": "Disabled", + "false": "Enabled" + } + } + ] + } + }, + "dashboard": { + "panels": [ + { + "name": "addon_version" + }, + { + "name": "events_ingested_by_sourcetype" + }, + { + "name": "errors_in_the_addon" + } + ] + } + }, + "meta": { + "name": "test_addon", + "restRoot": "test_addon", + "version": "5.34.1Racdcfb2e", + "displayName": "This is my add-on", + "schemaVersion": "0.0.3" + } +} diff --git a/tests/testdata/test_addons/package_files_conflict_test/package/LICENSE.txt b/tests/testdata/test_addons/package_files_conflict_test/package/LICENSE.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/testdata/test_addons/package_files_conflict_test/package/README.txt b/tests/testdata/test_addons/package_files_conflict_test/package/README.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/testdata/test_addons/package_files_conflict_test/package/README/inputs.conf.spec b/tests/testdata/test_addons/package_files_conflict_test/package/README/inputs.conf.spec new file mode 100644 index 000000000..00f172899 --- /dev/null +++ b/tests/testdata/test_addons/package_files_conflict_test/package/README/inputs.conf.spec @@ -0,0 +1,3 @@ +[my_first_input://] +interval = +account = \ No newline at end of file diff --git a/tests/testdata/test_addons/package_files_conflict_test/package/README/test_addon_account.conf.spec b/tests/testdata/test_addons/package_files_conflict_test/package/README/test_addon_account.conf.spec new file mode 100644 index 000000000..92b61cf81 --- /dev/null +++ b/tests/testdata/test_addons/package_files_conflict_test/package/README/test_addon_account.conf.spec @@ -0,0 +1,2 @@ +[] +api_key = \ No newline at end of file diff --git a/tests/testdata/test_addons/package_files_conflict_test/package/README/test_addon_settings.conf.spec b/tests/testdata/test_addons/package_files_conflict_test/package/README/test_addon_settings.conf.spec new file mode 100644 index 000000000..eb806fdeb --- /dev/null +++ b/tests/testdata/test_addons/package_files_conflict_test/package/README/test_addon_settings.conf.spec @@ -0,0 +1,2 @@ +[logging] +loglevel = \ No newline at end of file diff --git a/tests/testdata/test_addons/package_files_conflict_test/package/app.manifest b/tests/testdata/test_addons/package_files_conflict_test/package/app.manifest new file mode 100644 index 000000000..b6df09e11 --- /dev/null +++ b/tests/testdata/test_addons/package_files_conflict_test/package/app.manifest @@ -0,0 +1,57 @@ +{ + "schemaVersion": "2.0.0", + "info": { + "title": "This is my add-on", + "id": { + "group": null, + "name": "test_addon", + "version": "0.0.1" + }, + "author": [ + { + "name": "", + "email": null, + "company": null + } + ], + "releaseDate": null, + "description": "This is my add-on", + "classification": { + "intendedAudience": "IT Professionals", + "categories": [ + "Security, Fraud & Compliance" + ], + "developmentStatus": "Production/Stable" + }, + "commonInformationModels": null, + "license": { + "name": null, + "text": "LICENSE.txt", + "uri": null + }, + "privacyPolicy": { + "name": null, + "text": null, + "uri": null + }, + "releaseNotes": { + "name": "README", + "text": "README.txt", + "uri": "" + } + }, + "dependencies": null, + "tasks": null, + "inputGroups": null, + "incompatibleApps": null, + "platformRequirements": null, + "supportedDeployments": [ + "_standalone", + "_distributed", + "_search_head_clustering" + ], + "targetWorkloads": [ + "_search_heads", + "_indexers" + ] +} \ No newline at end of file diff --git a/tests/testdata/test_addons/package_files_conflict_test/package/bin/import_declare_test.py b/tests/testdata/test_addons/package_files_conflict_test/package/bin/import_declare_test.py new file mode 100644 index 000000000..7cef973f9 --- /dev/null +++ b/tests/testdata/test_addons/package_files_conflict_test/package/bin/import_declare_test.py @@ -0,0 +1,12 @@ + +import os +import sys +import re +from os.path import dirname + +ta_name = 'test_addon' +pattern = re.compile(r'[\\/]etc[\\/]apps[\\/][^\\/]+[\\/]bin[\\/]?$') +new_paths = [path for path in sys.path if not pattern.search(path) or ta_name in path] +new_paths.insert(0, os.path.join(dirname(dirname(__file__)), "lib")) +new_paths.insert(0, os.path.sep.join([os.path.dirname(__file__), ta_name])) +sys.path = new_paths diff --git a/tests/testdata/test_addons/package_files_conflict_test/package/bin/my_first_input.py b/tests/testdata/test_addons/package_files_conflict_test/package/bin/my_first_input.py new file mode 100644 index 000000000..27cab33e7 --- /dev/null +++ b/tests/testdata/test_addons/package_files_conflict_test/package/bin/my_first_input.py @@ -0,0 +1,110 @@ +import json +import logging +import sys +import traceback + +import import_declare_test +from solnlib import conf_manager, log +from splunklib import modularinput as smi + +ADDON_NAME = "test_addon" + + +def logger_for_input(input_name: str) -> logging.Logger: + return log.Logs().get_logger(f"{ADDON_NAME.lower()}_{input_name}") + + +def get_account_api_key(session_key: str, account_name: str): + cfm = conf_manager.ConfManager( + session_key, + ADDON_NAME, + realm=f"__REST_CREDENTIAL__#{ADDON_NAME}#configs/conf-test_addon_account", + ) + account_conf_file = cfm.get_conf("test_addon_account") + return account_conf_file.get(account_name).get("api_key") + + +def get_data_from_api(logger: logging.Logger, api_key: str): + logger.info("Getting data from an external API") + dummy_data = [ + { + "line1": "hello", + }, + { + "line2": "world", + }, + ] + return dummy_data + + +class Input(smi.Script): + def __init__(self): + super().__init__() + + def get_scheme(self): + scheme = smi.Scheme("my_first_input") + scheme.description = "my_first_input input" + scheme.use_external_validation = True + scheme.streaming_mode_xml = True + scheme.use_single_instance = False + scheme.add_argument( + smi.Argument( + "name", title="Name", description="Name", required_on_create=True + ) + ) + return scheme + + def validate_input(self, definition: smi.ValidationDefinition): + return + + def stream_events(self, inputs: smi.InputDefinition, event_writer: smi.EventWriter): + # inputs.inputs is a Python dictionary object like: + # { + # "my_first_input://": { + # "account": "", + # "disabled": "0", + # "host": "$decideOnStartup", + # "index": "", + # "interval": "", + # "python.version": "python3", + # }, + # } + for input_name, input_item in inputs.inputs.items(): + normalized_input_name = input_name.split("/")[-1] + logger = logger_for_input(normalized_input_name) + try: + session_key = self._input_definition.metadata["session_key"] + log_level = conf_manager.get_log_level( + logger=logger, + session_key=session_key, + app_name=ADDON_NAME, + conf_name=f"{ADDON_NAME}_settings", + ) + logger.setLevel(log_level) + log.modular_input_start(logger, normalized_input_name) + api_key = get_account_api_key(session_key, input_item.get("account")) + data = get_data_from_api(logger, api_key) + sourcetype = "dummy-data" + for line in data: + event_writer.write_event( + smi.Event( + data=json.dumps(line, ensure_ascii=False, default=str), + index=input_item.get("index"), + sourcetype=sourcetype, + ) + ) + log.events_ingested( + logger, normalized_input_name, sourcetype, len(data) + ) + log.modular_input_end(logger, normalized_input_name) + except Exception as e: + logger.error( + f"Exception raised while ingesting data for " + f"my_first_input: {e}. Traceback: " + f"{traceback.format_exc()}" + ) + + +if __name__ == "__main__": + exit_code = Input().run(sys.argv) + sys.exit(exit_code) diff --git a/tests/testdata/test_addons/package_files_conflict_test/package/bin/test_addon_rh_account.py b/tests/testdata/test_addons/package_files_conflict_test/package/bin/test_addon_rh_account.py new file mode 100644 index 000000000..601c0024e --- /dev/null +++ b/tests/testdata/test_addons/package_files_conflict_test/package/bin/test_addon_rh_account.py @@ -0,0 +1,41 @@ + +import import_declare_test + +from splunktaucclib.rest_handler.endpoint import ( + field, + validator, + RestModel, + SingleModel, +) +from splunktaucclib.rest_handler import admin_external, util +from splunktaucclib.rest_handler.admin_external import AdminExternalHandler +import logging + +util.remove_http_proxy_env_vars() + + +fields = [ + field.RestField( + 'api_key', + required=True, + encrypted=True, + default=None, + validaator=None + ) +] +model = RestModel(fields, name=None) + + +endpoint = SingleModel( + 'test_addon_account', + model, + config_name='account' +) + + +if __name__ == '__main__': + logging.getLogger().addHandler(logging.NullHandler()) + admin_external.handle( + endpoint, + handler=AdminExternalHandler, + ) diff --git a/tests/testdata/test_addons/package_files_conflict_test/package/lib/requirements.txt b/tests/testdata/test_addons/package_files_conflict_test/package/lib/requirements.txt new file mode 100644 index 000000000..725958191 --- /dev/null +++ b/tests/testdata/test_addons/package_files_conflict_test/package/lib/requirements.txt @@ -0,0 +1,2 @@ +splunktaucclib + diff --git a/tests/testdata/test_addons/package_global_config_everything/globalConfig.json b/tests/testdata/test_addons/package_global_config_everything/globalConfig.json index f15aa1150..5bdf62a5a 100644 --- a/tests/testdata/test_addons/package_global_config_everything/globalConfig.json +++ b/tests/testdata/test_addons/package_global_config_everything/globalConfig.json @@ -1338,7 +1338,7 @@ "meta": { "name": "Splunk_TA_UCCExample", "restRoot": "splunk_ta_uccexample", - "version": "5.32.0R3be5ef5e", + "version": "5.34.1Racdcfb2e", "displayName": "Splunk UCC test Add-on", "schemaVersion": "0.0.3" } diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index f14c81899..a238bb749 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -16,6 +16,7 @@ "addon_version": None, "output_directory": None, "python_binary_name": "python3", + "verbose_file_summary_report": False, }, ), ( @@ -26,6 +27,7 @@ "addon_version": None, "output_directory": None, "python_binary_name": "python3", + "verbose_file_summary_report": False, }, ), ( @@ -36,6 +38,7 @@ "addon_version": None, "output_directory": None, "python_binary_name": "python3", + "verbose_file_summary_report": False, }, ), ( @@ -46,16 +49,18 @@ "addon_version": None, "output_directory": None, "python_binary_name": "python3", + "verbose_file_summary_report": False, }, ), ( - ["--source", "package", "--ta-version", "2.1.0"], + ["-v", "--source", "package", "--ta-version", "2.1.0"], { "source": "package", "config_path": None, "addon_version": "2.1.0", "output_directory": None, "python_binary_name": "python3", + "verbose_file_summary_report": True, }, ), ( @@ -73,6 +78,7 @@ "addon_version": "2.2.0", "output_directory": None, "python_binary_name": "python.exe", + "verbose_file_summary_report": False, }, ), ( @@ -92,6 +98,7 @@ "addon_version": "2.2.0", "output_directory": None, "python_binary_name": "python.exe", + "verbose_file_summary_report": False, }, ), ( @@ -113,6 +120,7 @@ "addon_version": "2.2.0", "output_directory": "new_output", "python_binary_name": "python.exe", + "verbose_file_summary_report": False, }, ), ( @@ -135,11 +143,13 @@ "addon_version": "2.2.0", "output_directory": "new_output", "python_binary_name": "python.exe", + "verbose_file_summary_report": False, }, ), ( [ "build", + "-v", "--source", "package", "--config", @@ -157,6 +167,7 @@ "addon_version": "2.2.0", "output_directory": "new_output", "python_binary_name": "python.exe", + "verbose_file_summary_report": True, }, ), ],