From 5e6be9d1112942994ffd17758cda02606b52d0f0 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sat, 6 Jul 2024 21:41:52 +0100 Subject: [PATCH] Enable dbconfig as part of defaults within setup --- redisbench_admin/run/common.py | 34 +++++- redisbench_admin/run/run.py | 38 +++++-- redisbench_admin/utils/benchmark_config.py | 10 +- tests/test_data/defaults-with-dbconfig.yml | 12 ++- tests/test_run.py | 116 ++++++++++++++------- 5 files changed, 162 insertions(+), 48 deletions(-) diff --git a/redisbench_admin/run/common.py b/redisbench_admin/run/common.py index 51d722e..57dd252 100644 --- a/redisbench_admin/run/common.py +++ b/redisbench_admin/run/common.py @@ -499,6 +499,36 @@ def extract_test_feasible_setups( for setup_name in feasible_setups_list: if default_specs is not None: feasible_setups_map[setup_name] = {} + # spec: + # setups: + # - name: oss-standalone + # type: oss-standalone + # redis_topology: + # primaries: 1 + # replicas: 1 + # placement: "sparse" + # resources: + # requests: + # cpus: "2" + # memory: "10g" + # - name: oss-standalone-threads-6 + # type: oss-standalone + # redis_topology: + # primaries: 1 + # replicas: 1 + # placement: "sparse" + # resources: + # requests: + # cpus: "2" + # memory: "10g" + # dbconfig: + # module-configuration-parameters: + # redisearch: + # WORKERS: 6 + # MIN_OPERATION_WORKERS: 6 + # module-oss: + # WORKERS: 6 + # MIN_OPERATION_WORKERS: 6 if "setups" in default_specs: for setup in default_specs["setups"]: if setup_name == setup["name"]: @@ -530,7 +560,9 @@ def extract_test_feasible_setups( feasible_setups_map[setup_name] ) ) - + logging.info( + f"There a total of {len(feasible_setups_map.keys())} setups. Setups: {feasible_setups_map}" + ) return feasible_setups_map diff --git a/redisbench_admin/run/run.py b/redisbench_admin/run/run.py index 2f2bee6..eb46021 100644 --- a/redisbench_admin/run/run.py +++ b/redisbench_admin/run/run.py @@ -4,6 +4,7 @@ # All rights reserved. # import logging +import copy from redisbench_admin.run.common import extract_test_feasible_setups from redisbench_admin.run_remote.consts import min_recommended_benchmark_duration @@ -31,7 +32,7 @@ def calculate_client_tool_duration_and_check( def merge_dicts(dict1, dict2): - result = dict1.copy() # Start with dict1's keys and values + result = copy.deepcopy(dict1) # Start with dict1's keys and values for key, value in dict2.items(): if key in result: if isinstance(result[key], dict) and isinstance(value, dict): @@ -56,8 +57,11 @@ def define_benchmark_plan(benchmark_definitions, default_specs): benchmark_runs_plan[benchmark_type] = {} # extract dataset-name - dbconfig_present, dataset_name, _, _, _ = extract_redis_dbconfig_parameters( - benchmark_config, "dbconfig" + benchmark_contains_dbconfig, dataset_name, _, _, _ = ( + extract_redis_dbconfig_parameters(benchmark_config, "dbconfig") + ) + logging.info( + f"Benchmark contains specific dbconfig on test {test_name}: {benchmark_contains_dbconfig}" ) if dataset_name is None: dataset_name = test_name @@ -74,6 +78,13 @@ def define_benchmark_plan(benchmark_definitions, default_specs): ) for setup_name, setup_settings in test_setups.items(): + setup_contains_dbconfig = False + if "dbconfig" in setup_settings: + setup_contains_dbconfig = True + logging.error( + f"setup ({setup_name}): {setup_settings}. contains dbconfig {setup_contains_dbconfig}" + ) + if setup_name not in benchmark_runs_plan[benchmark_type][dataset_name]: benchmark_runs_plan[benchmark_type][dataset_name][setup_name] = {} benchmark_runs_plan[benchmark_type][dataset_name][setup_name][ @@ -82,6 +93,7 @@ def define_benchmark_plan(benchmark_definitions, default_specs): benchmark_runs_plan[benchmark_type][dataset_name][setup_name][ "benchmarks" ] = {} + if ( test_name in benchmark_runs_plan[benchmark_type][dataset_name][setup_name][ @@ -94,11 +106,23 @@ def define_benchmark_plan(benchmark_definitions, default_specs): ) ) else: - # add benchmark - if "dbconfig" in setup_settings: - benchmark_config["dbconfig"] = merge_dicts( - benchmark_config["dbconfig"], setup_settings["dbconfig"] + # check if we need to merge dbconfigs from the setup defaults + if setup_contains_dbconfig: + if "dbconfig" not in benchmark_config: + benchmark_config["dbconfig"] = {} + setup_dbconfig = setup_settings["dbconfig"] + benchmark_dbconfig = benchmark_config["dbconfig"] + logging.info( + f"Merging setup dbconfig: {setup_dbconfig}, with benchmark dbconfig {benchmark_dbconfig}" ) + final_db_config = merge_dicts(benchmark_dbconfig, setup_dbconfig) + logging.info(f"FINAL DB CONFIG: {final_db_config}") + benchmark_config["dbconfig"] = final_db_config + + logging.info( + f"final benchmark config for setup: {setup_name} and test: {test_name}. {benchmark_config}" + ) + # add benchmark benchmark_runs_plan[benchmark_type][dataset_name][setup_name][ "benchmarks" ][test_name] = benchmark_config diff --git a/redisbench_admin/utils/benchmark_config.py b/redisbench_admin/utils/benchmark_config.py index 2404dee..807d56c 100644 --- a/redisbench_admin/utils/benchmark_config.py +++ b/redisbench_admin/utils/benchmark_config.py @@ -258,10 +258,12 @@ def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname): cp = benchmark_config[dbconfig_keyname]["configuration-parameters"] for k, v in cp.items(): redis_configuration_parameters[k] = v - if "dataset_load_timeout_secs" in cp: - dataset_load_timeout_secs = cp["dataset_load_timeout_secs"] - if "dataset_name" in cp: - dataset_name = cp["dataset_name"] + if "dataset_load_timeout_secs" in benchmark_config[dbconfig_keyname]: + dataset_load_timeout_secs = benchmark_config[dbconfig_keyname][ + "dataset_load_timeout_secs" + ] + if "dataset_name" in benchmark_config[dbconfig_keyname]: + dataset_name = benchmark_config[dbconfig_keyname]["dataset_name"] return ( dbconfig_present, diff --git a/tests/test_data/defaults-with-dbconfig.yml b/tests/test_data/defaults-with-dbconfig.yml index 7b0fe26..d4330a7 100644 --- a/tests/test_data/defaults-with-dbconfig.yml +++ b/tests/test_data/defaults-with-dbconfig.yml @@ -21,7 +21,7 @@ exporter: baseline-branch: master spec: setups: - - name: oss-standalone-threads-6 + - name: oss-standalone type: oss-standalone redis_topology: primaries: 1 @@ -31,6 +31,16 @@ spec: requests: cpus: "2" memory: "10g" + - name: oss-standalone-threads-6 + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 1 + placement: "sparse" + resources: + requests: + cpus: "7" + memory: "10g" dbconfig: module-configuration-parameters: redisearch: diff --git a/tests/test_run.py b/tests/test_run.py index 388c0d0..183ed6e 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -9,6 +9,7 @@ from redisbench_admin.run.run import calculate_client_tool_duration_and_check from redisbench_admin.run.run import define_benchmark_plan +from redisbench_admin.utils.benchmark_config import process_default_yaml_properties_file def test_calculate_client_tool_duration_and_check(): @@ -25,46 +26,91 @@ def test_calculate_client_tool_duration_and_check(): def test_define_benchmark_plan(): benchmark_definitions = { "test1": { - "type": "performance", "dbconfig": {"dataset_name": "dataset1"}, - "setups": { - "setup1": {"config1": "value1", "dbconfig": {"host": "localhost"}} - }, - } + "setups": ["oss-standalone", "oss-standalone-threads-6"], + }, + "test2": { + "setups": ["oss-standalone", "oss-standalone-threads-6"], + }, } - with open("./tests/test_data/defaults-with-dbconfig.yml") as yaml_fd: - defaults_dict = yaml.safe_load(yaml_fd) + default_specs = {} + with open("./tests/test_data/defaults-with-dbconfig.yml", "r") as yml_file: + ( + _, + _, + _, + _, + default_specs, + _, + ) = process_default_yaml_properties_file( + None, None, None, "1.yml", None, yml_file + ) expected_output = { - "performance": { - "dataset1": { - "setup1": { - "setup_settings": { - "config1": "value1", - "dbconfig": {"host": "localhost"}, - }, - "benchmarks": { - "test1": { - "type": "performance", - "dbconfig": { - "dataset_name": "dataset1", - "host": "localhost", - }, - "setups": { - "setup1": { - "config1": "value1", - "dbconfig": {"host": "localhost"}, - } - }, - } - }, - } - } - } + # "performance": { + # "dataset1": { + # "setup1": { + # "setup_settings": { + # "config1": "value1", + # "dbconfig": {"host": "localhost"}, + # }, + # "benchmarks": { + # "test1": { + # "type": "performance", + # "dbconfig": { + # "dataset_name": "dataset1", + # "host": "localhost", + # }, + # "setups": { + # "setup1": { + # "config1": "value1", + # "dbconfig": {"host": "localhost"}, + # } + # }, + # } + # }, + # } + # } + # } } - output = define_benchmark_plan(benchmark_definitions, defaults_dict) + benchmark_plan = define_benchmark_plan(benchmark_definitions, default_specs) - assert True + # ensure we merge properly the 2 configs + assert ( + benchmark_plan["mixed"]["dataset1"]["oss-standalone"]["benchmarks"]["test1"][ + "dbconfig" + ]["dataset_name"] + == "dataset1" + ) + + assert ( + benchmark_plan["mixed"]["dataset1"]["oss-standalone"]["benchmarks"]["test1"][ + "dbconfig" + ]["dataset_name"] + == "dataset1" + ) + assert ( + benchmark_plan["mixed"]["dataset1"]["oss-standalone-threads-6"]["benchmarks"][ + "test1" + ]["dbconfig"]["dataset_name"] + == "dataset1" + ) + assert ( + benchmark_plan["mixed"]["dataset1"]["oss-standalone-threads-6"]["benchmarks"][ + "test1" + ]["dbconfig"]["module-configuration-parameters"]["redisearch"]["WORKERS"] + == 6 + ) + assert "module-configuration-parameters" not in ( + benchmark_plan["mixed"]["dataset1"]["oss-standalone"]["benchmarks"]["test1"][ + "dbconfig" + ] + ) + # 'module-configuration-parameters' + + # assert True # TODO: add this check when the feature is ready - # output == expected_output, f"Expected {expected_output}, but got {output}" + # assert ( + # benchmark_plan == expected_output + # ), f"Expected {expected_output}, but got {benchmark_plan}"