Modify config semantic validation of benchmark engine

Pass to validation the full benchmark configuration instead of
extracting parts of it (ex. passing only args).

Change-Id: I8f4cf3e2e8a0392612a0d0828a9c3bd6ab3da90f

rally/benchmark/engine.py

@@ -125,17 +125,16 @@ def _validate_config_syntax(self, config):
 
     def _validate_config_semantic_helper(self, admin, user, name, pos,
                                          task, kwargs):
-        args = {} if not kwargs else kwargs.get("args", {})
         context = {} if not kwargs else kwargs.get("context", {})
 
         try:
-            base_scenario.Scenario.validate(name, args, admin=admin,
+            base_scenario.Scenario.validate(name, kwargs, admin=admin,
                                             users=[user], task=task)
             base_ctx.ContextManager.validate_semantic(context, admin=admin,
                                                       users=[user], task=task)
         except exceptions.InvalidScenarioArgument as e:
             kw = {"name": name, "pos": pos,
-                  "args": args, "reason": six.text_type(e)}
+                  "config": kwargs, "reason": six.text_type(e)}
             raise exceptions.InvalidBenchmarkConfig(**kw)
 
     @rutils.log_task_wrapper(LOG.info, _("Task validation of semantic."))

rally/benchmark/scenarios/base.py

@@ -99,14 +99,14 @@ def list_benchmark_scenarios():
         return benchmark_scenarios_flattened
 
     @staticmethod
-    def _validate_helper(validators, clients, args, task):
+    def _validate_helper(validators, clients, config, task):
         for validator in validators:
-            result = validator(clients=clients, task=task, **args)
+            result = validator(config, clients=clients, task=task)
             if not result.is_valid:
                 raise exceptions.InvalidScenarioArgument(message=result.msg)
 
     @classmethod
-    def validate(cls, name, args, admin=None, users=None, task=None):
+    def validate(cls, name, config, admin=None, users=None, task=None):
         """Semantic check of benchmark arguments."""
         validators = cls.meta(name, "validators", default=[])
 
@@ -121,10 +121,10 @@ def validate(cls, name, args, admin=None, users=None, task=None):
         # NOTE(boris-42): Potential bug, what if we don't have "admin" client
         #                 and scenario have "admin" validators.
         if admin:
-            cls._validate_helper(admin_validators, admin, args, task)
+            cls._validate_helper(admin_validators, admin, config, task)
         if users:
             for user in users:
-                cls._validate_helper(user_validators, user, args, task)
+                cls._validate_helper(user_validators, user, config, task)
 
     @staticmethod
     def meta(cls, attr_name, method_name=None, default=None):

rally/benchmark/validation.py

@@ -41,10 +41,17 @@ def validator(fn):
     :returns: rally scenario validator
     """
     def wrap_given(*args, **kwargs):
-        def wrap_validator(**options):
-            options.update({"args": args, "kwargs": kwargs})
+        """Dynamic validation decorator for scenario.
+
+        :param args: the arguments of the decorator of the benchmark scenario
+        ex. @my_decorator("arg1"), then args = ('arg1',)
+        :param kwargs: the keyword arguments of the decorator of the scenario
+        ex. @my_decorator(kwarg1="kwarg1"), then kwargs = {"kwarg1": "kwarg1"}
+        """
+        def wrap_validator(config, clients, task):
             # NOTE(amaretskiy): validator is successful by default
-            return fn(*args, **options) or ValidationResult()
+            return (fn(config, clients, task, *args, **kwargs) or
+                    ValidationResult())
 
         def wrap_scenario(scenario):
             # NOTE(amaretskiy): user permission by default
@@ -95,13 +102,13 @@ def number(param_name=None, minval=None, maxval=None, nullable=False,
     :param integer_only: Only accept integers
     """
 
-    def number_validator(**kwargs):
+    def number_validator(config, clients, task):
 
         num_func = float
         if integer_only:
             num_func = int
 
-        val = kwargs.get(param_name, None)
+        val = config.get("args", {}).get(param_name)
 
         # None may be valid if the scenario sets a sensible default.
         if nullable and val is None:
@@ -153,8 +160,8 @@ def file_exists(param_name, mode=os.R_OK):
             mode=os.R_OK+os.W_OK
     """
 
-    def file_exists_validator(**kwargs):
-        file_name = kwargs.get(param_name)
+    def file_exists_validator(config, clients, task):
+        file_name = config.get("args", {}).get(param_name)
         if os.access(file_name, mode):
             return ValidationResult()
         else:
@@ -173,11 +180,10 @@ def image_exists(param_name):
     :param param_name: defines which variable should be used
                        to get image id value.
     """
-    def image_exists_validator(**kwargs):
-        clients = kwargs.get('clients')
+    def image_exists_validator(config, clients, task):
         image_id = types.ImageResourceType.transform(
             clients=clients,
-            resource_config=kwargs.get(param_name))
+            resource_config=config.get("args", {}).get(param_name))
         try:
             clients.glance().images.get(image=image_id)
             return ValidationResult()
@@ -193,11 +199,10 @@ def flavor_exists(param_name):
     :param param_name: defines which variable should be used
                        to get flavor id value.
     """
-    def flavor_exists_validator(**kwargs):
-        clients = kwargs.get('clients')
+    def flavor_exists_validator(config, clients, task):
         flavor_id = types.FlavorResourceType.transform(
             clients=clients,
-            resource_config=kwargs.get(param_name))
+            resource_config=config.get("args", {}).get(param_name))
         try:
             clients.nova().flavors.get(flavor=flavor_id)
             return ValidationResult()
@@ -216,12 +221,10 @@ def image_valid_on_flavor(flavor_name, image_name):
                        to get image id value.
 
     """
-    def image_valid_on_flavor_validator(**kwargs):
-        clients = kwargs.get('clients')
-
+    def image_valid_on_flavor_validator(config, clients, task):
         flavor_id = types.FlavorResourceType.transform(
             clients=clients,
-            resource_config=kwargs.get(flavor_name))
+            resource_config=config.get("args", {}).get(flavor_name))
         try:
             flavor = clients.nova().flavors.get(flavor=flavor_id)
         except nova_exc.NotFound:
@@ -230,7 +233,7 @@ def image_valid_on_flavor_validator(**kwargs):
 
         image_id = types.ImageResourceType.transform(
             clients=clients,
-            resource_config=kwargs.get(image_name))
+            resource_config=config.get("args", {}).get(image_name))
         try:
             image = clients.glance().images.get(image=image_id)
         except glance_exc.HTTPNotFound:
@@ -257,11 +260,11 @@ def image_valid_on_flavor_validator(**kwargs):
 
 
 def network_exists(network_name):
-    def network_exists_validator(**kwargs):
-        network = kwargs.get(network_name, "private")
+    def network_exists_validator(config, clients, task):
+        network = config.get("args", {}).get(network_name, "private")
 
         networks = [net.label for net in
-                    kwargs["clients"].nova().networks.list()]
+                    clients.nova().networks.list()]
         if network not in networks:
             message = _("Network with name %(network)s not found. "
                         "Available networks: %(networks)s") % {
@@ -275,14 +278,14 @@ def network_exists_validator(**kwargs):
 
 
 def external_network_exists(network_name, use_external_network):
-    def external_network_exists_validator(**kwargs):
-        if not kwargs.get(use_external_network, True):
+    def external_network_exists_validator(config, clients, task):
+        if not config.get("args", {}).get(use_external_network, True):
             return ValidationResult()
 
-        ext_network = kwargs.get(network_name, "public")
+        ext_network = config.get("args", {}).get(network_name, "public")
 
         networks = [net.name for net in
-                    kwargs["clients"].nova().floating_ip_pools.list()]
+                    clients.nova().floating_ip_pools.list()]
         if ext_network not in networks:
             message = _("External (floating) network with name %(network)s "
                         "not found. "
@@ -345,8 +348,8 @@ def required_parameters(params):
 
     :param params: list of required parameters
     """
-    def required_parameters_validator(**kwargs):
-        missing = set(params) - set(kwargs)
+    def required_parameters_validator(config, clients, task):
+        missing = set(params) - set(config.get("args", {}))
         if missing:
             message = _("%s parameters are not defined in "
                         "the benchmark config file") % ", ".join(missing)
@@ -356,13 +359,13 @@ def required_parameters_validator(**kwargs):
 
 
 @validator
-def required_services(*args, **kwargs):
+def required_services(config, clients, task, *required_services):
     """Check if specified services are available.
 
     :param args: list of servives names
     """
-    available_services = kwargs.get("clients").services().values()
-    for service in args:
+    available_services = clients.services().values()
+    for service in required_services:
         if service not in consts.Service:
             return ValidationResult(False, _("Unknown service: %s") % service)
         if service not in available_services:

rally/exceptions.py

@@ -110,7 +110,8 @@ class InvalidBenchmarkConfig(InvalidTaskException):
     msg_fmt = _("Task config is invalid.\n"
                 "\tBenchmark %(name)s has wrong configuration at"
                 " position %(pos)s"
-                "\n\tReason: %(reason)s")
+                "\n\tReason: %(reason)s"
+                "\n\tBenchmark configuration: %(config)s")
 
 
 class TestException(RallyException):

tests/benchmark/scenarios/test_base.py

@@ -50,12 +50,12 @@ def test__validate_helper(self):
             mock.MagicMock(return_value=validation.ValidationResult())
         ]
         clients = mock.MagicMock()
-        args = {"a": 1, "b": 2}
+        config = {"a": 1, "b": 2}
         task = mock.MagicMock()
         scenario_base.Scenario._validate_helper(validators, clients,
-                                                args, task)
+                                                config, task)
         for validator in validators:
-            validator.assert_called_with(clients=clients, task=task, **args)
+            validator.assert_called_with(config, clients=clients, task=task)
 
     def test__validate_helper__no_valid(self):
         validators = [

tests/benchmark/test_engine.py

@@ -163,8 +163,7 @@ def test__validate_config_semantic_helper(self, mock_validate):
         eng._validate_config_semantic_helper("admin", "user", "name", "pos",
                                              task, {"args": "args"})
         mock_validate.assert_called_once_with(
-            "name", "args", admin="admin", users=["user"],
-            task=task)
+            "name", {"args": "args"}, admin="admin", users=["user"], task=task)
 
     @mock.patch("rally.benchmark.engine.base_scenario.Scenario.validate")
     def test__validate_config_semanitc_helper_invalid_arg(self, mock_validate):