Support for MAP type model parameters (0.7 back port)

tracdap-runtime/python/src/tracdap/rt/_exec/dev_mode.py

@@ -37,12 +37,12 @@
     re.compile(r"job\.\w+\.model"),
     re.compile(r"job\.\w+\.flow"),
 
-    re.compile(r".*\.jobs\.\d+\.\w+\.parameters\.\w+"),
-    re.compile(r".*\.jobs\.\d+\.\w+\.inputs\.\w+"),
-    re.compile(r".*\.jobs\.\d+\.\w+\.outputs\.\w+"),
-    re.compile(r".*\.jobs\.\d+\.\w+\.models\.\w+"),
-    re.compile(r".*\.jobs\.\d+\.\w+\.model"),
-    re.compile(r".*\.jobs\.\d+\.\w+\.flow")
+    re.compile(r".*\.jobs\[\d+]\.\w+\.parameters\.\w+"),
+    re.compile(r".*\.jobs\[\d+]\.\w+\.inputs\.\w+"),
+    re.compile(r".*\.jobs\[\d+]\.\w+\.outputs\.\w+"),
+    re.compile(r".*\.jobs\[\d+]\.\w+\.models\.\w+"),
+    re.compile(r".*\.jobs\[\d+]\.\w+\.model"),
+    re.compile(r".*\.jobs\[\d+]\.\w+\.flow")
 ]
 
 DEV_MODE_SYS_CONFIG = []
@@ -764,10 +764,15 @@ def _process_parameters_dict(
             else:
                 p_spec = param_specs[p_name]
 
-                cls._log.info(f"Encoding parameter [{p_name}] as {p_spec.paramType.basicType}")
+                try:
+                    cls._log.info(f"Encoding parameter [{p_name}] as {p_spec.paramType.basicType}")
+                    encoded_value = _types.MetadataCodec.convert_value(p_value, p_spec.paramType)
+                    encoded_values[p_name] = encoded_value
 
-                encoded_value = _types.MetadataCodec.convert_value(p_value, p_spec.paramType)
-                encoded_values[p_name] = encoded_value
+                except Exception as e:
+                    msg = f"Failed to encode parameter [{p_name}]: {str(e)}"
+                    cls._log.error(msg)
+                    raise _ex.EConfigParse(msg) from e
 
         return encoded_values
 

tracdap-runtime/python/src/tracdap/rt/_impl/config_parser.py

@@ -321,6 +321,23 @@ def parse(self, config_dict: dict, config_file: tp.Union[str, pathlib.Path] = No
 
     def _parse_value(self, location: str, raw_value: tp.Any, annotation: type):
 
+        if self._is_dev_mode_location(location):
+
+            if type(raw_value) in ConfigParser.__primitive_types:
+                return self._parse_primitive(location, raw_value, type(raw_value))
+
+            if isinstance(raw_value, list):
+                if len(raw_value) == 0:
+                    return []
+                items = iter((self._child_location(location, i), x) for i, x in enumerate(raw_value))
+                return list(self._parse_value(loc, x, tp.Any) for loc, x in items)
+
+            if isinstance(raw_value, dict):
+                if len(raw_value) == 0:
+                    return {}
+                items = iter((self._child_location(location, k), k, v) for k, v in raw_value.items())
+                return dict((k, self._parse_value(loc, v, tp.Any)) for loc, k, v in items)
+
         if raw_value is None:
             return None
 
@@ -339,24 +356,13 @@ def _parse_value(self, location: str, raw_value: tp.Any, annotation: type):
             return self._parse_enum(location, raw_value, annotation)
 
         if _dc.is_dataclass(annotation):
-
-            if isinstance(raw_value, tp.Dict):
-                return self._parse_simple_class(location, raw_value, annotation)
-
-            if self._is_dev_mode_location(location):
-                if type(raw_value) in ConfigParser.__primitive_types:
-                    return self._parse_primitive(location, raw_value, type(raw_value))
-                if isinstance(raw_value, list):
-                    if len(raw_value) == 0:
-                        return []
-                    list_type = type(raw_value[0])
-                    return list(map(lambda x: self._parse_primitive(location, x, list_type), raw_value))
-
-            return self._error(location, f"Expected type {annotation.__name__}, got '{str(raw_value)}'")
+            return self._parse_simple_class(location, raw_value, annotation)
 
         if isinstance(annotation, self.__generic_metaclass):
             return self._parse_generic_class(location, raw_value, annotation)  # noqa
 
+        return self._error(location, f"Cannot parse value of type {annotation.__name__}")
+
     def _is_dev_mode_location(self, location):
 
         return any(map(lambda pattern: re.match(pattern, location), self._dev_mode_locations))
@@ -416,7 +422,7 @@ def _parse_enum_value(raw_value: str, metaclass: enum.EnumMeta):
     def _parse_simple_class(self, location: str, raw_dict: tp.Any, metaclass: type) -> object:
 
         if raw_dict is not None and not isinstance(raw_dict, dict):
-            pass
+            return self._error(location, f"Expected type {metaclass.__name__}, got '{str(raw_dict)}'")
 
         obj = metaclass.__new__(metaclass, object())  # noqa
 
@@ -510,7 +516,7 @@ def _parse_generic_class(self, location: str, raw_value: tp.Any, metaclass:  __g
                 return self._error(location, f"Expected a list, got {type(raw_value)}")
 
             return [
-                self._parse_value(self._child_location(location, str(idx)), item, list_type)
+                self._parse_value(self._child_location(location, idx), item, list_type)
                 for (idx, item) in enumerate(raw_value)]
 
         if origin == tp.Dict or origin == dict:
@@ -541,12 +547,14 @@ def _error(self, location: str, error: str) -> None:
         return None
 
     @staticmethod
-    def _child_location(parent_location: str, item: str):
+    def _child_location(parent_location: str, item: tp.Union[str, int]):
 
         if parent_location is None or parent_location == "":
             return item
+        elif isinstance(item, int):
+            return f"{parent_location}[{item}]"
         else:
-            return parent_location + "." + item
+            return f"{parent_location}.{item}"
 
 
 class ConfigQuoter:

tracdap-runtime/python/src/tracdap/rt/_impl/static_api.py

@@ -55,7 +55,7 @@ def map_type(self, entry_type: _meta.BasicType) -> _meta.TypeDescriptor:
         if not _val.is_primitive_type(entry_type):
             raise _ex.EModelValidation(f"Maps can only contain primitive types, [{entry_type}] is not primitive")
 
-        return _meta.TypeDescriptor(_meta.BasicType.MAP, arrayType=_meta.TypeDescriptor(entry_type))
+        return _meta.TypeDescriptor(_meta.BasicType.MAP, mapType=_meta.TypeDescriptor(entry_type))
 
     def define_attribute(
             self, attr_name: str, attr_value: _tp.Any,

tracdap-runtime/python/src/tracdap/rt/_impl/type_system.py

@@ -133,9 +133,13 @@ def _decode_value_for_type(value: _meta.Value, type_desc: _meta.TypeDescriptor):
 
         if basic_type == _meta.BasicType.ARRAY:
             items = value.arrayValue.items
-            return list(map(lambda x: MetadataCodec._decode_value_for_type(x, type_desc.arrayType), items))
+            return list(MetadataCodec._decode_value_for_type(x, type_desc.arrayType) for x in items)
 
-        raise _ex.ETracInternal(f"Decoding value type [{basic_type}] is not supported yet")
+        if basic_type == _meta.BasicType.MAP:
+            items = value.mapValue.entries.items()
+            return dict((k, MetadataCodec._decode_value_for_type(v, type_desc.mapType)) for k, v in items)
+
+        raise _ex.ETracInternal(f"Cannot decode value of type [{basic_type}]")
 
     @classmethod
     def encode_value(cls, value: tp.Any) -> _meta.Value:
@@ -183,19 +187,36 @@ def encode_value(cls, value: tp.Any) -> _meta.Value:
             if any(map(lambda x: type(x) != array_raw_type, value)):
                 raise _ex.ETracInternal("Cannot encode a list with values of different types")
 
-            encoded_items = list(map(lambda x: cls.convert_value(x, array_trac_type), value))
+            encoded_items = list(map(lambda x: cls.convert_value(x, array_trac_type, True), value))
 
             return _meta.Value(
                 _meta.TypeDescriptor(_meta.BasicType.ARRAY, arrayType=array_trac_type),
                 arrayValue=_meta.ArrayValue(encoded_items))
 
-        raise _ex.ETracInternal(f"Value type [{type(value)}] is not supported yet")
+        if isinstance(value, dict):
+
+            if len(value) == 0:
+                raise _ex.ETracInternal("Cannot encode an empty dict")
+
+            map_raw_type = type(next(iter(value.values())))
+            map_trac_type = TypeMapping.python_to_trac(map_raw_type)
+
+            if any(map(lambda x: type(x) != array_raw_type, value.values())):
+                raise _ex.ETracInternal("Cannot encode a dict with values of different types")
+
+            encoded_entries = dict(map(lambda kv: (kv[0], cls.convert_value(kv[1], map_trac_type, True)), value.items()))
+
+            return _meta.Value(
+                _meta.TypeDescriptor(_meta.BasicType.ARRAY, mapType=map_trac_type),
+                mapValue=_meta.MapValue(encoded_entries))
+
+        raise _ex.ETracInternal(f"Cannot encode value of type [{type(value).__name__}]")
 
     @classmethod
-    def convert_value(cls, raw_value: tp.Any, type_desc: _meta.TypeDescriptor):
+    def convert_value(cls, raw_value: tp.Any, type_desc: _meta.TypeDescriptor, nested: bool = False):
 
         if type_desc.basicType == _meta.BasicType.BOOLEAN:
-            return cls.convert_boolean_value(raw_value)
+            return cls.convert_boolean_value(raw_value, nested)
 
         if type_desc.basicType == _meta.BasicType.INTEGER:
             return cls.convert_integer_value(raw_value)
@@ -218,78 +239,97 @@ def convert_value(cls, raw_value: tp.Any, type_desc: _meta.TypeDescriptor):
         if type_desc.basicType == _meta.BasicType.ARRAY:
             return cls.convert_array_value(raw_value, type_desc.arrayType)
 
+        if type_desc.basicType == _meta.BasicType.MAP:
+            return cls.convert_map_value(raw_value, type_desc.mapType)
+
         raise _ex.ETracInternal(f"Conversion to value type [{type_desc.basicType.name}] is not supported yet")
 
     @staticmethod
     def convert_array_value(raw_value: tp.List[tp.Any], array_type: _meta.TypeDescriptor) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.ARRAY, array_type)
+        type_desc = _meta.TypeDescriptor(basicType=_meta.BasicType.ARRAY, arrayType=array_type)
 
         if not isinstance(raw_value, list):
-            msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.ARRAY.name}"
+            msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.ARRAY.name}"
             raise _ex.ETracInternal(msg)
 
-        items = list(map(lambda x: MetadataCodec.convert_value(x, array_type), raw_value))
+        items = list(map(lambda x: MetadataCodec.convert_value(x, array_type, True), raw_value))
 
         return _meta.Value(type_desc, arrayValue=_meta.ArrayValue(items))
 
     @staticmethod
-    def convert_boolean_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_map_value(raw_value: tp.Dict[str, tp.Any], map_type: _meta.TypeDescriptor) -> _meta.Value:
+
+        type_desc = _meta.TypeDescriptor(basicType=_meta.BasicType.MAP, mapType=map_type)
+
+        if not isinstance(raw_value, dict):
+            msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.MAP.name}"
+            raise _ex.ETracInternal(msg)
+
+        entries = dict(map(lambda kv: (kv[0], MetadataCodec.convert_value(kv[1], map_type, True)), raw_value.items()))
+
+        return _meta.Value(type_desc, mapValue=_meta.MapValue(entries))
+
+    @staticmethod
+    def convert_boolean_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.BOOLEAN)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.BOOLEAN) if not nested else None
 
         if isinstance(raw_value, bool):
             return _meta.Value(type_desc, booleanValue=raw_value)
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.BOOLEAN.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.BOOLEAN.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_integer_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_integer_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.INTEGER)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.INTEGER) if not nested else None
 
-        if isinstance(raw_value, int):
+        # isinstance(bool_value, int) returns True! An explicit check is needed
+        if isinstance(raw_value, int) and not isinstance(raw_value, bool):
             return _meta.Value(type_desc, integerValue=raw_value)
 
         if isinstance(raw_value, float) and raw_value.is_integer():
             return _meta.Value(type_desc, integerValue=int(raw_value))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.INTEGER.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.INTEGER.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_float_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_float_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.FLOAT)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.FLOAT) if not nested else None
 
         if isinstance(raw_value, float):
             return _meta.Value(type_desc, floatValue=raw_value)
 
-        if isinstance(raw_value, int):
+        # isinstance(bool_value, int) returns True! An explicit check is needed
+        if isinstance(raw_value, int) and not isinstance(raw_value, bool):
             return _meta.Value(type_desc, floatValue=float(raw_value))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.FLOAT.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.FLOAT.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_decimal_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_decimal_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.DECIMAL)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.DECIMAL) if not nested else None
 
         if isinstance(raw_value, decimal.Decimal):
             return _meta.Value(type_desc, decimalValue=_meta.DecimalValue(str(raw_value)))
 
-        if isinstance(raw_value, int) or isinstance(raw_value, float):
+        # isinstance(bool_value, int) returns True! An explicit check is needed
+        if isinstance(raw_value, int) or isinstance(raw_value, float) and not isinstance(raw_value, bool):
             return _meta.Value(type_desc, decimalValue=_meta.DecimalValue(str(raw_value)))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.DECIMAL.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.DECIMAL.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_string_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_string_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.STRING)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.STRING) if not nested else None
 
         if isinstance(raw_value, str):
             return _meta.Value(type_desc, stringValue=raw_value)
@@ -301,13 +341,13 @@ def convert_string_value(raw_value: tp.Any) -> _meta.Value:
 
             return _meta.Value(type_desc, stringValue=str(raw_value))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.STRING.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.STRING.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_date_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_date_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.DATE)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.DATE) if not nested else None
 
         if isinstance(raw_value, dt.date):
             return _meta.Value(type_desc, dateValue=_meta.DateValue(isoDate=raw_value.isoformat()))
@@ -316,13 +356,13 @@ def convert_date_value(raw_value: tp.Any) -> _meta.Value:
             date_value = dt.date.fromisoformat(raw_value)
             return _meta.Value(type_desc, dateValue=_meta.DateValue(isoDate=date_value.isoformat()))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.DATE.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.DATE.name}"
         raise _ex.ETracInternal(msg)
 
     @staticmethod
-    def convert_datetime_value(raw_value: tp.Any) -> _meta.Value:
+    def convert_datetime_value(raw_value: tp.Any, nested: bool = False) -> _meta.Value:
 
-        type_desc = _meta.TypeDescriptor(_meta.BasicType.DATETIME)
+        type_desc = _meta.TypeDescriptor(_meta.BasicType.DATETIME) if not nested else None
 
         if isinstance(raw_value, dt.datetime):
             return _meta.Value(type_desc, datetimeValue=_meta.DatetimeValue(isoDatetime=raw_value.isoformat()))
@@ -331,5 +371,5 @@ def convert_datetime_value(raw_value: tp.Any) -> _meta.Value:
             datetime_value = dt.datetime.fromisoformat(raw_value)
             return _meta.Value(type_desc, datetimeValue=_meta.DatetimeValue(isoDatetime=datetime_value.isoformat()))
 
-        msg = f"Value of type [{type(raw_value)}] cannot be converted to {_meta.BasicType.DATETIME.name}"
+        msg = f"Value of type [{type(raw_value).__name__}] cannot be converted to {_meta.BasicType.DATETIME.name}"
         raise _ex.ETracInternal(msg)