Introduce MSTx-NectarCam and MSTx-FlashCam

.env_template

@@ -6,7 +6,7 @@ SIMTOOLS_DB_SERVER='cta-simpipe-protodb.zeuthen.desy.de' # MongoDB server
 SIMTOOLS_DB_API_USER=YOUR_USERNAME # username for MongoDB: ask the responsible person
 SIMTOOLS_DB_API_PW=YOUR_PASSWORD # Password for MongoDB: ask the responsible person
 SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin'
-SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-v0-3-0'
+SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-v0-4-0'
 SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'
 
 # The dashboards to monitor the MongoDB instance are in (accessible only from within DESY)

docs/changes/1362.model.md

@@ -0,0 +1 @@
+Introduce MSTx-NectarCam and MSTx-FlashCam design models for MSTs (allow both sites).

src/simtools/applications/validate_file_using_schema.py

@@ -170,9 +170,9 @@ def validate_data_file(args_dict, logger):
         data_file=args_dict["file_name"],
         check_exact_data_type=args_dict["require_exact_data_type"],
     )
-    data_validator.validate_and_transform(is_model_parameter=True)
-    if args_dict["data_type"].lower() == "model_parameter":
-        data_validator.validate_parameter_and_file_name()
+    data_validator.validate_and_transform(
+        is_model_parameter=(args_dict["data_type"].lower() == "model_parameter")
+    )
 
     logger.info(f"Successful validation of data file {args_dict['file_name']}")
 

src/simtools/data_model/schema.py

@@ -121,9 +121,13 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
     try:
         jsonschema.validate(data, schema=json_schema, format_checker=format_checkers.format_checker)
     except jsonschema.exceptions.ValidationError as exc:
-        _logger.error(f"Validation failed using schema: {json_schema}")
+        _logger.error(f"Validation failed using schema: {json_schema} for data: {data}")
         raise exc
-    if data.get("meta_schema_url") and not gen.url_exists(data["meta_schema_url"]):
+    if (
+        isinstance(data, dict)
+        and data.get("meta_schema_url")
+        and not gen.url_exists(data["meta_schema_url"])
+    ):
         raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
 
     _logger.debug(f"Successful validation of data using schema ({json_schema.get('name')})")

src/simtools/data_model/validate_data.py

@@ -12,7 +12,7 @@
 
 import simtools.utils.general as gen
 from simtools.data_model import schema
-from simtools.utils import value_conversion
+from simtools.utils import names, value_conversion
 
 __all__ = ["DataValidator"]
 
@@ -79,19 +79,24 @@ def validate_and_transform(self, is_model_parameter=False, lists_as_strings=Fals
 
         """
         if self.data_file_name:
-            self.validate_data_file()
+            self.validate_data_file(is_model_parameter)
         if isinstance(self.data_dict, dict):
             return self._validate_data_dict(is_model_parameter, lists_as_strings)
         if isinstance(self.data_table, Table):
             return self._validate_data_table()
         self._logger.error("No data or data table to validate")
         raise TypeError
 
-    def validate_data_file(self):
+    def validate_data_file(self, is_model_parameter=None):
         """
         Open data file and read data from file.
 
         Doing this successfully is understood as file validation.
+
+        Parameters
+        ----------
+        is_model_parameter: bool
+            This is a model parameter file.
         """
         try:
             if Path(self.data_file_name).suffix in (".yml", ".yaml", ".json"):
@@ -102,14 +107,30 @@ def validate_data_file(self):
                 self._logger.info(f"Validating tabled data from: {self.data_file_name}")
         except (AttributeError, TypeError):
             pass
+        if is_model_parameter:
+            self.validate_parameter_and_file_name()
 
     def validate_parameter_and_file_name(self):
-        """Validate that file name and key 'parameter_name' in data dict are the same."""
-        if not str(Path(self.data_file_name).stem).startswith(self.data_dict.get("parameter")):
-            raise ValueError(
-                f"Parameter name in data dict {self.data_dict.get('parameter')} and "
-                f"file name {Path(self.data_file_name).stem} do not match."
-            )
+        """
+        Validate model parameter file name.
+
+        Expect that the following convention is used:
+
+        - file name starts with the parameter name
+        - file name ends with parameter version string
+
+        """
+        file_stem = Path(self.data_file_name).stem
+        param = self.data_dict.get("parameter")
+        param_version = self.data_dict.get("parameter_version")
+        if not file_stem.startswith(param):
+            raise ValueError(f"Mismatch: parameter '{param}' vs. file '{file_stem}'.")
+
+        if param_version and not file_stem.endswith(param_version):
+            raise ValueError(f"Mismatch: version '{param_version}' vs. file '{file_stem}'.")
+
+        if param_version is None:
+            self._logger.warning(f"File '{file_stem}' has no parameter version defined.")
 
     @staticmethod
     def validate_model_parameter(par_dict):
@@ -170,7 +191,17 @@ def _validate_data_dict(self, is_model_parameter=False, lists_as_strings=False):
         else:
             self.data_dict["value"], self.data_dict["unit"] = value_as_list, unit_as_list
 
-        self._check_version_string(self.data_dict.get("version"))
+        if self.data_dict.get("instrument"):
+            if self.data_dict["instrument"] == self.data_dict["site"]:  # site parameters
+                names.validate_site_name(self.data_dict["site"])
+            else:
+                names.validate_array_element_name(self.data_dict["instrument"])
+                self._check_site_and_array_element_consistency(
+                    self.data_dict.get("instrument"), self.data_dict.get("site")
+                )
+
+        for version_string in ("version", "parameter_version", "model_version"):
+            self._check_version_string(self.data_dict.get(version_string))
 
         if lists_as_strings:
             self._convert_results_to_model_format()
@@ -814,3 +845,24 @@ def _check_version_string(self, version):
         if not re.match(semver_regex, version):
             raise ValueError(f"Invalid version string '{version}'")
         self._logger.debug(f"Valid version string '{version}'")
+
+    def _check_site_and_array_element_consistency(self, instrument, site):
+        """
+        Check that site and array element names are consistent.
+
+        An example for an inconsistency is 'LSTN' at site 'South'
+        """
+        if not all([instrument, site]) or "OBS" in instrument:
+            return
+
+        def to_sorted_list(value):
+            """Return value as sorted list."""
+            return [value] if isinstance(value, str) else sorted(value)
+
+        instrument_site = to_sorted_list(names.get_site_from_array_element_name(instrument))
+        site = to_sorted_list(site)
+
+        if instrument_site != site:
+            raise ValueError(
+                f"Site '{site}' and instrument site '{instrument_site}' are inconsistent."
+            )

src/simtools/db/db_handler.py

@@ -227,7 +227,7 @@ def get_model_parameters(
         site: str
             Site name.
         array_element_name: str
-            Name of the array element model (e.g. LSTN-01, MSTS-design, ILLN-01).
+            Name of the array element model (e.g. LSTN-01, MSTx-FlashCam, ILLN-01).
         model_version: str, list
             Version(s) of the model.
         collection: str
@@ -475,7 +475,7 @@ def get_model_versions(self, collection_name="telescopes"):
         """
         collection = self.get_collection(self._get_db_name(), "production_tables")
         return sorted(
-            [post["model_version"] for post in collection.find({"collection": collection_name})]
+            {post["model_version"] for post in collection.find({"collection": collection_name})}
         )
 
     def get_array_elements(self, model_version, collection="telescopes"):
@@ -871,6 +871,6 @@ def _get_array_element_list(self, array_element_name, site, production_table, co
             ]
         except KeyError:
             return [
-                f"{names.get_array_element_type_from_name(array_element_name)}-design",
+                names.guess_design_model_from_name(array_element_name),
                 array_element_name,
             ]

src/simtools/layout/array_layout.py

@@ -447,7 +447,7 @@ def _set_telescope_auxiliary_parameters(self, telescope, telescope_name=None):
                 tel_model = self._get_telescope_model(telescope_name)
             except ValueError:
                 tel_model = self._get_telescope_model(
-                    names.get_array_element_type_from_name(telescope_name) + "-design",
+                    names.guess_design_model_from_name(telescope_name),
                 )
 
             for para in ("telescope_axis_height", "telescope_sphere_radius"):

src/simtools/schemas/array_elements.yml

@@ -33,6 +33,12 @@ data:
     observatory: "CTAO"
     site: "South"
     description: "Mid-sized Schwarzschild-Couder telescope"
+  MSTx:
+    collection: "telescopes"
+    observatory: "CTAO"
+    site: ["North", "South"]
+    design_types: ["design", "test", "FlashCam", "NectarCam"]
+    description: "Mid-sized telescope"
   ILLN:
     collection: "calibration_devices"
     observatory: "CTAO"

src/simtools/schemas/model_parameter.metaschema.yml

@@ -2,6 +2,102 @@
 $schema: http://json-schema.org/draft-06/schema#
 $ref: '#/definitions/SimtoolsModelParameters'
 title: SimPipe DB Model Parameter Metaschema
+description: |
+  YAML representation of db model parameter metaschema
+  (based on simulation model DB).
+version: 0.3.0
+name: modelparameter.metaschema
+type: object
+additionalProperties: false
+
+definitions:
+  SimtoolsModelParameters:
+    description: ""
+    type: object
+    properties:
+      _id:
+        type: string
+        description: "DB unique identifier."
+      entry_date:
+        type: string
+        description: "Value entry date."
+      file:
+        type: boolean
+        description: "This parameter is a file."
+      instrument:
+        type:
+          - string
+          - "null"
+        description: "Associated instrument."
+      site:
+        anyOf:
+          - type: string
+            enum:
+              - North
+              - South
+          - type: "null"
+          - type: array
+            items:
+              type: string
+              enum:
+                - North
+                - South
+        description: "Associated CTAO site."
+      type:
+        type: string
+        description: "Data type"
+        enum:
+          - boolean
+          - dict
+          - double
+          - file
+          - float64
+          - int
+          - int64
+          - string
+          - uint
+          - uint32
+          - uint64
+      unit:
+        anyOf:
+          - type: string
+          - type: "null"
+        description: "Unit of the parameter."
+      value:
+        anyOf:
+          - type: boolean
+          - type: number
+          - type: string
+          - type: "null"
+          - type: array
+        description: "Value of the parameter."
+      parameter_version:
+        anyOf:
+          - type: string
+        description: "Parameter version."
+      schema_version:
+        anyOf:
+          - type: string
+        description: "Metaschema version."
+      unique_id:
+        anyOf:
+          - type: string
+          - type: "null"
+        description: "Unique ID of parameter definition."
+    required:
+      - file
+      - instrument
+      - parameter_version
+      - schema_version
+      - site
+      - type
+      - unit
+      - value
+...
+---
+$schema: http://json-schema.org/draft-06/schema#
+$ref: '#/definitions/SimtoolsModelParameters'
+title: SimPipe DB Model Parameter Metaschema
 description: |
   YAML representation of db model parameter metaschema
   (based on simulation model DB).

src/simtools/schemas/production_tables.schema.yml

@@ -29,7 +29,7 @@ definitions:
             pattern: '^\d+\.\d+\.\d+$'
         propertyNames:
           description: Allowed parameter name patterns.
-          pattern: '^([A-Za-z](ST|LL)[N,S,x]-\d{2,3}|[A-Za-z](ST|LL)[N,S,x]-design|OBS-(North|South)|Dummy-Telescope)$'
+          pattern: '^([A-Za-z](ST|LL)[N,S,x]-\d{2,3}|[A-Za-z](ST|LL)[N,S,x]-(design|FlashCam|NectarCam)|OBS-(North|South)|Dummy-Telescope)$'
       design_model:
         type: object
         description: Design models.

src/simtools/testing/validate_output.py

@@ -141,6 +141,7 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
     Compare two json or yaml files.
 
     Take into account float comparison for sim_telarray string-embedded floats.
+    Allow differences in 'schema_version' field.
 
     Parameters
     ----------
@@ -159,6 +160,8 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
     """
     data1 = gen.collect_data_from_file(file1)
     data2 = gen.collect_data_from_file(file2)
+    data1.pop("schema_version", None)
+    data2.pop("schema_version", None)
 
     _logger.debug(f"Comparing json/yaml files: {file1} and {file2}")
 
@@ -181,7 +184,7 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
     return _comparison
 
 
-def _compare_value_from_parameter_dict(data1, data2, tolerance):
+def _compare_value_from_parameter_dict(data1, data2, tolerance=1.0e-5):
     """Compare value fields given in different formats."""
 
     def _as_list(value):
@@ -193,7 +196,11 @@ def _as_list(value):
 
     _logger.info(f"Comparing values: {data1} and {data2} (tolerance: {tolerance})")
 
-    return np.allclose(_as_list(data1), _as_list(data2), rtol=tolerance)
+    _as_list_1 = _as_list(data1)
+    _as_list_2 = _as_list(data2)
+    if isinstance(_as_list_1, str):
+        return _as_list_1 == _as_list_2
+    return np.allclose(_as_list_1, _as_list_2, rtol=tolerance)
 
 
 def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):