Add generic simtools-run-application and improvement for setting workflows.

docs/changes/1379.feature.md

@@ -0,0 +1 @@
+Add generic `simtools-run-application` to run one or several simtools using a single configuration file.

docs/source/user-guide/applications.md

@@ -45,11 +45,12 @@ simtools-generate-default-metadata <applications/simtools-generate-default-metad
 simtools-generate-regular-arrays <applications/simtools-generate-regular-arrays>
 simtools-generate-simtel-array-histograms <applications/simtools-generate-simtel-array-histograms>
 simtools-plot-array-layout <applications/simtools-plot-array-layout>
+simtools-plot-tabular-data <applications/simtools-plot-tabular-data>
 simtools-production-derive-limits <applications/simtools-production-derive-limits>
 simtools-production-generate-simulation-config <applications/simtools-production-generate-simulation-config>
 simtools-production-scale-events <applications/simtools-production-scale-events>
+simtools-run-application <applications/simtools-run-application>
 simtools-simulate-light-emission <applications/simtools-simulate-light-emission>
-simtools-plot-tabular-data <applications/simtools-plot-tabular-data>
 simtools-simulate-prod <applications/simtools-simulate-prod>
 simtools-simulate-prod-htcondor-generator <applications/simtools-simulate-prod-htcondor-generator>
 simtools-submit-data-from-external <applications/simtools-submit-data-from-external>

docs/source/user-guide/applications/simtools-run-application.rst

@@ -0,0 +1,6 @@
+
+simtools-run-application
+========================
+
+.. automodule:: run_application
+   :members:

pyproject.toml

@@ -95,6 +95,7 @@ scripts.simtools-plot-tabular-data = "simtools.applications.plot_tabular_data:ma
 scripts.simtools-production-derive-limits = "simtools.applications.production_derive_limits:main"
 scripts.simtools-production-generate-simulation-config = "simtools.applications.production_generate_simulation_config:main"
 scripts.simtools-production-scale-events = "simtools.applications.production_scale_events:main"
+scripts.simtools-run-application = "simtools.applications.run_application:main"
 scripts.simtools-simulate-light-emission = "simtools.applications.simulate_light_emission:main"
 scripts.simtools-simulate-prod = "simtools.applications.simulate_prod:main"
 scripts.simtools-simulate-prod-htcondor-generator = "simtools.applications.simulate_prod_htcondor_generator:main"

src/simtools/applications/run_application.py

@@ -0,0 +1,98 @@
+#!/usr/bin/python3
+
+"""
+Run simtools applications from configuration files.
+
+Allows to run several simtools applications with a single configuration file, which includes
+both the name of the simtools application and the configuration for the application.
+
+"""
+
+import logging
+import subprocess
+import tempfile
+from pathlib import Path
+
+import yaml
+
+import simtools.utils.general as gen
+from simtools import dependencies
+from simtools.configuration import configurator
+
+
+def _parse(label, description, usage):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing the application.
+    description : str
+        Description of the application.
+    usage : str
+        Example on how to use the application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description, usage=usage)
+
+    config.parser.add_argument(
+        "--configuration_file",
+        help="Application configuration.",
+        type=str,
+        required=True,
+        default=None,
+    )
+    return config.initialize(db_config=False)
+
+
+def run_application(application, configuration):
+    """Run a simtools application and return stdout and stderr."""
+    with tempfile.NamedTemporaryFile(mode="w", delete=True, suffix=".yml") as temp_config:
+        yaml.dump(configuration, temp_config, default_flow_style=False)
+        temp_config.flush()
+        configuration_file = Path(temp_config.name)
+        result = subprocess.run(
+            [application, "--config", configuration_file],
+            check=False,
+            capture_output=True,
+            text=True,
+        )
+        return result.stdout, result.stderr
+
+
+def main():  # noqa: D103
+
+    args_dict, _ = _parse(
+        Path(__file__).stem,
+        description="Run simtools applications from configuration file.",
+        usage="simtools-run-application --config_file config_file_name",
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    application_config = gen.collect_data_from_file(args_dict["configuration_file"]).get(
+        "CTA_SIMPIPE"
+    )
+    log_file = Path(application_config.get("LOG_PATH", "./")) / "simtools.log"
+    log_file.parent.mkdir(parents=True, exist_ok=True)
+    configurations = application_config.get("APPLICATIONS")
+    with log_file.open("w", encoding="utf-8") as file:
+        file.write("Running simtools applications\n")
+        file.write(dependencies.get_version_string())
+        for config in configurations:
+            logger.info(f"Running application: {config.get('APPLICATION')}")
+            config = gen.change_dict_keys_case(config, False)
+            stdout, stderr = run_application(config.get("APPLICATION"), config.get("CONFIGURATION"))
+            file.write("=" * 80 + "\n")
+            file.write(f"Application: {config.get('APPLICATION')}\n")
+            file.write("STDOUT:\n" + stdout)
+            file.write("STDERR:\n" + stderr)
+
+
+if __name__ == "__main__":
+    main()

src/simtools/applications/submit_model_parameter_from_external.py

@@ -73,7 +73,6 @@ def _parse(label, description):
     config.parser.add_argument(
         "--parameter_version", type=str, required=True, help="Parameter version"
     )
-
     config.parser.add_argument(
         "--value",
         type=str,
@@ -84,18 +83,22 @@ def _parse(label, description):
             'Examples: "--value=5", "--value=\'5 km\'", "--value=\'5 cm, 0.5 deg\'"'
         ),
     )
-
     config.parser.add_argument(
         "--input_meta",
         help="meta data file associated to input data",
         type=str,
         required=False,
     )
-    return config.initialize(output=True)
+    config.parser.add_argument(
+        "--check_parameter_version",
+        help="Check if the parameter version exists in the database",
+        action="store_true",
+    )
+    return config.initialize(output=True, db_config=True)
 
 
 def main():  # noqa: D103
-    args_dict, _ = _parse(
+    args_dict, db_config = _parse(
         label=Path(__file__).stem,
         description="Submit and validate a model parameters).",
     )
@@ -104,19 +107,21 @@ def main():  # noqa: D103
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     output_path = (
-        Path(args_dict["output_path"]) / args_dict["parameter_version"] / args_dict["instrument"]
+        Path(args_dict["output_path"]) / args_dict["instrument"] / args_dict["parameter"]
         if args_dict.get("output_path")
         else None
     )
+
     writer.ModelDataWriter.dump_model_parameter(
         parameter_name=args_dict["parameter"],
         value=args_dict["value"],
         instrument=args_dict["instrument"],
         parameter_version=args_dict["parameter_version"],
-        output_file=Path(args_dict["parameter"]).with_suffix(".json"),
+        output_file=Path(args_dict["parameter"] + "-" + args_dict["parameter_version"] + ".json"),
         output_path=output_path,
         use_plain_output_path=args_dict.get("use_plain_output_path"),
         metadata_input_dict=args_dict,
+        db_config=db_config if args_dict.get("check_parameter_version") else None,
     )
 
 

src/simtools/data_model/metadata_collector.py

@@ -15,6 +15,7 @@
 import simtools.constants
 import simtools.utils.general as gen
 import simtools.version
+from simtools.constants import METADATA_JSON_SCHEMA
 from simtools.data_model import metadata_model, schema
 from simtools.io_operations import io_handler
 from simtools.utils import names
@@ -266,7 +267,7 @@ def _read_input_metadata_from_file(self, metadata_file_name=None):
             self._logger.error("Unknown metadata file format: %s", metadata_file_name)
             raise gen.InvalidConfigDataError
 
-        schema.validate_dict_using_schema(_input_metadata, None)
+        schema.validate_dict_using_schema(_input_metadata, schema_file=METADATA_JSON_SCHEMA)
 
         return gen.change_dict_keys_case(
             self._process_metadata_from_file(_input_metadata),

src/simtools/data_model/model_data_writer.py

@@ -12,6 +12,7 @@
 import simtools.utils.general as gen
 from simtools.data_model import schema, validate_data
 from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.db import db_handler
 from simtools.io_operations import io_handler
 from simtools.utils import names, value_conversion
 
@@ -126,6 +127,7 @@
         output_path=None,
         use_plain_output_path=False,
         metadata_input_dict=None,
+        db_config=None,
     ):
         """
         Generate DB-style model parameter dict and write it to json file.
@@ -148,6 +150,8 @@
             Use plain output path.
         metadata_input_dict: dict
             Input to metadata collector.
+        db_config: dict
+            Database configuration. If not None, check if parameter with the same version exists.
 
         Returns
         -------
@@ -161,21 +165,72 @@
             output_path=output_path,
             use_plain_output_path=use_plain_output_path,
         )
-        _json_dict = writer.get_validated_parameter_dict(
-            parameter_name, value, instrument, parameter_version
-        )
-        writer.write_dict_to_model_parameter_json(output_file, _json_dict)
+        if db_config is not None:
+            writer.check_db_for_existing_parameter(
+                parameter_name, instrument, parameter_version, db_config
+            )
+
+        unique_id = None
         if metadata_input_dict is not None:
             metadata_input_dict["output_file"] = output_file
             metadata_input_dict["output_file_format"] = Path(output_file).suffix.lstrip(".")
+            metadata = MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata()
             writer.write_metadata_to_yml(
-                metadata=MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata(),
-                yml_file=output_path / f"{Path(output_file).stem}",
+                metadata=metadata, yml_file=output_path / f"{Path(output_file).stem}"
             )
+            unique_id = metadata.get("cta", {}).get("product", {}).get("id")
+
+        _json_dict = writer.get_validated_parameter_dict(
+            parameter_name, value, instrument, parameter_version, unique_id
+        )
+        writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         return _json_dict
 
+    def check_db_for_existing_parameter(
+        self, parameter_name, instrument, parameter_version, db_config
+    ):
+        """
+        Check if a parameter with the same version exists in the simulation model database.
+
+        Parameters
+        ----------
+        parameter_name: str
+            Name of the parameter.
+        instrument: str
+            Name of the instrument.
+        parameter_version: str
+            Version of the parameter.
+        db_config: dict
+            Database configuration.
+
+        Raises
+        ------
+        ValueError
+            If parameter with the same version exists in the database.
+        """
+        db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+        try:
+            db.get_model_parameter(
+                parameter=parameter_name,
+                parameter_version=parameter_version,
+                site=names.get_site_from_array_element_name(instrument),
+                array_element_name=instrument,
+            )
+        except ValueError:
+            pass  # parameter does not exist - expected behavior
+        else:
+            raise ValueError(
+                f"Parameter {parameter_name} with version {parameter_version} already exists."
+            )
+
     def get_validated_parameter_dict(
-        self, parameter_name, value, instrument, parameter_version, schema_version=None
+        self,
+        parameter_name,
+        value,
+        instrument,
+        parameter_version,
+        unique_id=None,
+        schema_version=None,
     ):
         """
         Get validated parameter dictionary.
@@ -202,20 +257,15 @@
         schema_file = schema.get_model_parameter_schema_file(parameter_name)
         self.schema_dict = gen.collect_data_from_file(schema_file)
 
-        try:  # e.g. instrument is 'North"
-            site = names.validate_site_name(instrument)
-        except ValueError:  # e.g. instrument is 'LSTN-01'
-            site = names.get_site_from_array_element_name(instrument)
-
         value, unit = value_conversion.split_value_and_unit(value)
 
         data_dict = {
             "schema_version": schema.get_model_parameter_schema_version(schema_version),
             "parameter": parameter_name,
             "instrument": instrument,
-            "site": site,
+            "site": names.get_site_from_array_element_name(instrument),
             "parameter_version": parameter_version,
-            "unique_id": None,
+            "unique_id": unique_id,
             "value": value,
             "unit": unit,
             "type": self._get_parameter_type(),
@@ -425,7 +475,9 @@
             If yml_file is not defined.
         """
         try:
-            yml_file = Path(yml_file or self.product_data_file).with_suffix(".metadata.yml")
+            yml_file = names.file_name_with_version(
+                yml_file or self.product_data_file, ".metadata.yml"
+            )
             with open(yml_file, "w", encoding="UTF-8") as file:
                 yaml.safe_dump(
                     gen.change_dict_keys_case(metadata, keys_lower_case),

src/simtools/dependencies.py

@@ -63,7 +63,7 @@ def get_sim_telarray_version():
     if sim_telarray_path is None:
         _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
         return None
-    sim_telarray_path = Path(sim_telarray_path) / "bin" / "sim_telarray"
+    sim_telarray_path = Path(sim_telarray_path) / "sim_telarray" / "bin" / "sim_telarray"
 
     # expect stdout with e.g. a line 'Release: 2024.271.0 from 2024-09-27'
     result = subprocess.run(

src/simtools/utils/names.py

@@ -380,7 +380,10 @@ def get_site_from_array_element_name(name):
     str, list
         Site name(s).
     """
-    return array_elements()[get_array_element_type_from_name(name)]["site"]
+    try:  # e.g. instrument is 'North' as given for the site parameters
+        return validate_site_name(name)
+    except ValueError:  # e.g. instrument is 'LSTN' as given for the array element types
+        return array_elements()[get_array_element_type_from_name(name)]["site"]
 
 
 def get_collection_name_from_array_element_name(name, array_elements_only=True):
@@ -704,3 +707,29 @@ def sanitize_name(name):
         _logger.error(msg)
         raise ValueError(msg)
     return sanitized
+
+
+def file_name_with_version(file_name, suffix):
+    """
+    Return a file name including a semantic version with the correct suffix.
+
+    Replaces 'Path.suffix()', which removes trailing numbers (and therefore version numbers).
+
+    Parameters
+    ----------
+    file_name: str
+        File name.
+    suffix: str
+        File suffix.
+
+    Returns
+    -------
+    Path
+        File name with version number.
+    """
+    if file_name is None or suffix is None:
+        return None
+    file_name = str(file_name)
+    if bool(re.search(r"\d+\.\d+\.\d+$", file_name)):
+        return Path(file_name + suffix)
+    return Path(file_name).with_suffix(suffix)