Added an idem exec and state module

changelog/57969.added

@@ -0,0 +1,2 @@
+- Added an execution module for running idem exec modules
+- Added a state module for running idem states

changelog/57993.added

@@ -0,0 +1 @@
+- Added the ability for states to return `sub_state_run`s -- results from external state engines

doc/ref/modules/all/index.rst

@@ -194,6 +194,7 @@ execution modules
     hosts
     http
     icinga2
+    idem
     ifttt
     ilo
     incron

doc/ref/modules/all/salt.modules.idem.rst

@@ -0,0 +1,5 @@
+salt.modules.idem module
+========================
+
+.. automodule:: salt.modules.idem
+    :members:

doc/ref/states/all/index.rst

@@ -131,6 +131,7 @@ state modules
     host
     http
     icinga2
+    idem
     ifttt
     incron
     influxdb08_database

doc/ref/states/all/salt.states.idem.rst

@@ -0,0 +1,5 @@
+salt.states.idem
+================
+
+.. automodule:: salt.states.idem
+    :members:

doc/ref/states/writing.rst

@@ -259,6 +259,45 @@ A State Module must return a dict containing the following keys/values:
 
     States should not return data which cannot be serialized such as frozensets.
 
+Sub State Runs
+--------------
+
+Some states can return multiple state runs from an external engine.
+State modules that extend tools like Puppet, Chef, Ansible, and idem can run multiple external
+states and then return their results individually in the "sub_state_run" portion of their return
+as long as their individual state runs are formatted like salt states with low and high data.
+
+For example, the idem state module can execute multiple idem states
+via it's runtime and report the status of all those runs by attaching them to "sub_state_run" in it's state return.
+These sub_state_runs will be formatted and printed alongside other salt states.
+
+Example:
+
+.. code-block:: python
+
+    state_return = {
+        "name": None,  # The parent state name
+        "result": None,  # The overall status of the external state engine run
+        "comment": None,  # Comments on the overall external state engine run
+        "changes": {},  # An empty dictionary, each sub state run has it's own changes to report
+        "sub_state_run": [
+            {
+                "changes": {},  # A dictionary describing the changes made in the external state run
+                "result": None,  # The external state run name
+                "comment": None,  # Comment on the external state run
+                "duration": None,  # Optional, the duration in seconds of the external state run
+                "start_time": None,  # Optional, the timestamp of the external state run's start time
+                "low": {
+                    "name": None,  # The name of the state from the external state run
+                    "state": None,  # Name of the external state run
+                    "__id__": None,  # ID of the external state run
+                    "fun": None,  # The Function name from the external state run
+                },
+            }
+        ],
+    }
+
+
 Test State
 ==========
 

salt/modules/idem.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+#
+# Author: Tyler Johnson <tjohnson@saltstack.com>
+#
+
+"""
+Idem Support
+============
+
+This module provides access to idem execution modules
+
+.. versionadded:: Magnesium
+"""
+# Function alias to make sure not to shadow built-in's
+__func_alias__ = {"exec_": "exec"}
+__virtualname__ = "idem"
+
+
+def __virtual__():
+    if "idem.hub" in __utils__:
+        return __virtualname__
+    else:
+        return False, "idem is not available"
+
+
+def exec_(path, acct_file=None, acct_key=None, acct_profile=None, *args, **kwargs):
+    """
+    Call an idem execution module
+
+    path
+        The idem path of the idem execution module to run
+
+    acct_file
+        Path to the acct file used in generating idem ctx parameters.
+        Defaults to the value in the ACCT_FILE environment variable.
+
+    acct_key
+        Key used to decrypt the acct file.
+        Defaults to the value in the ACCT_KEY environment variable.
+
+    acct_profile
+        Name of the profile to add to idem's ctx.acct parameter.
+        Defaults to the value in the ACCT_PROFILE environment variable.
+
+    args
+        Any positional arguments to pass to the idem exec function
+
+    kwargs
+        Any keyword arguments to pass to the idem exec function
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt '*' idem.exec test.ping
+
+    :maturity:      new
+    :depends:       acct, pop, pop-config, idem
+    :platform:      all
+    """
+    hub = __utils__["idem.hub"]()
+
+    coro = hub.idem.ex.run(
+        path,
+        args,
+        {k: v for k, v in kwargs.items() if not k.startswith("__")},
+        acct_file=acct_file or hub.OPT.acct.acct_file,
+        acct_key=acct_key or hub.OPT.acct.acct_key,
+        acct_profile=acct_profile or hub.OPT.acct.acct_profile or "default",
+    )
+
+    return hub.pop.Loop.run_until_complete(coro)

salt/state.py

@@ -3082,6 +3082,22 @@ def call_chunk(self, low, running, chunks):
                 running[tag] = self.call(low, chunks, running)
         if tag in running:
             self.event(running[tag], len(chunks), fire_event=low.get("fire_event"))
+
+            for sub_state_data in running[tag].pop("sub_state_run", ()):
+                self.__run_num += 1
+                sub_tag = _gen_tag(sub_state_data["low"])
+                running[sub_tag] = {
+                    "name": sub_state_data["low"]["name"],
+                    "changes": sub_state_data["changes"],
+                    "result": sub_state_data["result"],
+                    "duration": sub_state_data.get("duration"),
+                    "start_time": sub_state_data.get("start_time"),
+                    "comment": sub_state_data.get("comment"),
+                    "__state_ran__": True,
+                    "__run_num__": self.__run_num,
+                    "__sls__": low["__sls__"],
+                }
+
         return running
 
     def call_listen(self, chunks, running):

salt/states/idem.py

@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*-
+#
+# Author: Tyler Johnson <tjohnson@saltstack.com>
+#
+
+"""
+Idem Support
+============
+
+This state provides access to idem states
+
+.. versionadded:: Magnesium
+"""
+import pathlib
+import re
+
+__virtualname__ = "idem"
+
+
+def __virtual__():
+    if "idem.hub" in __utils__:
+        return __virtualname__
+    else:
+        return False, "idem is not available"
+
+
+def _get_refs(sources, tree):
+    """
+    Determine where the sls sources are
+    """
+    sls_sources = []
+    SLSs = []
+    if tree:
+        sls_sources.append("file://{}".format(tree))
+    for sls in sources:
+        path = pathlib.Path(sls)
+        if path.is_file():
+            ref = str(path.stem if path.suffix == ".sls" else path.name)
+            SLSs.append(ref)
+            implied = "file://{}".format(path.parent)
+            if implied not in sls_sources:
+                sls_sources.append(implied)
+        else:
+            SLSs.append(sls)
+    return sls_sources, SLSs
+
+
+def _get_low_data(low_data):
+    """
+    Get salt-style low data from an idem state name
+    """
+    # state_|-id_|-name_|-function
+    match = re.match(r"(\w+)_\|-(\w+)\|-(\w+)_\|-(\w+)", low_data)
+    return {
+        "state": match.group(1),
+        "__id__": match.group(2),
+        "name": match.group(3),
+        "fun": match.group(4),
+    }
+
+
+def state(
+    name,
+    sls,
+    acct_file=None,
+    acct_key=None,
+    acct_profile=None,
+    cache_dir=None,
+    render=None,
+    runtime=None,
+    source_dir=None,
+    test=False,
+):
+    """
+    Call an idem state through a salt state
+
+    sls
+        A list of idem sls files or sources
+
+    acct_file
+        Path to the acct file used in generating idem ctx parameters.
+        Defaults to the value in the ACCT_FILE environment variable.
+
+    acct_key
+        Key used to decrypt the acct file.
+        Defaults to the value in the ACCT_KEY environment variable.
+
+    acct_profile
+        Name of the profile to add to idem's ctx.acct parameter
+        Defaults to the value in the ACCT_PROFILE environment variable.
+
+    cache_dir
+        The location to use for the cache directory
+
+    render
+        The render pipe to use, this allows for the language to be specified (jinja|yaml)
+
+    runtime
+        Select which execution runtime to use (serial|parallel)
+
+    source_dir
+        The directory containing sls files
+
+    .. code-block:: yaml
+
+    cheese:
+      idem.state:
+        - runtime: parallel
+        - sls:
+          - idem_state.sls
+          - sls_source
+
+    :maturity:      new
+    :depends:       acct, pop, pop-config, idem
+    :platform:      all
+    """
+    hub = __utils__["idem.hub"]()
+
+    if isinstance(sls, str):
+        sls = [sls]
+
+    sls_sources, SLSs = _get_refs(sls, source_dir or hub.OPT.idem.tree)
+
+    coro = hub.idem.state.apply(
+        name=name,
+        sls_sources=sls_sources,
+        render=render or hub.OPT.idem.render,
+        runtime=runtime or hub.OPT.idem.runtime,
+        subs=["states"],
+        cache_dir=cache_dir or hub.OPT.idem.cache_dir,
+        sls=SLSs,
+        test=test,
+        acct_file=acct_file or hub.OPT.acct.acct_file,
+        acct_key=acct_key or hub.OPT.acct.acct_key,
+        acct_profile=acct_profile or hub.OPT.acct.acct_profile or "default",
+    )
+    hub.pop.Loop.run_until_complete(coro)
+
+    errors = hub.idem.RUNS[name]["errors"]
+    success = not errors
+
+    running = []
+    for idem_name, idem_return in hub.idem.RUNS[name]["running"].items():
+        standardized_idem_return = {
+            "name": idem_return["name"],
+            "changes": idem_return["changes"],
+            "result": idem_return["result"],
+            "comment": idem_return.get("comment"),
+            "low": _get_low_data(idem_name),
+        }
+        running.append(standardized_idem_return)
+
+    return {
+        "name": name,
+        "result": success,
+        "comment": "Ran {} idem states".format(len(running)) if success else errors,
+        "changes": {},
+        "sub_state_run": running,
+    }

salt/utils/decorators/state.py

@@ -26,26 +26,36 @@ def __init__(self, *policies):
             else:
                 self.policies.append(getattr(self, pls))
 
+    def _run_policies(self, data):
+        for pls in self.policies:
+            try:
+                data = pls(data)
+            except Exception as exc:  # pylint: disable=broad-except
+                log.debug(
+                    "An exception occurred in this state: %s",
+                    exc,
+                    exc_info_on_loglevel=logging.DEBUG,
+                )
+                data = {
+                    "result": False,
+                    "name": "later",
+                    "changes": {},
+                    "comment": "An exception occurred in this state: {0}".format(exc),
+                }
+        return data
+
     def __call__(self, func):
         def _func(*args, **kwargs):
             result = func(*args, **kwargs)
-            for pls in self.policies:
-                try:
-                    result = pls(result)
-                except Exception as exc:  # pylint: disable=broad-except
-                    log.debug(
-                        "An exception occurred in this state: %s",
-                        exc,
-                        exc_info_on_loglevel=logging.DEBUG,
-                    )
-                    result = {
-                        "result": False,
-                        "name": "later",
-                        "changes": {},
-                        "comment": "An exception occurred in this state: {0}".format(
-                            exc
-                        ),
-                    }
+            sub_state_run = None
+            if isinstance(result, dict):
+                sub_state_run = result.get("sub_state_run", ())
+            result = self._run_policies(result)
+            if sub_state_run:
+                result["sub_state_run"] = [
+                    self._run_policies(sub_state_data)
+                    for sub_state_data in sub_state_run
+                ]
             return result
 
         return _func
@@ -77,6 +87,9 @@ def content_check(self, result):
         if err_msg:
             raise SaltException(err_msg)
 
+        for sub_state in result.get("sub_state_run", ()):
+            self.content_check(sub_state)
+
         return result
 
     def unify(self, result):