Merge branch 'master' into backport_49981

CHANGELOG.md

@@ -15,6 +15,9 @@ Versions are `MAJOR.PATCH`.
 ### Changed
 - [#56751](https://github.com/saltstack/salt/pull/56751) - Backport 49981
 
+- [#56731](https://github.com/saltstack/salt/pull/56731) - Backport #53994
+- [#56753](https://github.com/saltstack/salt/pull/56753) - Backport 51095
+
 ### Fixed
 - [#56237](https://github.com/saltstack/salt/pull/56237) - Fix alphabetical ordering and remove duplicates across all documentation indexes - [@myii](https://github.com/myii)
 - [#56325](https://github.com/saltstack/salt/pull/56325) - Fix hyperlinks to `salt.serializers` and other documentation issues - [@myii](https://github.com/myii)

doc/topics/jinja/index.rst

@@ -87,7 +87,7 @@ the context into the included file is required:
 .. code-block:: jinja
 
     {% from 'lib.sls' import test with context %}
-    
+
 Includes must use full paths, like so:
 
 .. code-block:: jinja
@@ -649,6 +649,56 @@ Returns:
   1, 4
 
 
+.. jinja_ref:: method_call
+
+``method_call``
+---------------
+
+.. versionadded:: Sodium
+
+Returns a result of object's method call.
+
+Example #1:
+
+.. code-block:: jinja
+
+  {{ [1, 2, 1, 3, 4] | method_call('index', 1, 1, 3) }}
+
+Returns:
+
+.. code-block:: text
+
+  2
+
+This filter can be used with the `map filter`_ to apply object methods without
+using loop constructs or temporary variables.
+
+Example #2:
+
+.. code-block:: jinja
+
+  {% set host_list = ['web01.example.com', 'db01.example.com'] %}
+  {% set host_list_split = [] %}
+  {% for item in host_list %}
+    {% do host_list_split.append(item.split('.', 1)) %}
+  {% endfor %}
+  {{ host_list_split }}
+
+Example #3:
+
+.. code-block:: jinja
+
+  {{ host_list|map('method_call', 'split', '.', 1)|list }}
+
+Return of examples #2 and #3:
+
+.. code-block:: text
+
+  [[web01, example.com], [db01, example.com]]
+
+.. _`map filter`: http://jinja.pocoo.org/docs/2.10/templates/#map
+
+
 .. jinja_ref:: is_sorted
 
 ``is_sorted``

salt/cloud/__init__.py

@@ -79,6 +79,11 @@ def _call(queue, args, kwargs):
             queue.put("ERROR")
             queue.put("Exception")
             queue.put("{0}\n{1}\n".format(ex, trace))
+        except SystemExit as ex:
+            trace = traceback.format_exc()
+            queue.put("ERROR")
+            queue.put("System exit")
+            queue.put("{0}\n{1}\n".format(ex, trace))
         return ret
 
     return _call

salt/cloud/clouds/saltify.py

@@ -268,6 +268,10 @@ def create(vm_):
         "deploy", vm_, __opts__, default=False
     )
 
+    # If ssh_host is not set, default to the minion name
+    if not config.get_cloud_config_value("ssh_host", vm_, __opts__, default=""):
+        vm_["ssh_host"] = vm_["name"]
+
     if deploy_config:
         wol_mac = config.get_cloud_config_value(
             "wake_on_lan_mac", vm_, __opts__, default=""

salt/modules/boto3_sns.py

@@ -116,6 +116,14 @@ def describe_topic(name, region=None, key=None, keyid=None, profile=None):
             ret["Attributes"] = get_topic_attributes(
                 arn, region=region, key=key, keyid=keyid, profile=profile
             )
+            # Grab extended attributes for the above subscriptions
+            for sub in range(len(ret["Subscriptions"])):
+                sub_arn = ret["Subscriptions"][sub]["SubscriptionArn"]
+                if not sub_arn.startswith("arn:aws:sns:"):
+                    # Sometimes a sub is in e.g. PendingAccept or other
+                    # wierd states and doesn't have an ARN yet
+                    log.debug("Subscription with invalid ARN %s skipped...", sub_arn)
+                    continue
     return ret
 
 
@@ -382,6 +390,17 @@ def unsubscribe(SubscriptionArn, region=None, key=None, keyid=None, profile=None
 
         salt myminion boto3_sns.unsubscribe my_subscription_arn region=us-east-1
     """
+    if not SubscriptionArn.startswith("arn:aws:sns:"):
+        # Grrr, AWS sent us an ARN that's NOT and ARN....
+        # This can happen if, for instance, a subscription is left in PendingAcceptance or similar
+        # Note that anything left in PendingConfirmation will be auto-deleted by AWS after 30 days
+        # anyway, so this isn't as ugly a hack as it might seem at first...
+        log.info(
+            "Invalid subscription ARN `%s` passed - likely a PendingConfirmaton or such.  "
+            "Skipping unsubscribe attempt as it would almost certainly fail...",
+            SubscriptionArn,
+        )
+        return True
     subs = list_subscriptions(region=region, key=key, keyid=keyid, profile=profile)
     sub = [s for s in subs if s.get("SubscriptionArn") == SubscriptionArn]
     if not sub:

salt/modules/boto_lambda.py

@@ -264,6 +264,7 @@ def create_function(
     .. code-block:: bash
 
         salt myminion boto_lamba.create_function my_function python2.7 my_role my_file.my_function my_function.zip
+        salt myminion boto_lamba.create_function my_function python2.7 my_role my_file.my_function salt://files/my_function.zip
 
     """
 
@@ -276,6 +277,13 @@ def create_function(
                     "Either ZipFile must be specified, or "
                     "S3Bucket and S3Key must be provided."
                 )
+            if "://" in ZipFile:  # Looks like a remote URL to me...
+                dlZipFile = __salt__["cp.cache_file"](path=ZipFile)
+                if dlZipFile is False:
+                    ret["result"] = False
+                    ret["comment"] = "Failed to cache ZipFile `{0}`.".format(ZipFile)
+                    return ret
+                ZipFile = dlZipFile
             code = {
                 "ZipFile": _filedata(ZipFile),
             }

salt/modules/boto_secgroup.py

@@ -398,10 +398,20 @@ def convert_to_group_ids(
         )
         if not group_id:
             # Security groups are a big deal - need to fail if any can't be resolved...
-            raise CommandExecutionError(
-                "Could not resolve Security Group name "
-                "{0} to a Group ID".format(group)
-            )
+            # But... if we're running in test mode, it may just be that the SG is scheduled
+            # to be created, and thus WOULD have been there if running "for real"...
+            if __opts__["test"]:
+                log.warn(
+                    "Security Group `%s` could not be resolved to an ID.  This may "
+                    "cause a failure when not running in test mode.",
+                    group,
+                )
+                return []
+            else:
+                raise CommandExecutionError(
+                    "Could not resolve Security Group name "
+                    "{0} to a Group ID".format(group)
+                )
         else:
             group_ids.append(six.text_type(group_id))
     log.debug("security group contents %s post-conversion", group_ids)

salt/modules/opkg.py

@@ -289,14 +289,91 @@ def refresh_db(failhard=False, **kwargs):  # pylint: disable=unused-argument
     return ret
 
 
+def _is_testmode(**kwargs):
+    """
+    Returns whether a test mode (noaction) operation was requested.
+    """
+    return bool(kwargs.get("test") or __opts__.get("test"))
+
+
 def _append_noaction_if_testmode(cmd, **kwargs):
     """
     Adds the --noaction flag to the command if it's running in the test mode.
     """
-    if bool(kwargs.get("test") or __opts__.get("test")):
+    if _is_testmode(**kwargs):
         cmd.append("--noaction")
 
 
+def _build_install_command_list(cmd_prefix, to_install, to_downgrade, to_reinstall):
+    """
+    Builds a list of install commands to be executed in sequence in order to process
+    each of the to_install, to_downgrade, and to_reinstall lists.
+    """
+    cmds = []
+    if to_install:
+        cmd = copy.deepcopy(cmd_prefix)
+        cmd.extend(to_install)
+        cmds.append(cmd)
+    if to_downgrade:
+        cmd = copy.deepcopy(cmd_prefix)
+        cmd.append("--force-downgrade")
+        cmd.extend(to_downgrade)
+        cmds.append(cmd)
+    if to_reinstall:
+        cmd = copy.deepcopy(cmd_prefix)
+        cmd.append("--force-reinstall")
+        cmd.extend(to_reinstall)
+        cmds.append(cmd)
+
+    return cmds
+
+
+def _parse_reported_packages_from_install_output(output):
+    """
+    Parses the output of "opkg install" to determine what packages would have been
+    installed by an operation run with the --noaction flag.
+
+    We are looking for lines like:
+        Installing <package> (<version>) on <target>
+    or
+        Upgrading <package> from <oldVersion> to <version> on root
+    """
+    reported_pkgs = {}
+    install_pattern = re.compile(
+        r"Installing\s(?P<package>.*?)\s\((?P<version>.*?)\)\son\s(?P<target>.*?)"
+    )
+    upgrade_pattern = re.compile(
+        r"Upgrading\s(?P<package>.*?)\sfrom\s(?P<oldVersion>.*?)\sto\s(?P<version>.*?)\son\s(?P<target>.*?)"
+    )
+    for line in salt.utils.itertools.split(output, "\n"):
+        match = install_pattern.match(line)
+        if match is None:
+            match = upgrade_pattern.match(line)
+        if match:
+            reported_pkgs[match.group("package")] = match.group("version")
+
+    return reported_pkgs
+
+
+def _execute_install_command(cmd, parse_output, errors, parsed_packages):
+    """
+    Executes a command for the install operation.
+    If the command fails, its error output will be appended to the errors list.
+    If the command succeeds and parse_output is true, updated packages will be appended
+    to the parsed_packages dictionary.
+    """
+    out = __salt__["cmd.run_all"](cmd, output_loglevel="trace", python_shell=False)
+    if out["retcode"] != 0:
+        if out["stderr"]:
+            errors.append(out["stderr"])
+        else:
+            errors.append(out["stdout"])
+    elif parse_output:
+        parsed_packages.update(
+            _parse_reported_packages_from_install_output(out["stdout"])
+        )
+
+
 def install(
     name=None, refresh=False, pkgs=None, sources=None, reinstall=False, **kwargs
 ):
@@ -440,24 +517,9 @@ def install(
                         # This should cause the command to fail.
                         to_install.append(pkgstr)
 
-    cmds = []
-
-    if to_install:
-        cmd = copy.deepcopy(cmd_prefix)
-        cmd.extend(to_install)
-        cmds.append(cmd)
-
-    if to_downgrade:
-        cmd = copy.deepcopy(cmd_prefix)
-        cmd.append("--force-downgrade")
-        cmd.extend(to_downgrade)
-        cmds.append(cmd)
-
-    if to_reinstall:
-        cmd = copy.deepcopy(cmd_prefix)
-        cmd.append("--force-reinstall")
-        cmd.extend(to_reinstall)
-        cmds.append(cmd)
+    cmds = _build_install_command_list(
+        cmd_prefix, to_install, to_downgrade, to_reinstall
+    )
 
     if not cmds:
         return {}
@@ -466,16 +528,17 @@ def install(
         refresh_db()
 
     errors = []
+    is_testmode = _is_testmode(**kwargs)
+    test_packages = {}
     for cmd in cmds:
-        out = __salt__["cmd.run_all"](cmd, output_loglevel="trace", python_shell=False)
-        if out["retcode"] != 0:
-            if out["stderr"]:
-                errors.append(out["stderr"])
-            else:
-                errors.append(out["stdout"])
+        _execute_install_command(cmd, is_testmode, errors, test_packages)
 
     __context__.pop("pkg.list_pkgs", None)
     new = list_pkgs()
+    if is_testmode:
+        new = copy.deepcopy(new)
+        new.update(test_packages)
+
     ret = salt.utils.data.compare_dicts(old, new)
 
     if pkg_type == "file" and reinstall:
@@ -513,6 +576,26 @@ def install(
     return ret
 
 
+def _parse_reported_packages_from_remove_output(output):
+    """
+    Parses the output of "opkg remove" to determine what packages would have been
+    removed by an operation run with the --noaction flag.
+
+    We are looking for lines like
+        Removing <package> (<version>) from <Target>...
+    """
+    reported_pkgs = {}
+    remove_pattern = re.compile(
+        r"Removing\s(?P<package>.*?)\s\((?P<version>.*?)\)\sfrom\s(?P<target>.*?)..."
+    )
+    for line in salt.utils.itertools.split(output, "\n"):
+        match = remove_pattern.match(line)
+        if match:
+            reported_pkgs[match.group("package")] = ""
+
+    return reported_pkgs
+
+
 def remove(name=None, pkgs=None, **kwargs):  # pylint: disable=unused-argument
     """
     Remove packages using ``opkg remove``.
@@ -576,6 +659,9 @@ def remove(name=None, pkgs=None, **kwargs):  # pylint: disable=unused-argument
 
     __context__.pop("pkg.list_pkgs", None)
     new = list_pkgs()
+    if _is_testmode(**kwargs):
+        reportedPkgs = _parse_reported_packages_from_remove_output(out["stdout"])
+        new = {k: v for k, v in new.items() if k not in reportedPkgs}
     ret = salt.utils.data.compare_dicts(old, new)
 
     rs_result = _get_restartcheck_result(errors)