Unproxy objects before serializing them.

The salt.utils.json module now takes care of unwrapping objects that are
proxied using the ThreadLocalProxy.

The salt.utils.msgpack module has been added and basically provides the
same functions as the salt.utils.json module, but for msgpack. Like the
json module, it takes care of unwrapping proxies.

salt/cloud/clouds/ec2.py

@@ -82,7 +82,6 @@
 import binascii
 import datetime
 import base64
-import msgpack
 import re
 import decimal
 
@@ -91,6 +90,7 @@
 import salt.utils.files
 import salt.utils.hashutils
 import salt.utils.json
+import salt.utils.msgpack
 import salt.utils.stringutils
 import salt.utils.yaml
 from salt._compat import ElementTree as ET
@@ -4828,7 +4828,7 @@ def _parse_pricing(url, name):
         __opts__['cachedir'], 'ec2-pricing-{0}.p'.format(name)
     )
     with salt.utils.files.fopen(outfile, 'w') as fho:
-        msgpack.dump(regions, fho)
+        salt.utils.msgpack.dump(regions, fho)
 
     return True
 
@@ -4896,7 +4896,8 @@ def show_pricing(kwargs=None, call=None):
         update_pricing({'type': name}, 'function')
 
     with salt.utils.files.fopen(pricefile, 'r') as fhi:
-        ec2_price = salt.utils.stringutils.to_unicode(msgpack.load(fhi))
+        ec2_price = salt.utils.stringutils.to_unicode(
+            salt.utils.msgpack.load(fhi))
 
     region = get_location(profile)
     size = profile.get('size', None)

salt/cloud/clouds/gce.py

@@ -53,7 +53,6 @@
 import re
 import pprint
 import logging
-import msgpack
 from ast import literal_eval
 from salt.utils.versions import LooseVersion as _LooseVersion
 
@@ -90,6 +89,7 @@
 import salt.utils.cloud
 import salt.utils.files
 import salt.utils.http
+import salt.utils.msgpack
 import salt.config as config
 from salt.cloud.libcloudfuncs import *  # pylint: disable=redefined-builtin,wildcard-import,unused-wildcard-import
 from salt.exceptions import (
@@ -2618,7 +2618,7 @@ def update_pricing(kwargs=None, call=None):
         __opts__['cachedir'], 'gce-pricing.p'
     )
     with salt.utils.files.fopen(outfile, 'w') as fho:
-        msgpack.dump(price_json['dict'], fho)
+        salt.utils.msgpack.dump(price_json['dict'], fho)
 
     return True
 
@@ -2657,7 +2657,7 @@ def show_pricing(kwargs=None, call=None):
         update_pricing()
 
     with salt.utils.files.fopen(pricefile, 'r') as fho:
-        sizes = msgpack.load(fho)
+        sizes = salt.utils.msgpack.load(fho)
 
     per_hour = float(sizes['gcp_price_list'][size][region])
 

salt/engines/stalekey.py

@@ -28,11 +28,11 @@
 import salt.key
 import salt.utils.files
 import salt.utils.minions
+import salt.utils.msgpack
 import salt.wheel
 
 # Import 3rd-party libs
 from salt.ext import six
-import msgpack
 
 log = logging.getLogger(__name__)
 
@@ -60,7 +60,7 @@ def start(interval=3600, expire=604800):
         if os.path.exists(presence_file):
             try:
                 with salt.utils.files.fopen(presence_file, 'r') as f:
-                    minions = msgpack.load(f)
+                    minions = salt.utils.msgpack.load(f)
             except IOError as e:
                 log.error('Could not open presence file %s: %s', presence_file, e)
                 time.sleep(interval)
@@ -95,7 +95,7 @@ def start(interval=3600, expire=604800):
 
         try:
             with salt.utils.files.fopen(presence_file, 'w') as f:
-                msgpack.dump(minions, f)
+                salt.utils.msgpack.dump(minions, f)
         except IOError as e:
             log.error('Could not write to presence file %s: %s', presence_file, e)
         time.sleep(interval)

salt/key.py

@@ -38,9 +38,10 @@
 from salt.ext.six.moves import input
 # pylint: enable=import-error,no-name-in-module,redefined-builtin
 
-# Import third party libs
+# We do not always need msgpack, so we do not want to fail here if msgpack is
+# not available.
 try:
-    import msgpack
+    import salt.utils.msgpack
 except ImportError:
     pass
 
@@ -1035,7 +1036,8 @@ def check_minion_cache(self, preserve_minions=False):
                     if ext == '.json':
                         data = salt.utils.json.load(fp_)
                     elif ext == '.msgpack':
-                        data = msgpack.load(fp_)
+                        data = salt.utils.msgpack.load(fp_,
+                                                       _msgpack_module=msgpack)
                     role = salt.utils.stringutils.to_unicode(data['role'])
                     if role not in minions:
                         os.remove(path)

salt/log/handlers/fluent_mod.py

@@ -96,15 +96,17 @@
 
 try:
     # Attempt to import msgpack
-    import msgpack
+    import salt.utils.msgpack
     # There is a serialization issue on ARM and potentially other platforms
     # for some msgpack bindings, check for it
     if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None:
         raise ImportError
+    import salt.utils.msgpack
 except ImportError:
     # Fall back to msgpack_pure
     try:
         import msgpack_pure as msgpack
+        import salt.utils.msgpack
     except ImportError:
         # TODO: Come up with a sane way to get a configured logfile
         #       and write to the logfile when this error is hit also
@@ -456,7 +458,7 @@ def _make_packet(self, label, timestamp, data):
         packet = (tag, timestamp, data)
         if self.verbose:
             print(packet)
-        return msgpack.packb(packet)
+        return salt.utils.msgpack.packb(packet, _msgpack_module=msgpack)
 
     def _send(self, bytes_):
         self.lock.acquire()

salt/modules/saltcheck.py

@@ -51,7 +51,7 @@
 import logging
 import os
 import time
-from json import loads, dumps
+from salt.utils.json import loads, dumps
 try:
     import salt.utils.files
     import salt.utils.path

salt/modules/state.py

@@ -34,6 +34,7 @@
 import salt.utils.hashutils
 import salt.utils.jid
 import salt.utils.json
+import salt.utils.msgpack
 import salt.utils.platform
 import salt.utils.state
 import salt.utils.stringutils
@@ -45,7 +46,6 @@
 
 # Import 3rd-party libs
 from salt.ext import six
-import msgpack
 
 __proxyenabled__ = ['*']
 
@@ -185,7 +185,7 @@ def _get_pause(jid, state_id=None):
             data[state_id] = {}
     if os.path.exists(pause_path):
         with salt.utils.files.fopen(pause_path, 'rb') as fp_:
-            data = msgpack.loads(fp_.read())
+            data = salt.utils.msgpack.loads(fp_.read())
     return data, pause_path
 
 
@@ -256,7 +256,7 @@ def soft_kill(jid, state_id=None):
     data, pause_path = _get_pause(jid, state_id)
     data[state_id]['kill'] = True
     with salt.utils.files.fopen(pause_path, 'wb') as fp_:
-        fp_.write(msgpack.dumps(data))
+        fp_.write(salt.utils.msgpack.dumps(data))
 
 
 def pause(jid, state_id=None, duration=None):
@@ -291,7 +291,7 @@ def pause(jid, state_id=None, duration=None):
     if duration:
         data[state_id]['duration'] = int(duration)
     with salt.utils.files.fopen(pause_path, 'wb') as fp_:
-        fp_.write(msgpack.dumps(data))
+        fp_.write(salt.utils.msgpack.dumps(data))
 
 
 def resume(jid, state_id=None):
@@ -325,7 +325,7 @@ def resume(jid, state_id=None):
     if state_id == '__all__':
         data = {}
     with salt.utils.files.fopen(pause_path, 'wb') as fp_:
-        fp_.write(msgpack.dumps(data))
+        fp_.write(salt.utils.msgpack.dumps(data))
 
 
 def orchestrate(mods,

salt/modules/win_repo.py

@@ -31,11 +31,8 @@
     PER_REMOTE_ONLY
 )
 from salt.ext import six
-try:
-    import msgpack
-except ImportError:
-    import msgpack_pure as msgpack  # pylint: disable=import-error
 import salt.utils.gitfs
+import salt.utils.msgpack
 # pylint: enable=unused-import
 
 log = logging.getLogger(__name__)

salt/payload.py

@@ -54,6 +54,10 @@
         #sys.exit(salt.defaults.exitcodes.EX_GENERIC)
 
 
+if HAS_MSGPACK:
+    import salt.utils.msgpack
+
+
 if HAS_MSGPACK and not hasattr(msgpack, 'exceptions'):
     class PackValueError(Exception):
         '''
@@ -74,14 +78,15 @@ def package(payload):
     This method for now just wraps msgpack.dumps, but it is here so that
     we can make the serialization a custom option in the future with ease.
     '''
-    return msgpack.dumps(payload)
+    return salt.utils.msgpack.dumps(payload, _msgpack_module=msgpack)
 
 
 def unpackage(package_):
     '''
     Unpackages a payload
     '''
-    return msgpack.loads(package_, use_list=True)
+    return salt.utils.msgpack.loads(package_, use_list=True,
+                                    _msgpack_module=msgpack)
 
 
 def format_payload(enc, **kwargs):
@@ -134,9 +139,12 @@ def loads(self, msg, encoding=None, raw=False):
                 # Due to this, if we don't need it, don't pass it at all so
                 # that under Python 2 we can still work with older versions
                 # of msgpack.
-                ret = msgpack.loads(msg, use_list=True, encoding=encoding)
+                ret = salt.utils.msgpack.loads(msg, use_list=True,
+                                               encoding=encoding,
+                                               _msgpack_module=msgpack)
             else:
-                ret = msgpack.loads(msg, use_list=True)
+                ret = salt.utils.msgpack.loads(msg, use_list=True,
+                                               _msgpack_module=msgpack)
             if six.PY3 and encoding is None and not raw:
                 ret = salt.transport.frame.decode_embedded_strs(ret)
         except Exception as exc:
@@ -181,9 +189,10 @@ def dumps(self, msg, use_bin_type=False):
                 # Due to this, if we don't need it, don't pass it at all so
                 # that under Python 2 we can still work with older versions
                 # of msgpack.
-                return msgpack.dumps(msg, use_bin_type=use_bin_type)
+                return salt.utils.msgpack.dumps(msg, use_bin_type=use_bin_type,
+                                                _msgpack_module=msgpack)
             else:
-                return msgpack.dumps(msg)
+                return salt.utils.msgpack.dumps(msg, _msgpack_module=msgpack)
         except (OverflowError, msgpack.exceptions.PackValueError):
             # msgpack can't handle the very long Python longs for jids
             # Convert any very long longs to strings
@@ -207,9 +216,12 @@ def verylong_encoder(obj):
                 else:
                     return obj
             if msgpack.version >= (0, 4, 0):
-                return msgpack.dumps(verylong_encoder(msg), use_bin_type=use_bin_type)
+                return salt.utils.msgpack.dumps(verylong_encoder(msg),
+                                                use_bin_type=use_bin_type,
+                                                _msgpack_module=msgpack)
             else:
-                return msgpack.dumps(verylong_encoder(msg))
+                return salt.utils.msgpack.dumps(verylong_encoder(msg),
+                                                _msgpack_module=msgpack)
         except TypeError as e:
             # msgpack doesn't support datetime.datetime datatype
             # So here we have converted datetime.datetime to custom datatype
@@ -220,9 +232,14 @@ def default(obj):
             def dt_encode(obj):
                 datetime_str = obj.strftime("%Y%m%dT%H:%M:%S.%f")
                 if msgpack.version >= (0, 4, 0):
-                    return msgpack.packb(datetime_str, default=default, use_bin_type=use_bin_type)
+                    return salt.utils.msgpack.packb(datetime_str,
+                                                    default=default,
+                                                    use_bin_type=use_bin_type,
+                                                    _msgpack_module=msgpack)
                 else:
-                    return msgpack.packb(datetime_str, default=default)
+                    return salt.utils.msgpack.packb(datetime_str,
+                                                    default=default,
+                                                    _msgpack_module=msgpack)
 
             def datetime_encoder(obj):
                 if isinstance(obj, dict):
@@ -254,14 +271,22 @@ def immutable_encoder(obj):
 
             if "datetime.datetime" in six.text_type(e):
                 if msgpack.version >= (0, 4, 0):
-                    return msgpack.dumps(datetime_encoder(msg), use_bin_type=use_bin_type)
+                    return salt.utils.msgpack.dumps(datetime_encoder(msg),
+                                                    use_bin_type=use_bin_type,
+                                                    _msgpack_module=msgpack)
                 else:
-                    return msgpack.dumps(datetime_encoder(msg))
+                    return salt.utils.msgpack.dumps(datetime_encoder(msg),
+                                                    _msgpack_module=msgpack)
             elif "Immutable" in six.text_type(e):
                 if msgpack.version >= (0, 4, 0):
-                    return msgpack.dumps(msg, default=immutable_encoder, use_bin_type=use_bin_type)
+                    return salt.utils.msgpack.dumps(msg,
+                                                    default=immutable_encoder,
+                                                    use_bin_type=use_bin_type,
+                                                    _msgpack_module=msgpack)
                 else:
-                    return msgpack.dumps(msg, default=immutable_encoder)
+                    return salt.utils.msgpack.dumps(msg,
+                                                    default=immutable_encoder,
+                                                    _msgpack_module=msgpack)
 
             if msgpack.version >= (0, 2, 0):
                 # Should support OrderedDict serialization, so, let's
@@ -286,9 +311,12 @@ def odict_encoder(obj):
                     return obj
                 return obj
             if msgpack.version >= (0, 4, 0):
-                return msgpack.dumps(odict_encoder(msg), use_bin_type=use_bin_type)
+                return salt.utils.msgpack.dumps(odict_encoder(msg),
+                                                use_bin_type=use_bin_type,
+                                                _msgpack_module=msgpack)
             else:
-                return msgpack.dumps(odict_encoder(msg))
+                return salt.utils.msgpack.dumps(odict_encoder(msg),
+                                                _msgpack_module=msgpack)
         except (SystemError, TypeError) as exc:  # pylint: disable=W0705
             log.critical(
                 'Unable to serialize message! Consider upgrading msgpack. '

salt/renderers/msgpack.py

@@ -1,10 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, unicode_literals
 
-# Import third party libs
-import msgpack
-
 # Import salt libs
+import salt.utils.msgpack
 from salt.ext import six
 
 
@@ -28,4 +26,4 @@ def render(msgpack_data, saltenv='base', sls='', **kws):
         msgpack_data = msgpack_data[(msgpack_data.find('\n') + 1):]
     if not msgpack_data.strip():
         return {}
-    return msgpack.loads(msgpack_data)
+    return salt.utils.msgpack.loads(msgpack_data)