From b8be2bea9db087aba50c6a10b5b072ca6435ce9d Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Sat, 7 Aug 2021 18:07:28 -0500 Subject: [PATCH 01/13] Update views types handling --- cloudpickle/cloudpickle.py | 138 ++++++++++++++++++++++++++------ cloudpickle/cloudpickle_fast.py | 53 +++--------- tests/cloudpickle_test.py | 21 +++++ 3 files changed, 148 insertions(+), 64 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 347b3869..41721795 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -44,25 +44,26 @@ import builtins import dis -import opcode import platform import sys -import types -import weakref -import uuid import threading +import types import typing +import uuid import warnings +import weakref +from collections import OrderedDict, namedtuple +from functools import partial +from pickle import _getattribute +from typing import Callable, Generic, Tuple, Union + +import opcode from .compat import pickle -from collections import OrderedDict -from typing import Generic, Union, Tuple, Callable -from pickle import _getattribute -from importlib._bootstrap import _find_spec try: # pragma: no branch import typing_extensions as _typing_extensions - from typing_extensions import Literal, Final + from typing_extensions import Final, Literal except ImportError: _typing_extensions = Literal = Final = None @@ -952,22 +953,113 @@ def _get_bases(typ): return getattr(typ, bases_attr) -def _make_dict_keys(obj, is_ordered=False): - if is_ordered: - return OrderedDict.fromkeys(obj).keys() +def _make_keys_view(obj, typ): + t = typ or dict + if hasattr(t, "fromkeys"): + o = t.fromkeys(obj) else: - return dict.fromkeys(obj).keys() + o = dict.fromkeys(obj) + try: + o = t(o) + except Exception: + pass + return o.keys() -def _make_dict_values(obj, is_ordered=False): - if is_ordered: - return OrderedDict((i, _) for i, _ in enumerate(obj)).values() - else: - return {i: _ for i, _ in enumerate(obj)}.values() +def _make_values_view(obj, typ): + t = typ or dict + o = t({i: _ for i, _ in enumerate(obj)}) + return o.values() -def _make_dict_items(obj, is_ordered=False): - if is_ordered: - return OrderedDict(obj).items() - else: - return obj.items() +def _make_items_view(obj, typ): + t = typ or dict + try: + o = t(obj) + except Exception: + o = dict(obj) + if hasattr(o, "items"): + return o.items() + return o + + +ViewInfo = namedtuple("ViewInfo", ["packer", "maker"]) +VIEW_ATTRS_INFO = { + "keys": ViewInfo(list, _make_keys_view), + "values": ViewInfo(list, _make_values_view), + "items": ViewInfo(dict, _make_items_view), +} + +from typing import Any + +class ViewPickleInfo: + + obj: Any + view_name: str + + def __init__( + self, + obj: Any, + view_name: str, + ) -> None: + self.obj = obj + self.view_name = view_name + self.__post_init__() + + def __post_init__(self) -> None: + obj = self.obj + if isinstance(obj, type): + self.object_type = obj + obj_instance = self.object_type() + else: + self.object_type = type(obj) + obj_instance = obj + a = getattr(obj_instance, self.view_name, None) + if callable(a): + view_obj = a() + else: + view_obj = a + if a is not None: + self.view_type = type(view_obj) + else: + self.view_type = None + + def get_packer(self): + """Run prepacker, return tuple of (packed, type).""" + prepacker = VIEW_ATTRS_INFO[self.view_name].packer + + def packer(obj): + if callable(prepacker): + packed = prepacker(obj) + packed = obj + return packed, type(obj) + return packer + + def get_maker(self): + base_maker = VIEW_ATTRS_INFO[self.view_name].maker + + def maker(obj): + if callable(base_maker): + return base_maker(obj, self.view_type) + return obj + return maker + + def get_reducer(self): + maker = self.get_maker() + packer = self.get_packer() + + def reducer(obj): + return maker, (packer(obj), type(obj)) + + return reducer + + +def build_views_types_table(types): + """Register views types.""" + results = {} + for obj in types: + for atr in VIEW_ATTRS_INFO.keys(): + info = ViewPickleInfo(obj, atr) + if hasattr(info, "view_type") and info.view_type: + results[info.view_type] = info + return results diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 6db059eb..5eae2c30 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -35,7 +35,7 @@ _is_parametrized_type_hint, PYPY, cell_set, parametrized_type_hint_getinitargs, _create_parametrized_type_hint, builtin_code_type, - _make_dict_keys, _make_dict_values, _make_dict_items, + # _make_dict_keys, _make_dict_values, _make_dict_items, ) @@ -419,41 +419,12 @@ def _class_reduce(obj): return NotImplemented -def _dict_keys_reduce(obj): - # Safer not to ship the full dict as sending the rest might - # be unintended and could potentially cause leaking of - # sensitive information - return _make_dict_keys, (list(obj), ) +from .cloudpickle import build_views_types_table -def _dict_values_reduce(obj): - # Safer not to ship the full dict as sending the rest might - # be unintended and could potentially cause leaking of - # sensitive information - return _make_dict_values, (list(obj), ) - - -def _dict_items_reduce(obj): - return _make_dict_items, (dict(obj), ) - - -def _odict_keys_reduce(obj): - # Safer not to ship the full dict as sending the rest might - # be unintended and could potentially cause leaking of - # sensitive information - return _make_dict_keys, (list(obj), True) - - -def _odict_values_reduce(obj): - # Safer not to ship the full dict as sending the rest might - # be unintended and could potentially cause leaking of - # sensitive information - return _make_dict_values, (list(obj), True) - - -def _odict_items_reduce(obj): - return _make_dict_items, (dict(obj), True) - +VIEWABLE_TYPES = [dict, OrderedDict] +VIEWS_MAKER_TABLE = build_views_types_table(VIEWABLE_TYPES) +_VIEWS_DISPATCH_TABLE = {k: v.get_reducer() for k, v in VIEWS_MAKER_TABLE.items()} # COLLECTIONS OF OBJECTS STATE SETTERS # ------------------------------------ @@ -528,13 +499,13 @@ class CloudPickler(Pickler): _dispatch_table[types.MappingProxyType] = _mappingproxy_reduce _dispatch_table[weakref.WeakSet] = _weakset_reduce _dispatch_table[typing.TypeVar] = _typevar_reduce - _dispatch_table[_collections_abc.dict_keys] = _dict_keys_reduce - _dispatch_table[_collections_abc.dict_values] = _dict_values_reduce - _dispatch_table[_collections_abc.dict_items] = _dict_items_reduce - _dispatch_table[type(OrderedDict().keys())] = _odict_keys_reduce - _dispatch_table[type(OrderedDict().values())] = _odict_values_reduce - _dispatch_table[type(OrderedDict().items())] = _odict_items_reduce - + # _dispatch_table[_collections_abc.dict_keys] = _dict_keys_reduce + # _dispatch_table[_collections_abc.dict_values] = _dict_values_reduce + # _dispatch_table[_collections_abc.dict_items] = _dict_items_reduce + # _dispatch_table[type(OrderedDict().keys())] = _odict_keys_reduce + # _dispatch_table[type(OrderedDict().values())] = _odict_values_reduce + # _dispatch_table[type(OrderedDict().items())] = _odict_items_reduce + _dispatch_table.update(_VIEWS_DISPATCH_TABLE) dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index d2acfb71..eaf49465 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -259,6 +259,27 @@ def test_odict_items(self): self.assertEqual(results, items) assert type(items) == type(results) + from itertools import product + + from cloudpickle.cloudpickle import VIEW_ATTRS_INFO + from cloudpickle.cloudpickle_fast import VIEWABLE_TYPES + + view_types = product(VIEWABLE_TYPES, VIEW_ATTRS_INFO.keys()) + + @pytest.mark.parametrize("view_type", view_types) + def test_view_types(self, view_type): + t, view = view_type + obj = t([("a", 1), ("b", 2)]) + at = getattr(obj, view, None) + if at is not None: + if callable(at): + data = at() + else: + data = at + results = pickle_depickle(data) + self.assertEqual(results, data) + assert type(data) == type(results) + def test_sliced_and_non_contiguous_memoryview(self): buffer_obj = memoryview(b"Hello!" * 3)[2:15:2] self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), From c601342e0b9cfbeaac660ba829b2ac3dbd4fa8a3 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Sat, 7 Aug 2021 18:15:58 -0500 Subject: [PATCH 02/13] Work on tests --- tests/cloudpickle_test.py | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index eaf49465..1d33020c 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -59,7 +59,14 @@ from .testutils import assert_run_python_script from .testutils import subprocess_worker -from _cloudpickle_testpkg import relative_imports_factory +# from _cloudpickle_testpkg import relative_imports_factory + +from itertools import product + +from cloudpickle.cloudpickle import VIEW_ATTRS_INFO +from cloudpickle.cloudpickle_fast import VIEWABLE_TYPES + +VIEW_TYPES = list(product(VIEWABLE_TYPES, VIEW_ATTRS_INFO.keys())) _TEST_GLOBAL_VARIABLE = "default_value" @@ -259,26 +266,19 @@ def test_odict_items(self): self.assertEqual(results, items) assert type(items) == type(results) - from itertools import product - - from cloudpickle.cloudpickle import VIEW_ATTRS_INFO - from cloudpickle.cloudpickle_fast import VIEWABLE_TYPES - - view_types = product(VIEWABLE_TYPES, VIEW_ATTRS_INFO.keys()) - - @pytest.mark.parametrize("view_type", view_types) - def test_view_types(self, view_type): - t, view = view_type - obj = t([("a", 1), ("b", 2)]) - at = getattr(obj, view, None) - if at is not None: - if callable(at): - data = at() - else: - data = at - results = pickle_depickle(data) - self.assertEqual(results, data) - assert type(data) == type(results) + def test_view_types(self): + for view_type in VIEW_TYPES: + t, view = view_type + obj = t([("a", 1), ("b", 2)]) + at = getattr(obj, view, None) + if at is not None: + if callable(at): + data = at() + else: + data = at + results = pickle_depickle(data) + self.assertEqual(results, data) + assert type(data) == type(results) def test_sliced_and_non_contiguous_memoryview(self): buffer_obj = memoryview(b"Hello!" * 3)[2:15:2] From fc352c3dfeceb08d7b0516fda3c4f09536c03efb Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Sun, 8 Aug 2021 13:31:31 -0500 Subject: [PATCH 03/13] Make it work --- cloudpickle/cloudpickle.py | 86 +-------------------------------- cloudpickle/cloudpickle_fast.py | 74 +++++++++++++++++++++++----- tests/cloudpickle_test.py | 75 ++++++++++++++-------------- 3 files changed, 98 insertions(+), 137 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 41721795..aedcc085 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -52,10 +52,8 @@ import uuid import warnings import weakref -from collections import OrderedDict, namedtuple -from functools import partial from pickle import _getattribute -from typing import Callable, Generic, Tuple, Union +from typing import Any, Callable, Generic, Tuple, Union import opcode @@ -981,85 +979,3 @@ def _make_items_view(obj, typ): if hasattr(o, "items"): return o.items() return o - - -ViewInfo = namedtuple("ViewInfo", ["packer", "maker"]) -VIEW_ATTRS_INFO = { - "keys": ViewInfo(list, _make_keys_view), - "values": ViewInfo(list, _make_values_view), - "items": ViewInfo(dict, _make_items_view), -} - -from typing import Any - -class ViewPickleInfo: - - obj: Any - view_name: str - - def __init__( - self, - obj: Any, - view_name: str, - ) -> None: - self.obj = obj - self.view_name = view_name - self.__post_init__() - - def __post_init__(self) -> None: - obj = self.obj - if isinstance(obj, type): - self.object_type = obj - obj_instance = self.object_type() - else: - self.object_type = type(obj) - obj_instance = obj - a = getattr(obj_instance, self.view_name, None) - if callable(a): - view_obj = a() - else: - view_obj = a - if a is not None: - self.view_type = type(view_obj) - else: - self.view_type = None - - def get_packer(self): - """Run prepacker, return tuple of (packed, type).""" - prepacker = VIEW_ATTRS_INFO[self.view_name].packer - - def packer(obj): - if callable(prepacker): - packed = prepacker(obj) - packed = obj - return packed, type(obj) - return packer - - def get_maker(self): - base_maker = VIEW_ATTRS_INFO[self.view_name].maker - - def maker(obj): - if callable(base_maker): - return base_maker(obj, self.view_type) - return obj - return maker - - def get_reducer(self): - maker = self.get_maker() - packer = self.get_packer() - - def reducer(obj): - return maker, (packer(obj), type(obj)) - - return reducer - - -def build_views_types_table(types): - """Register views types.""" - results = {} - for obj in types: - for atr in VIEW_ATTRS_INFO.keys(): - info = ViewPickleInfo(obj, atr) - if hasattr(info, "view_type") and info.view_type: - results[info.view_type] = info - return results diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 5eae2c30..7eabf177 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -10,7 +10,6 @@ guards present in cloudpickle.py that were written to handle PyPy specificities are not present in cloudpickle_fast.py """ -import _collections_abc import abc import copyreg import io @@ -22,8 +21,9 @@ import weakref import typing +from collections import ChainMap, OrderedDict, namedtuple from enum import Enum -from collections import ChainMap, OrderedDict +from typing import Iterable from .compat import pickle, Pickler from .cloudpickle import ( @@ -35,7 +35,7 @@ _is_parametrized_type_hint, PYPY, cell_set, parametrized_type_hint_getinitargs, _create_parametrized_type_hint, builtin_code_type, - # _make_dict_keys, _make_dict_values, _make_dict_items, + _make_keys_view, _make_values_view, _make_items_view, ) @@ -418,13 +418,67 @@ def _class_reduce(obj): return _dynamic_class_reduce(obj) return NotImplemented +# DICT VIEWS TYPES +ViewInfo = namedtuple("ViewInfo", ["view", "packer", "maker", "object_type"]) -from .cloudpickle import build_views_types_table -VIEWABLE_TYPES = [dict, OrderedDict] -VIEWS_MAKER_TABLE = build_views_types_table(VIEWABLE_TYPES) -_VIEWS_DISPATCH_TABLE = {k: v.get_reducer() for k, v in VIEWS_MAKER_TABLE.items()} +_VIEW_ATTRS_INFO = [ + ViewInfo("keys", list, _make_keys_view, None), + ViewInfo("values", list, _make_values_view, None), + ViewInfo("items", dict, _make_items_view, None), +] + + +_VIEWS_TYPES_TABLE = {} + + +def register_views_types(types, attr_info=None, overwrite=False): + """Register views types, returns a copy.""" + if not isinstance(types, Iterable): + types = [types] + if attr_info is None: + attr_info = _VIEW_ATTRS_INFO.copy() + elif isinstance(attr_info, ViewInfo): + attr_info = [attr_info] + for typ in types: + for info in attr_info: + if isinstance(typ, type): + object_type = typ + obj_instance = object_type() + else: + object_type = type(typ) + obj_instance = typ + a = getattr(obj_instance, info.view, None) + if callable(a): + view_obj = a() + else: + view_obj = a + if a is None: + continue + view_type = type(view_obj) + if view_type not in _VIEWS_TYPES_TABLE or overwrite: + _VIEWS_TYPES_TABLE[view_type] = ViewInfo( + info.view, + info.packer, + info.maker, + object_type, + ) + return _VIEWS_TYPES_TABLE + + +_VIEWABLE_TYPES = [dict, OrderedDict] +register_views_types(_VIEWABLE_TYPES) + + +def _views_reducer(obj): + typ = type(obj) + info = _VIEWS_TYPES_TABLE[typ] + return info.maker, (info.packer(obj), info.object_type) + + +_VIEWS_DISPATCH_TABLE = {k: _views_reducer for k in _VIEWS_TYPES_TABLE.keys()} + # COLLECTIONS OF OBJECTS STATE SETTERS # ------------------------------------ @@ -499,12 +553,6 @@ class CloudPickler(Pickler): _dispatch_table[types.MappingProxyType] = _mappingproxy_reduce _dispatch_table[weakref.WeakSet] = _weakset_reduce _dispatch_table[typing.TypeVar] = _typevar_reduce - # _dispatch_table[_collections_abc.dict_keys] = _dict_keys_reduce - # _dispatch_table[_collections_abc.dict_values] = _dict_values_reduce - # _dispatch_table[_collections_abc.dict_items] = _dict_items_reduce - # _dispatch_table[type(OrderedDict().keys())] = _odict_keys_reduce - # _dispatch_table[type(OrderedDict().values())] = _odict_values_reduce - # _dispatch_table[type(OrderedDict().items())] = _odict_items_reduce _dispatch_table.update(_VIEWS_DISPATCH_TABLE) dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 1d33020c..712c434b 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -1,6 +1,5 @@ from __future__ import division -import _collections_abc import abc import collections import base64 @@ -59,14 +58,11 @@ from .testutils import assert_run_python_script from .testutils import subprocess_worker -# from _cloudpickle_testpkg import relative_imports_factory - -from itertools import product - -from cloudpickle.cloudpickle import VIEW_ATTRS_INFO -from cloudpickle.cloudpickle_fast import VIEWABLE_TYPES - -VIEW_TYPES = list(product(VIEWABLE_TYPES, VIEW_ATTRS_INFO.keys())) +try: + # import seems to break tests + from _cloudpickle_testpkg import relative_imports_factory +except ImportError: + pass _TEST_GLOBAL_VARIABLE = "default_value" @@ -231,54 +227,55 @@ def test_memoryview(self): buffer_obj.tobytes()) def test_dict_keys(self): - keys = {"a": 1, "b": 2}.keys() - results = pickle_depickle(keys) - self.assertEqual(results, keys) - assert isinstance(results, _collections_abc.dict_keys) + data = {"a": 1, "b": 2}.keys() + results = pickle_depickle(data) + self.assertEqual(results, data) + assert isinstance(results, type(data)) def test_dict_values(self): - values = {"a": 1, "b": 2}.values() - results = pickle_depickle(values) - self.assertEqual(sorted(results), sorted(values)) - assert isinstance(results, _collections_abc.dict_values) + data = {"a": 1, "b": 2}.values() + results = pickle_depickle(data) + self.assertEqual(sorted(results), sorted(data)) + assert isinstance(results, type(data)) def test_dict_items(self): - items = {"a": 1, "b": 2}.items() - results = pickle_depickle(items) - self.assertEqual(results, items) - assert isinstance(results, _collections_abc.dict_items) + data = {"a": 1, "b": 2}.items() + results = pickle_depickle(data) + self.assertEqual(results, data) + assert isinstance(results, type(data)) def test_odict_keys(self): - keys = collections.OrderedDict([("a", 1), ("b", 2)]).keys() - results = pickle_depickle(keys) - self.assertEqual(results, keys) - assert type(keys) == type(results) + data = collections.OrderedDict([("a", 1), ("b", 2)]).keys() + results = pickle_depickle(data) + self.assertEqual(results, data) + assert isinstance(results, type(data)) def test_odict_values(self): - values = collections.OrderedDict([("a", 1), ("b", 2)]).values() - results = pickle_depickle(values) - self.assertEqual(list(results), list(values)) - assert type(values) == type(results) + data = collections.OrderedDict([("a", 1), ("b", 2)]).values() + results = pickle_depickle(data) + self.assertEqual(list(results), list(data)) + assert isinstance(results, type(data)) def test_odict_items(self): - items = collections.OrderedDict([("a", 1), ("b", 2)]).items() - results = pickle_depickle(items) - self.assertEqual(results, items) - assert type(items) == type(results) + data = collections.OrderedDict([("a", 1), ("b", 2)]).items() + results = pickle_depickle(data) + self.assertEqual(results, data) + assert isinstance(results, type(data)) def test_view_types(self): - for view_type in VIEW_TYPES: - t, view = view_type - obj = t([("a", 1), ("b", 2)]) - at = getattr(obj, view, None) + from cloudpickle.cloudpickle_fast import _VIEWS_TYPES_TABLE + + for view_type, info in _VIEWS_TYPES_TABLE.items(): + obj = info.object_type([("a", 1), ("b", 2)]) + at = getattr(obj, info.view, None) if at is not None: if callable(at): data = at() else: data = at results = pickle_depickle(data) - self.assertEqual(results, data) - assert type(data) == type(results) + assert isinstance(results, view_type) + self.assertEqual(list(results), list(data)) def test_sliced_and_non_contiguous_memoryview(self): buffer_obj = memoryview(b"Hello!" * 3)[2:15:2] From 5579a2fb4f65fe52ff84683d8c3f241527076e19 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Sun, 8 Aug 2021 13:37:30 -0500 Subject: [PATCH 04/13] Revert some unnecessary changes to imports --- cloudpickle/cloudpickle.py | 16 ++++++++-------- cloudpickle/cloudpickle_fast.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index aedcc085..fd1a9827 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -44,24 +44,24 @@ import builtins import dis +import opcode import platform import sys -import threading import types -import typing +import weakref import uuid +import threading +import typing import warnings -import weakref -from pickle import _getattribute -from typing import Any, Callable, Generic, Tuple, Union - -import opcode from .compat import pickle +from typing import Any, Generic, Union, Tuple, Callable +from pickle import _getattribute +from importlib._bootstrap import _find_spec try: # pragma: no branch import typing_extensions as _typing_extensions - from typing_extensions import Final, Literal + from typing_extensions import Literal, Final except ImportError: _typing_extensions = Literal = Final = None diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 7eabf177..020a125a 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -21,8 +21,8 @@ import weakref import typing -from collections import ChainMap, OrderedDict, namedtuple from enum import Enum +from collections import ChainMap, OrderedDict, namedtuple from typing import Iterable from .compat import pickle, Pickler From a496f3a58ee89bff4eca48180ac183c06a3cf8f4 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Sun, 8 Aug 2021 13:49:37 -0500 Subject: [PATCH 05/13] Try to fix lint --- cloudpickle/cloudpickle.py | 2 +- cloudpickle/cloudpickle_fast.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index fd1a9827..7b8dba22 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -55,7 +55,7 @@ import warnings from .compat import pickle -from typing import Any, Generic, Union, Tuple, Callable +from typing import Generic, Union, Tuple, Callable from pickle import _getattribute from importlib._bootstrap import _find_spec diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 020a125a..bc8ba36d 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -420,7 +420,7 @@ def _class_reduce(obj): # DICT VIEWS TYPES -ViewInfo = namedtuple("ViewInfo", ["view", "packer", "maker", "object_type"]) +ViewInfo = namedtuple("ViewInfo", ["view", "packer", "maker", "object_type"]) #noqa _VIEW_ATTRS_INFO = [ From 2196ae1fc3c1651bde52335c9ea805fea6ffc6f0 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Sun, 8 Aug 2021 13:50:47 -0500 Subject: [PATCH 06/13] space --- cloudpickle/cloudpickle_fast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index bc8ba36d..17bb083b 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -420,7 +420,7 @@ def _class_reduce(obj): # DICT VIEWS TYPES -ViewInfo = namedtuple("ViewInfo", ["view", "packer", "maker", "object_type"]) #noqa +ViewInfo = namedtuple("ViewInfo", ["view", "packer", "maker", "object_type"]) # noqa _VIEW_ATTRS_INFO = [ From 94f03f70553759fe0b722bcb0a6ada4f16a7a713 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Mon, 9 Aug 2021 10:56:47 -0500 Subject: [PATCH 07/13] Only cast when needed --- cloudpickle/cloudpickle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 7b8dba22..6812ad61 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -957,10 +957,10 @@ def _make_keys_view(obj, typ): o = t.fromkeys(obj) else: o = dict.fromkeys(obj) - try: - o = t(o) - except Exception: - pass + try: + o = t(o) + except Exception: + pass return o.keys() From 159074c0409447e713e73a37640b3520f93e4667 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Fri, 27 Aug 2021 12:23:53 -0500 Subject: [PATCH 08/13] Register UserDict views --- cloudpickle/cloudpickle_fast.py | 4 ++-- pytest.ini | 4 ++++ tests/cloudpickle_test.py | 10 ++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 pytest.ini diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 17bb083b..2acdad20 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -22,7 +22,7 @@ import typing from enum import Enum -from collections import ChainMap, OrderedDict, namedtuple +from collections import ChainMap, OrderedDict, UserDict, namedtuple from typing import Iterable from .compat import pickle, Pickler @@ -467,7 +467,7 @@ def register_views_types(types, attr_info=None, overwrite=False): return _VIEWS_TYPES_TABLE -_VIEWABLE_TYPES = [dict, OrderedDict] +_VIEWABLE_TYPES = [dict, OrderedDict, UserDict] register_views_types(_VIEWABLE_TYPES) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..75da646a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +addopts = --color=yes +markers = + dict_views diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 712c434b..e56a21f7 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -52,6 +52,7 @@ from cloudpickle.cloudpickle import _make_empty_cell, cell_set from cloudpickle.cloudpickle import _extract_class_dict, _whichmodule from cloudpickle.cloudpickle import _lookup_module_and_qualname +from cloudpickle.cloudpickle_fast import _VIEWS_TYPES_TABLE, register_views_types from .testutils import subprocess_pickle_echo from .testutils import subprocess_pickle_string @@ -226,45 +227,50 @@ def test_memoryview(self): self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), buffer_obj.tobytes()) + @pytest.mark.dict_views def test_dict_keys(self): data = {"a": 1, "b": 2}.keys() results = pickle_depickle(data) self.assertEqual(results, data) assert isinstance(results, type(data)) + @pytest.mark.dict_views def test_dict_values(self): data = {"a": 1, "b": 2}.values() results = pickle_depickle(data) self.assertEqual(sorted(results), sorted(data)) assert isinstance(results, type(data)) + @pytest.mark.dict_views def test_dict_items(self): data = {"a": 1, "b": 2}.items() results = pickle_depickle(data) self.assertEqual(results, data) assert isinstance(results, type(data)) + @pytest.mark.dict_views def test_odict_keys(self): data = collections.OrderedDict([("a", 1), ("b", 2)]).keys() results = pickle_depickle(data) self.assertEqual(results, data) assert isinstance(results, type(data)) + @pytest.mark.dict_views def test_odict_values(self): data = collections.OrderedDict([("a", 1), ("b", 2)]).values() results = pickle_depickle(data) self.assertEqual(list(results), list(data)) assert isinstance(results, type(data)) + @pytest.mark.dict_views def test_odict_items(self): data = collections.OrderedDict([("a", 1), ("b", 2)]).items() results = pickle_depickle(data) self.assertEqual(results, data) assert isinstance(results, type(data)) + @pytest.mark.dict_views def test_view_types(self): - from cloudpickle.cloudpickle_fast import _VIEWS_TYPES_TABLE - for view_type, info in _VIEWS_TYPES_TABLE.items(): obj = info.object_type([("a", 1), ("b", 2)]) at = getattr(obj, info.view, None) From 3834956d374c6cc016cc181aedc47c742e82b715 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Fri, 27 Aug 2021 12:28:43 -0500 Subject: [PATCH 09/13] Remove unused import --- tests/cloudpickle_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index e56a21f7..cd5b2961 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -52,7 +52,7 @@ from cloudpickle.cloudpickle import _make_empty_cell, cell_set from cloudpickle.cloudpickle import _extract_class_dict, _whichmodule from cloudpickle.cloudpickle import _lookup_module_and_qualname -from cloudpickle.cloudpickle_fast import _VIEWS_TYPES_TABLE, register_views_types +from cloudpickle.cloudpickle_fast import _VIEWS_TYPES_TABLE from .testutils import subprocess_pickle_echo from .testutils import subprocess_pickle_string From e440abaf5a36d4795683cf86af6aa05f6ea5c29f Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Fri, 27 Aug 2021 12:42:26 -0500 Subject: [PATCH 10/13] Use importorskip for failing module --- tests/cloudpickle_test.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index cd5b2961..f6320181 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -59,12 +59,6 @@ from .testutils import assert_run_python_script from .testutils import subprocess_worker -try: - # import seems to break tests - from _cloudpickle_testpkg import relative_imports_factory -except ImportError: - pass - _TEST_GLOBAL_VARIABLE = "default_value" _TEST_GLOBAL_VARIABLE2 = "another_value" @@ -2090,7 +2084,8 @@ def test_relative_import_inside_function(self): # Make sure relative imports inside round-tripped functions is not # broken. This was a bug in cloudpickle versions <= 0.5.3 and was # re-introduced in 0.8.0. - f, g = relative_imports_factory() + pytest.importorskip("_cloudpickle_testpkg") + f, g = _cloudpickle_testpkg.relative_imports_factory() for func, source in zip([f, g], ["module", "package"]): # Make sure relative imports are initially working assert func() == "hello from a {}!".format(source) From bab5e616c785881d319eda8c745e7406a9c5c2d9 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Fri, 27 Aug 2021 12:46:42 -0500 Subject: [PATCH 11/13] Assign imported module --- tests/cloudpickle_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index f6320181..fcf3c46f 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2084,7 +2084,7 @@ def test_relative_import_inside_function(self): # Make sure relative imports inside round-tripped functions is not # broken. This was a bug in cloudpickle versions <= 0.5.3 and was # re-introduced in 0.8.0. - pytest.importorskip("_cloudpickle_testpkg") + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") f, g = _cloudpickle_testpkg.relative_imports_factory() for func, source in zip([f, g], ["module", "package"]): # Make sure relative imports are initially working From f1373004b25b5ea07e36fa2825e1e8aa53d60142 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Fri, 27 Aug 2021 13:06:23 -0500 Subject: [PATCH 12/13] Remove pytest.ini, markers, importorskip tests --- pytest.ini | 4 ---- tests/cloudpickle_test.py | 16 +++++++--------- 2 files changed, 7 insertions(+), 13 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 75da646a..00000000 --- a/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -addopts = --color=yes -markers = - dict_views diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index fcf3c46f..9614eb57 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -221,49 +221,42 @@ def test_memoryview(self): self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), buffer_obj.tobytes()) - @pytest.mark.dict_views def test_dict_keys(self): data = {"a": 1, "b": 2}.keys() results = pickle_depickle(data) self.assertEqual(results, data) assert isinstance(results, type(data)) - @pytest.mark.dict_views def test_dict_values(self): data = {"a": 1, "b": 2}.values() results = pickle_depickle(data) self.assertEqual(sorted(results), sorted(data)) assert isinstance(results, type(data)) - @pytest.mark.dict_views def test_dict_items(self): data = {"a": 1, "b": 2}.items() results = pickle_depickle(data) self.assertEqual(results, data) assert isinstance(results, type(data)) - @pytest.mark.dict_views def test_odict_keys(self): data = collections.OrderedDict([("a", 1), ("b", 2)]).keys() results = pickle_depickle(data) self.assertEqual(results, data) assert isinstance(results, type(data)) - @pytest.mark.dict_views def test_odict_values(self): data = collections.OrderedDict([("a", 1), ("b", 2)]).values() results = pickle_depickle(data) self.assertEqual(list(results), list(data)) assert isinstance(results, type(data)) - @pytest.mark.dict_views def test_odict_items(self): data = collections.OrderedDict([("a", 1), ("b", 2)]).items() results = pickle_depickle(data) self.assertEqual(results, data) assert isinstance(results, type(data)) - @pytest.mark.dict_views def test_view_types(self): for view_type, info in _VIEWS_TYPES_TABLE.items(): obj = info.object_type([("a", 1), ("b", 2)]) @@ -778,6 +771,7 @@ def test_module_importability(self): # their parent modules are considered importable by cloudpickle. # See the mod_with_dynamic_submodule documentation for more # details of this use case. + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") import _cloudpickle_testpkg.mod.dynamic_submodule as m assert _should_pickle_by_reference(m) assert pickle_depickle(m, protocol=self.protocol) is m @@ -2134,6 +2128,7 @@ def f(a, /, b=1): def test___reduce___returns_string(self): # Non regression test for objects with a __reduce__ method returning a # string, meaning "save by attribute using save_global" + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") from _cloudpickle_testpkg import some_singleton assert some_singleton.__reduce__() == "some_singleton" depickled_singleton = pickle_depickle( @@ -2206,6 +2201,7 @@ def test_pickle_dynamic_typevar_memoization(self): assert depickled_T1 is depickled_T2 def test_pickle_importable_typevar(self): + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") from _cloudpickle_testpkg import T T1 = pickle_depickle(T, protocol=self.protocol) assert T1 is T @@ -2531,6 +2527,7 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): def test_pickle_constructs_from_installed_packages_registered_for_pickling_by_value( # noqa self ): + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") for package_or_module in ["package", "module"]: if package_or_module == "package": import _cloudpickle_testpkg as m @@ -2565,7 +2562,7 @@ def test_pickle_various_versions_of_the_same_function_with_different_pickling_me # pickled in a different way - by value and/or by reference) can # peacefully co-exist (e.g. without globals interaction) in a remote # worker. - import _cloudpickle_testpkg + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") from _cloudpickle_testpkg import package_function_with_global as f _original_global = _cloudpickle_testpkg.global_variable @@ -2642,7 +2639,7 @@ def test_lookup_module_and_qualname_dynamic_typevar(): def test_lookup_module_and_qualname_importable_typevar(): - import _cloudpickle_testpkg + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") T = _cloudpickle_testpkg.T module_and_name = _lookup_module_and_qualname(T, name=T.__name__) assert module_and_name is not None @@ -2661,6 +2658,7 @@ def test_lookup_module_and_qualname_stdlib_typevar(): def test_register_pickle_by_value(): + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") import _cloudpickle_testpkg as pkg import _cloudpickle_testpkg.mod as mod From 513a93868e02476f98fd37eafe76af896eb80dd7 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Fri, 27 Aug 2021 13:14:01 -0500 Subject: [PATCH 13/13] Fix flake8 --- tests/cloudpickle_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 9614eb57..c371d10e 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -771,7 +771,7 @@ def test_module_importability(self): # their parent modules are considered importable by cloudpickle. # See the mod_with_dynamic_submodule documentation for more # details of this use case. - _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") # noqa F841 import _cloudpickle_testpkg.mod.dynamic_submodule as m assert _should_pickle_by_reference(m) assert pickle_depickle(m, protocol=self.protocol) is m @@ -2078,7 +2078,7 @@ def test_relative_import_inside_function(self): # Make sure relative imports inside round-tripped functions is not # broken. This was a bug in cloudpickle versions <= 0.5.3 and was # re-introduced in 0.8.0. - _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") # noqa F841 f, g = _cloudpickle_testpkg.relative_imports_factory() for func, source in zip([f, g], ["module", "package"]): # Make sure relative imports are initially working @@ -2128,7 +2128,7 @@ def f(a, /, b=1): def test___reduce___returns_string(self): # Non regression test for objects with a __reduce__ method returning a # string, meaning "save by attribute using save_global" - _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") # noqa F841 from _cloudpickle_testpkg import some_singleton assert some_singleton.__reduce__() == "some_singleton" depickled_singleton = pickle_depickle( @@ -2201,7 +2201,7 @@ def test_pickle_dynamic_typevar_memoization(self): assert depickled_T1 is depickled_T2 def test_pickle_importable_typevar(self): - _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") # noqa F841 from _cloudpickle_testpkg import T T1 = pickle_depickle(T, protocol=self.protocol) assert T1 is T @@ -2527,7 +2527,7 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): def test_pickle_constructs_from_installed_packages_registered_for_pickling_by_value( # noqa self ): - _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") # noqa F841 for package_or_module in ["package", "module"]: if package_or_module == "package": import _cloudpickle_testpkg as m @@ -2562,7 +2562,7 @@ def test_pickle_various_versions_of_the_same_function_with_different_pickling_me # pickled in a different way - by value and/or by reference) can # peacefully co-exist (e.g. without globals interaction) in a remote # worker. - _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") # noqa F841 from _cloudpickle_testpkg import package_function_with_global as f _original_global = _cloudpickle_testpkg.global_variable @@ -2639,7 +2639,7 @@ def test_lookup_module_and_qualname_dynamic_typevar(): def test_lookup_module_and_qualname_importable_typevar(): - _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") # noqa F841 T = _cloudpickle_testpkg.T module_and_name = _lookup_module_and_qualname(T, name=T.__name__) assert module_and_name is not None @@ -2658,7 +2658,7 @@ def test_lookup_module_and_qualname_stdlib_typevar(): def test_register_pickle_by_value(): - _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") # noqa F841 import _cloudpickle_testpkg as pkg import _cloudpickle_testpkg.mod as mod