From 2cf41322ddaddd011335b7a527164d5735d9bf5b Mon Sep 17 00:00:00 2001 From: Nate McMaster Date: Sat, 1 May 2021 21:07:05 -0700 Subject: [PATCH] Implement support for attrib converters --- docs/structuring.rst | 46 +++++++++++++++++++-- src/cattr/converters.py | 46 +++++++++++++++------ src/cattr/dispatch.py | 6 ++- src/cattr/errors.py | 2 + src/cattr/gen.py | 26 +++++++++++- tests/test_function_dispatch.py | 4 +- tests/test_structure.py | 5 ++- tests/test_structure_attrs.py | 73 ++++++++++++++++++++++++++++++++- 8 files changed, 183 insertions(+), 25 deletions(-) create mode 100644 src/cattr/errors.py diff --git a/docs/structuring.rst b/docs/structuring.rst index 2cdd9507..3cea0ff8 100644 --- a/docs/structuring.rst +++ b/docs/structuring.rst @@ -285,7 +285,7 @@ and their own converters work out of the box. Given a mapping ``d`` and class >>> @attr.s ... class A: ... a = attr.ib() - ... b = attr.ib(converter=int) + ... b = attr.ib() ... >>> cattr.structure({'a': 1, 'b': '2'}, A) A(a=1, b=2) @@ -299,7 +299,7 @@ Classes like these deconstructed into tuples can be structured using >>> @attr.s ... class A: ... a = attr.ib() - ... b = attr.ib(converter=int) + ... b = attr.ib(type=int) ... >>> cattr.structure_attrs_fromtuple(['string', '2'], A) A(a='string', b=2) @@ -313,7 +313,7 @@ Loading from tuples can be made the default by creating a new ``Converter`` with >>> @attr.s ... class A: ... a = attr.ib() - ... b = attr.ib(converter=int) + ... b = attr.ib(type=int) ... >>> converter.structure(['string', '2'], A) A(a='string', b=2) @@ -321,6 +321,44 @@ Loading from tuples can be made the default by creating a new ``Converter`` with Structuring from tuples can also be made the default for specific classes only; see registering custom structure hooks below. + +Using attribute types and converters +------------------------------------ + +By default, calling "structure" will use hooks registered using ``cattr.register_structure_hook``, +to convert values to the attribute type, and fallback to invoking any converters registered on +attributes with ``attrib``. + +.. doctest:: + + >>> from ipaddress import IPv4Address, ip_address + >>> converter = cattr.Converter() + + # Note: register_structure_hook has not been called, so this will fallback to 'ip_address' + >>> @attr.s + ... class A: + ... a = attr.ib(type=IPv4Address, converter=ip_address) + + >>> converter.structure({'a': '127.0.0.1'}, A) + A(a=IPv4Address('127.0.0.1')) + +Priority is still given to hooks registered with ``cattr.register_structure_hook``, but this priority +can be inverted by setting ``prefer_attrib_converters`` to ``True``. + +.. doctest:: + + >>> converter = cattr.Converter(prefer_attrib_converters=True) + + >>> converter.register_structure_hook(int, lambda v, t: int(v)) + + >>> @attr.s + ... class A: + ... a = attr.ib(type=int, converter=lambda v: int(v) + 5) + + >>> converter.structure({'a': '10'}, A) + A(a=15) + + Complex ``attrs`` classes and dataclasses ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -376,7 +414,7 @@ Here's an example involving a simple, classic (i.e. non-``attrs``) Python class. >>> cattr.structure({'a': 1}, C) Traceback (most recent call last): ... - ValueError: Unsupported type: . Register a structure hook for it. + StructureHandlerNotFoundError: Unsupported type: . Register a structure hook for it. >>> >>> cattr.register_structure_hook(C, lambda d, t: C(**d)) >>> cattr.structure({'a': 1}, C) diff --git a/src/cattr/converters.py b/src/cattr/converters.py index cdf8767d..a93f5a76 100644 --- a/src/cattr/converters.py +++ b/src/cattr/converters.py @@ -1,9 +1,11 @@ from collections import Counter from collections.abc import MutableSet as AbcMutableSet +from dataclasses import Field from enum import Enum from functools import lru_cache -from typing import Any, Callable, Dict, Optional, Tuple, Type, TypeVar +from typing import Any, Callable, Dict, Optional, Tuple, Type, TypeVar, Union +from attr import Attribute from attr import has as attrs_has from attr import resolve_types @@ -33,6 +35,7 @@ ) from .disambiguators import create_uniq_field_dis_func from .dispatch import MultiStrategyDispatch +from .errors import StructureHandlerNotFoundError from .gen import ( AttributeOverride, make_dict_structure_fn, @@ -71,14 +74,17 @@ class Converter(object): "_dict_factory", "_union_struct_registry", "_structure_func", + "_prefer_attrib_converters", ) def __init__( self, dict_factory: Callable[[], Any] = dict, unstruct_strat: UnstructureStrategy = UnstructureStrategy.AS_DICT, + prefer_attrib_converters: bool = False, ) -> None: unstruct_strat = UnstructureStrategy(unstruct_strat) + self._prefer_attrib_converters = prefer_attrib_converters # Create a per-instance cache. if unstruct_strat is UnstructureStrategy.AS_DICT: @@ -299,7 +305,7 @@ def _structure_default(self, obj, cl): "Unsupported type: {0}. Register a structure hook for " "it.".format(cl) ) - raise ValueError(msg) + raise StructureHandlerNotFoundError(msg) @staticmethod def _structure_call(obj, cl): @@ -320,18 +326,34 @@ def structure_attrs_fromtuple( conv_obj = [] # A list of converter parameters. for a, value in zip(fields(cl), obj): # type: ignore # We detect the type by the metadata. - converted = self._structure_attr_from_tuple(a, a.name, value) + converted = self._structure_attribute(a, value) conv_obj.append(converted) return cl(*conv_obj) # type: ignore - def _structure_attr_from_tuple(self, a, _, value): + def _structure_attribute( + self, a: Union[Attribute, Field], value: Any + ) -> Any: """Handle an individual attrs attribute.""" type_ = a.type + attrib_converter = getattr(a, "converter", None) + if self._prefer_attrib_converters and attrib_converter: + # A attrib converter is defined on this attribute, and prefer_attrib_converters is set + # to give these priority over registered structure hooks. So, pass through the raw + # value, which attrs will flow into the converter + return value if type_ is None: # No type metadata. return value - return self._structure_func.dispatch(type_)(value, type_) + + try: + return self._structure_func.dispatch(type_)(value, type_) + except StructureHandlerNotFoundError: + if attrib_converter: + # Return the original value and fallback to using an attrib converter. + return value + else: + raise def structure_attrs_fromdict( self, obj: Mapping[str, Any], cl: Type[T] @@ -340,10 +362,7 @@ def structure_attrs_fromdict( # For public use. conv_obj = {} # Start with a fresh dict, to ignore extra keys. - dispatch = self._structure_func.dispatch for a in fields(cl): # type: ignore - # We detect the type by metadata. - type_ = a.type name = a.name try: @@ -354,9 +373,7 @@ def structure_attrs_fromdict( if name[0] == "_": name = name[1:] - conv_obj[name] = ( - dispatch(type_)(val, type_) if type_ is not None else val - ) + conv_obj[name] = self._structure_attribute(a, val) return cl(**conv_obj) # type: ignore @@ -476,7 +493,7 @@ def _get_dis_func(union): ) if not all(has(get_origin(e) or e) for e in union_types): - raise ValueError( + raise StructureHandlerNotFoundError( "Only unions of attr classes supported " "currently. Register a loads hook manually." ) @@ -501,9 +518,12 @@ def __init__( forbid_extra_keys: bool = False, type_overrides: Mapping[Type, AttributeOverride] = {}, unstruct_collection_overrides: Mapping[Type, Callable] = {}, + prefer_attrib_converters: bool = False, ): super().__init__( - dict_factory=dict_factory, unstruct_strat=unstruct_strat + dict_factory=dict_factory, + unstruct_strat=unstruct_strat, + prefer_attrib_converters=prefer_attrib_converters, ) self.omit_if_default = omit_if_default self.forbid_extra_keys = forbid_extra_keys diff --git a/src/cattr/dispatch.py b/src/cattr/dispatch.py index 1a3b1023..5316e516 100644 --- a/src/cattr/dispatch.py +++ b/src/cattr/dispatch.py @@ -3,6 +3,8 @@ import attr +from .errors import StructureHandlerNotFoundError + @attr.s class _DispatchNotFound: @@ -121,4 +123,6 @@ def dispatch(self, typ): return handler(typ) else: return handler - raise KeyError("unable to find handler for {0}".format(typ)) + raise StructureHandlerNotFoundError( + "unable to find handler for {0}".format(typ) + ) diff --git a/src/cattr/errors.py b/src/cattr/errors.py new file mode 100644 index 00000000..644af73f --- /dev/null +++ b/src/cattr/errors.py @@ -0,0 +1,2 @@ +class StructureHandlerNotFoundError(Exception): + pass diff --git a/src/cattr/gen.py b/src/cattr/gen.py index cfbb73bb..f830b46e 100644 --- a/src/cattr/gen.py +++ b/src/cattr/gen.py @@ -1,3 +1,4 @@ +import functools import re from dataclasses import is_dataclass from typing import Any, Optional, Type, TypeVar @@ -6,6 +7,7 @@ from attr import NOTHING, resolve_types from ._compat import adapted_fields, get_args, get_origin, is_generic +from .errors import StructureHandlerNotFoundError @attr.s(slots=True, frozen=True) @@ -162,11 +164,18 @@ def make_dict_structure_fn( # For each attribute, we try resolving the type here and now. # If a type is manually overwritten, this function should be # regenerated. - if type is not None: + if converter._prefer_attrib_converters and a.converter is not None: + # The attribute has defined its own conversion, so pass + # the original value through without invoking cattr hooks + handler = _passthru + elif type is not None: handler = converter._structure_func.dispatch(type) else: handler = converter.structure + if not converter._prefer_attrib_converters and a.converter is not None: + handler = _fallback_to_passthru(handler) + struct_handler_name = f"__cattr_struct_handler_{an}" globs[struct_handler_name] = handler @@ -201,6 +210,21 @@ def make_dict_structure_fn( return globs[fn_name] +def _passthru(obj, _): + return obj + + +def _fallback_to_passthru(func): + @functools.wraps(func) + def invoke(obj, type_): + try: + return func(obj, type_) + except StructureHandlerNotFoundError: + return obj + + return invoke + + def make_iterable_unstructure_fn(cl: Any, converter, unstructure_to=None): """Generate a specialized unstructure function for an iterable.""" handler = converter.unstructure diff --git a/tests/test_function_dispatch.py b/tests/test_function_dispatch.py index 18d9360d..e3ee9b32 100644 --- a/tests/test_function_dispatch.py +++ b/tests/test_function_dispatch.py @@ -1,12 +1,12 @@ import pytest -from cattr.dispatch import FunctionDispatch +from cattr.dispatch import FunctionDispatch, StructureHandlerNotFoundError def test_function_dispatch(): dispatch = FunctionDispatch() - with pytest.raises(KeyError): + with pytest.raises(StructureHandlerNotFoundError): dispatch.dispatch(float) test_func = object() diff --git a/tests/test_structure.py b/tests/test_structure.py index e3714842..256c9dd6 100644 --- a/tests/test_structure.py +++ b/tests/test_structure.py @@ -33,6 +33,7 @@ from cattr import Converter from cattr._compat import is_bare, is_union_type from cattr.converters import NoneType +from cattr.errors import StructureHandlerNotFoundError from . import ( dicts_of_primitives, @@ -326,9 +327,9 @@ def test_structuring_enums(data, enum): def test_structuring_unsupported(): """Loading unsupported classes should throw.""" converter = Converter() - with raises(ValueError): + with raises(StructureHandlerNotFoundError): converter.structure(1, Converter) - with raises(ValueError): + with raises(StructureHandlerNotFoundError): converter.structure(1, Union[int, str]) diff --git a/tests/test_structure_attrs.py b/tests/test_structure_attrs.py index d3e14aaf..6ef521f3 100644 --- a/tests/test_structure_attrs.py +++ b/tests/test_structure_attrs.py @@ -1,11 +1,14 @@ """Loading of attrs classes.""" +from ipaddress import IPv4Address, IPv6Address, ip_address from typing import Union +from unittest.mock import Mock -from attr import NOTHING, Factory, asdict, astuple, fields +import pytest +from attr import NOTHING, Factory, asdict, astuple, attrib, fields, make_class from hypothesis import assume, given from hypothesis.strategies import data, lists, sampled_from -from cattr.converters import Converter +from cattr.converters import Converter, GenConverter from . import simple_classes @@ -134,3 +137,69 @@ def dis(obj, _): assert inst == converter.structure( converter.unstructure(inst), Union[cl_a, cl_b] ) + + +@pytest.mark.parametrize("converter_type", [Converter, GenConverter]) +def test_structure_fallback_to_attrib_converters(converter_type): + attrib_converter = Mock() + attrib_converter.side_effect = lambda val: str(val) + + def called_after_default_converter(val): + if not isinstance(val, int): + raise ValueError( + "The 'int' conversion should have happened first by the built-in hooks" + ) + return 42 + + converter = converter_type() + cl = make_class( + "HasConverter", + { + # non-built-in type with custom converter + "ip": attrib( + type=Union[IPv4Address, IPv6Address], converter=ip_address + ), + # attribute without type + "x": attrib(converter=attrib_converter), + # built-in types converters + "z": attrib(type=int, converter=called_after_default_converter), + }, + ) + + inst = converter.structure(dict(ip="10.0.0.0", x=1, z="3"), cl) + + assert inst.ip == IPv4Address("10.0.0.0") + assert inst.x == "1" + attrib_converter.assert_any_call(1) + assert inst.z == 42 + + +@pytest.mark.parametrize("converter_type", [Converter, GenConverter]) +def test_structure_prefers_attrib_converters(converter_type): + attrib_converter = Mock() + attrib_converter.side_effect = lambda val: str(val) + + converter = converter_type(prefer_attrib_converters=True) + cl = make_class( + "HasConverter", + { + # non-built-in type with custom converter + "ip": attrib( + type=Union[IPv4Address, IPv6Address], converter=ip_address + ), + # attribute without type + "x": attrib(converter=attrib_converter), + # built-in types converters + "z": attrib(type=int, converter=attrib_converter), + }, + ) + + inst = converter.structure(dict(ip="10.0.0.0", x=1, z=3), cl) + + assert inst.ip == IPv4Address("10.0.0.0") + + attrib_converter.assert_any_call(1) + assert inst.x == "1" + + attrib_converter.assert_any_call(3) + assert inst.z == "3"