From 8a2a39f253e4ba3c2bb3d2172b71570cc4edb0c5 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 14 Dec 2022 09:28:45 -0800 Subject: [PATCH 1/6] gh-83076: ~4x speed improvement in (Async)Mock instantiation --- Lib/unittest/mock.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index a273753d6a0abb..03e93d15101453 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -411,15 +411,18 @@ class NonCallableMock(Base): # necessary. _lock = RLock() - def __new__(cls, /, *args, **kw): + def __new__( + cls, spec=None, wraps=None, name=None, spec_set=None, + parent=None, _spec_state=None, _new_name='', _new_parent=None, + _spec_as_instance=False, _eat_self=None, unsafe=False, **kwargs + ): # every instance has its own class # so we can create magic methods on the # class without stomping on other mocks bases = (cls,) if not issubclass(cls, AsyncMockMixin): # Check if spec is an async object or function - bound_args = _MOCK_SIG.bind_partial(cls, *args, **kw).arguments - spec_arg = bound_args.get('spec_set', bound_args.get('spec')) + spec_arg = spec_set or spec if spec_arg is not None and _is_async_obj(spec_arg): bases = (AsyncMockMixin, cls) new = type(cls.__name__, bases, {'__doc__': cls.__doc__}) @@ -503,11 +506,6 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_class = None _spec_signature = None - _spec_asyncs = [] - - for attr in dir(spec): - if iscoroutinefunction(getattr(spec, attr, None)): - _spec_asyncs.append(attr) if spec is not None and not _is_list(spec): if isinstance(spec, type): @@ -525,7 +523,6 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, __dict__['_spec_set'] = spec_set __dict__['_spec_signature'] = _spec_signature __dict__['_mock_methods'] = spec - __dict__['_spec_asyncs'] = _spec_asyncs def __get_return_value(self): ret = self._mock_return_value @@ -1015,7 +1012,8 @@ def _get_child_mock(self, /, **kw): For non-callable mocks the callable variant will be used (rather than any custom subclass).""" _new_name = kw.get("_new_name") - if _new_name in self.__dict__['_spec_asyncs']: + _spec_val = getattr(self.__dict__["_spec_class"], _new_name, None) + if _spec_val is not None and asyncio.iscoroutinefunction(_spec_val): return AsyncMock(**kw) if self._mock_sealed: @@ -1057,9 +1055,6 @@ def _calls_repr(self, prefix="Calls"): return f"\n{prefix}: {safe_repr(self.mock_calls)}." -_MOCK_SIG = inspect.signature(NonCallableMock.__init__) - - class _AnyComparer(list): """A list which checks if it contains a call which may have an argument of ANY, flipping the components of item and self from @@ -2183,6 +2178,10 @@ def __get__(self, obj, _type=None): return self.create_mock() +_CODE_ATTRS = dir(CodeType) +_CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) + + class AsyncMockMixin(Base): await_count = _delegating_property('await_count') await_args = _delegating_property('await_args') @@ -2200,7 +2199,9 @@ def __init__(self, /, *args, **kwargs): self.__dict__['_mock_await_count'] = 0 self.__dict__['_mock_await_args'] = None self.__dict__['_mock_await_args_list'] = _CallList() - code_mock = NonCallableMock(spec_set=CodeType) + code_mock = NonCallableMock(spec_set=_CODE_ATTRS) + code_mock.__dict__["_spec_class"] = CodeType + code_mock.__dict__["_spec_signature"] = _CODE_SIG code_mock.co_flags = inspect.CO_COROUTINE self.__dict__['__code__'] = code_mock self.__dict__['__name__'] = 'AsyncMock' From 7a30565c2d70f45f7032a27ff7eb8b4641798d3a Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 17:37:04 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst diff --git a/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst b/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst new file mode 100644 index 00000000000000..6835a6931675bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst @@ -0,0 +1 @@ +Instantiation of `Mock()` and `AsyncMock()` is now 3.8x faster. From 597209dd488d846acee5432bec6a8d1f4654566f Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 14 Dec 2022 09:40:33 -0800 Subject: [PATCH 3/6] Remove redundant AsyncMagicMixin.__init__ method --- Lib/unittest/mock.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 03e93d15101453..50e9e5b287849a 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -2133,10 +2133,8 @@ def mock_add_spec(self, spec, spec_set=False): class AsyncMagicMixin(MagicMixin): - def __init__(self, /, *args, **kw): - self._mock_set_magics() # make magic work for kwargs in init - _safe_super(AsyncMagicMixin, self).__init__(*args, **kw) - self._mock_set_magics() # fix magic broken by upper level init + pass + class MagicMock(MagicMixin, Mock): """ From 246772c2c4c6039c754eb1b452b17a1b3fc88e45 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 14 Dec 2022 09:43:33 -0800 Subject: [PATCH 4/6] Fix rst syntax in news blurb --- .../next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst b/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst index 6835a6931675bf..a4984e695b43fc 100644 --- a/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst +++ b/Misc/NEWS.d/next/Library/2022-12-14-17-37-01.gh-issue-83076.NaYzWT.rst @@ -1 +1 @@ -Instantiation of `Mock()` and `AsyncMock()` is now 3.8x faster. +Instantiation of ``Mock()`` and ``AsyncMock()`` is now 3.8x faster. From a03f84360e4830ffa63883ba52f6b5958c6bd16a Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 16 Dec 2022 09:14:27 -0800 Subject: [PATCH 5/6] fix instance-only async attr --- Lib/test/test_unittest/testmock/testasync.py | 7 +++++++ Lib/unittest/mock.py | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index e05a22861d47bf..990b247e5ee975 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -300,6 +300,13 @@ def test_spec_normal_methods_on_class_with_mock(self): self.assertIsInstance(mock.async_method, AsyncMock) self.assertIsInstance(mock.normal_method, Mock) + def test_spec_async_attributes_instance(self): + async_instance = AsyncClass() + async_instance.async_func_attr = async_func + + mock_async_instance = Mock(async_instance) + self.assertIsInstance(mock_async_instance.async_func_attr, AsyncMock) + def test_spec_mock_type_kw(self): def inner_test(mock_type): async_mock = mock_type(spec=async_func) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 50e9e5b287849a..e37d31dda201c0 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -506,8 +506,10 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_class = None _spec_signature = None + _spec_obj = None if spec is not None and not _is_list(spec): + _spec_obj = spec if isinstance(spec, type): _spec_class = spec else: @@ -520,6 +522,7 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, __dict__ = self.__dict__ __dict__['_spec_class'] = _spec_class + __dict__['_spec_obj'] = _spec_obj __dict__['_spec_set'] = spec_set __dict__['_spec_signature'] = _spec_signature __dict__['_mock_methods'] = spec @@ -1012,7 +1015,7 @@ def _get_child_mock(self, /, **kw): For non-callable mocks the callable variant will be used (rather than any custom subclass).""" _new_name = kw.get("_new_name") - _spec_val = getattr(self.__dict__["_spec_class"], _new_name, None) + _spec_val = getattr(self.__dict__["_spec_obj"], _new_name, None) if _spec_val is not None and asyncio.iscoroutinefunction(_spec_val): return AsyncMock(**kw) From 6595272b2e5c64310c1eb4e7d1edba0689555c55 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 16 Dec 2022 13:31:59 -0800 Subject: [PATCH 6/6] determine spec shape only at mock construction time --- Lib/test/test_unittest/testmock/testasync.py | 8 +++++++- Lib/unittest/mock.py | 16 ++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 990b247e5ee975..52a3b71be1ef8d 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -303,9 +303,15 @@ def test_spec_normal_methods_on_class_with_mock(self): def test_spec_async_attributes_instance(self): async_instance = AsyncClass() async_instance.async_func_attr = async_func + async_instance.later_async_func_attr = normal_func + + mock_async_instance = Mock(spec_set=async_instance) + + async_instance.later_async_func_attr = async_func - mock_async_instance = Mock(async_instance) self.assertIsInstance(mock_async_instance.async_func_attr, AsyncMock) + # only the shape of the spec at the time of mock construction matters + self.assertNotIsInstance(mock_async_instance.later_async_func_attr, AsyncMock) def test_spec_mock_type_kw(self): def inner_test(mock_type): diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index e37d31dda201c0..583ab74a825531 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -506,10 +506,9 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_class = None _spec_signature = None - _spec_obj = None + _spec_asyncs = [] if spec is not None and not _is_list(spec): - _spec_obj = spec if isinstance(spec, type): _spec_class = spec else: @@ -518,14 +517,20 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_as_instance, _eat_self) _spec_signature = res and res[1] - spec = dir(spec) + spec_list = dir(spec) + + for attr in spec_list: + if iscoroutinefunction(getattr(spec, attr, None)): + _spec_asyncs.append(attr) + + spec = spec_list __dict__ = self.__dict__ __dict__['_spec_class'] = _spec_class - __dict__['_spec_obj'] = _spec_obj __dict__['_spec_set'] = spec_set __dict__['_spec_signature'] = _spec_signature __dict__['_mock_methods'] = spec + __dict__['_spec_asyncs'] = _spec_asyncs def __get_return_value(self): ret = self._mock_return_value @@ -1015,8 +1020,7 @@ def _get_child_mock(self, /, **kw): For non-callable mocks the callable variant will be used (rather than any custom subclass).""" _new_name = kw.get("_new_name") - _spec_val = getattr(self.__dict__["_spec_obj"], _new_name, None) - if _spec_val is not None and asyncio.iscoroutinefunction(_spec_val): + if _new_name in self.__dict__['_spec_asyncs']: return AsyncMock(**kw) if self._mock_sealed: