From f16a7afcb101af499a49e2d5a087012f2467d25a Mon Sep 17 00:00:00 2001 From: Adrien DELSALLE Date: Wed, 3 Aug 2022 15:30:15 +0200 Subject: [PATCH 1/2] allow cast of C++ class in its ctor allocate memory before calling the ctor register the instance before calling the ctor use a placement new call to set the value_ptr use `prealloacte` ennotation to select placement new refactor construct_or_initialize to construct inplace add a type trait to check if a template param pack contains a type add tests add documentation --- docs/advanced/classes.rst | 97 +++++++++++++++++++++++++++ include/pybind11/attr.h | 8 +++ include/pybind11/detail/common.h | 12 ++++ include/pybind11/detail/init.h | 72 ++++++++++++++------ tests/test_methods_and_attributes.cpp | 22 ++++++ tests/test_methods_and_attributes.py | 48 +++++++++++++ 6 files changed, 238 insertions(+), 21 deletions(-) diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 01a490b721..6f06dbc2df 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -1333,3 +1333,100 @@ You can do that using ``py::custom_type_setup``: cls.def_readwrite("value", &OwnsPythonObjects::value); .. versionadded:: 2.8 + +Accessing Python object from C++ +================================ + +In advanced cases, it's really handy to access the Python sibling of +a C++ object to get/set attributes, do some heavy computations, just hide +the implementation details, or even dynamically create a new attribute! + +One just need to rely on the `casting capabilities`_ of ``pybind11``: + +.. code-block:: cpp + + // main.cpp + #include "pybind11/pybind11.h" + + namespace py = pybind11; + + class Base { + public: + Base() = default; + std::string get_foo() { return py::cast(this).attr("foo").cast(); }; + void set_bar() { py::cast(this).attr("bar") = 10.; }; + }; + + PYBIND11_MODULE(test, m) { + py::class_(m, "Base") + .def(py::init<>()) + .def("get_foo", &Base::get_foo) + .def("set_bar", &Base::set_bar); + } + + +.. code-block:: python + + # test.py + from test import Base + + + class Derived(Base): + def __init__(self): + Base.__init__(self) + self.foo = "hello" + + + b = Derived() + assert b.get_foo() == "hello" + assert not hasattr(b, "bar") + + b.set_bar() + assert b.bar == 10.0 + + +However, there is a special case where such a behavior needs a hint to work as expected: +when the C++ constructor is called, the C++ object is not yet allocated and registered, +making impossible the casting operation. + +It's thus impossible to access the Python object from the C++ constructor one *as is*. + +Adding the ``py::preallocate()`` extra option to a constructor binding definition informs +``pybind11`` to allocate the memory and register the object location just before calling the C++ +constructor, enabling the use of ``py::cast(this)``: + + +.. code-block:: cpp + + // main.cpp + #include "pybind11/pybind11.h" + + namespace py = pybind11; + + class Base { + public: + Base() { py::cast(this).attr("bar") = 10.; }; + }; + + PYBIND11_MODULE(test, m) { + py::class_(m, "Base") + .def(py::init<>(), py::preallocate()); + } + + +.. code-block:: python + + # test.py + from test import Base + + + class Derived(Base): + ... + + + b = Derived() + assert hasattr(b, "bar") + assert b.bar == 10.0 + + +.. _casting capabilities: https://pybind11.readthedocs.io/en/stable/advanced/pycpp/object.html?highlight=cast#casting-back-and-forth diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index db7cd8efff..2d2f62e69e 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -72,6 +72,9 @@ struct keep_alive {}; /// Annotation indicating that a class is involved in a multiple inheritance relationship struct multiple_inheritance {}; +/// Annotation indicating that a class should be preallocated and registered before construction +struct preallocate {}; + /// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class struct dynamic_attr {}; @@ -438,6 +441,11 @@ struct process_attribute : process_attribute_default { static void init(const is_operator &, function_record *r) { r->is_operator = true; } }; +template <> +struct process_attribute : process_attribute_default { + static void init(const preallocate &, function_record *) {} +}; + template <> struct process_attribute : process_attribute_default { diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 9e6947daa3..5b9ed2cfe6 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -885,6 +885,18 @@ using is_lambda = satisfies_none_of, std::is_pointer, std::is_member_pointer>; +/// Check if T is part of a template parameter pack +template +struct contains : std::true_type {}; + +template +struct contains + : std::conditional::value, std::true_type, contains>::type { +}; + +template +struct contains : std::false_type {}; + // [workaround(intel)] Internal error on fold expression /// Apply a function over each element of a parameter pack #if defined(__cpp_fold_expressions) && !defined(__INTEL_COMPILER) diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 05f4fe54aa..5b6b01020b 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -61,24 +61,53 @@ constexpr bool is_alias(void *) { return false; } -// Constructs and returns a new object; if the given arguments don't map to a constructor, we fall +// Constructs a new object inplace; if the given arguments don't map to a constructor, we fall // back to brace aggregate initiailization so that for aggregate initialization can be used with // py::init, e.g. `py::init` to initialize a `struct T { int a; int b; }`. For // non-aggregate types, we need to use an ordinary T(...) constructor (invoking as `T{...}` usually // works, but will not do the expected thing when `T` has an `initializer_list` constructor). -template ::value, int> = 0> -inline Class *construct_or_initialize(Args &&...args) { - return new Class(std::forward(args)...); +template < + typename Class, + bool Preallocate, + typename... Args, + detail::enable_if_t::value && !Preallocate, int> = 0> +inline void construct_or_initialize(value_and_holder &v_h, Args &&...args) { + v_h.value_ptr() = new Class(std::forward(args)...); } -template ::value, int> = 0> -inline Class *construct_or_initialize(Args &&...args) { - return new Class{std::forward(args)...}; +template < + typename Class, + bool Preallocate, + typename... Args, + detail::enable_if_t::value && !Preallocate, int> = 0> +inline void construct_or_initialize(value_and_holder &v_h, Args &&...args) { + v_h.value_ptr() = new Class{std::forward(args)...}; +} +// The preallocated variants are performing memory allocation and registration before actually +// calling the constructor to allow casting the C++ pointer to its Python counterpart. +template < + typename Class, + bool Preallocate, + typename... Args, + detail::enable_if_t::value && Preallocate, int> = 0> +inline void construct_or_initialize(value_and_holder &v_h, Args &&...args) { + v_h.value_ptr() = malloc(sizeof(Class)); + register_instance(v_h.inst, v_h.value_ptr(), v_h.type); + v_h.set_instance_registered(); + + new (v_h.value_ptr()) Class(std::forward(args)...); +} +template < + typename Class, + bool Preallocate, + typename... Args, + detail::enable_if_t::value && Preallocate, int> = 0> +inline void construct_or_initialize(value_and_holder &v_h, Args &&...args) { + v_h.value_ptr() = malloc(sizeof(Class)); + register_instance(v_h.inst, v_h.value_ptr(), v_h.type); + v_h.set_instance_registered(); + + new (v_h.value_ptr()) Class{std::forward(args)...}; } - // Attempts to constructs an alias using a `Alias(Cpp &&)` constructor. This allows types with // an alias to provide only a single Cpp factory function as long as the Alias can be // constructed from an rvalue reference of the base Cpp type. This means that Alias classes @@ -200,7 +229,8 @@ struct constructor { cl.def( "__init__", [](value_and_holder &v_h, Args... args) { - v_h.value_ptr() = construct_or_initialize>(std::forward(args)...); + construct_or_initialize, contains::value>( + v_h, std::forward(args)...); }, is_new_style_constructor(), extra...); @@ -215,11 +245,11 @@ struct constructor { "__init__", [](value_and_holder &v_h, Args... args) { if (Py_TYPE(v_h.inst) == v_h.type->type) { - v_h.value_ptr() - = construct_or_initialize>(std::forward(args)...); + construct_or_initialize, contains::value>( + v_h, std::forward(args)...); } else { - v_h.value_ptr() - = construct_or_initialize>(std::forward(args)...); + construct_or_initialize, contains::value>( + v_h, std::forward(args)...); } }, is_new_style_constructor(), @@ -234,8 +264,8 @@ struct constructor { cl.def( "__init__", [](value_and_holder &v_h, Args... args) { - v_h.value_ptr() - = construct_or_initialize>(std::forward(args)...); + construct_or_initialize, contains::value>( + v_h, std::forward(args)...); }, is_new_style_constructor(), extra...); @@ -253,8 +283,8 @@ struct alias_constructor { cl.def( "__init__", [](value_and_holder &v_h, Args... args) { - v_h.value_ptr() - = construct_or_initialize>(std::forward(args)...); + construct_or_initialize, contains::value>( + v_h, std::forward(args)...); }, is_new_style_constructor(), extra...); diff --git a/tests/test_methods_and_attributes.cpp b/tests/test_methods_and_attributes.cpp index 815dd5e98a..dc6ecf694a 100644 --- a/tests/test_methods_and_attributes.cpp +++ b/tests/test_methods_and_attributes.cpp @@ -177,6 +177,20 @@ struct RValueRefParam { std::size_t func4(std::string &&s) const & { return s.size(); } }; +// Test Python init from C++ constructor +struct InitPyFromCpp1 { + InitPyFromCpp1() { py::cast(this).attr("bar") = 10.; }; +}; +struct InitPyFromCpp2 { + InitPyFromCpp2() { py::cast(this).attr("bar") = 10.; }; +}; +struct InitPyFromCppDynamic1 { + InitPyFromCppDynamic1() { py::cast(this).attr("bar") = 10.; }; +}; +struct InitPyFromCppDynamic2 { + InitPyFromCppDynamic2() { py::cast(this).attr("bar") = 10.; }; +}; + TEST_SUBMODULE(methods_and_attributes, m) { // test_methods_and_attributes py::class_ emna(m, "ExampleMandA"); @@ -456,4 +470,12 @@ TEST_SUBMODULE(methods_and_attributes, m) { .def("func2", &RValueRefParam::func2) .def("func3", &RValueRefParam::func3) .def("func4", &RValueRefParam::func4); + + // Test Python init from C++ constructor + py::class_(m, "InitPyFromCpp1").def(py::init<>()); + py::class_(m, "InitPyFromCpp2").def(py::init<>(), py::preallocate()); + py::class_(m, "InitPyFromCppDynamic1", py::dynamic_attr()) + .def(py::init<>()); + py::class_(m, "InitPyFromCppDynamic2", py::dynamic_attr()) + .def(py::init<>(), py::preallocate()); } diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 0a2ae1239a..a98ecba69b 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -525,3 +525,51 @@ def test_rvalue_ref_param(): assert r.func2("1234") == 4 assert r.func3("12345") == 5 assert r.func4("123456") == 6 + + +@pytest.mark.xfail("env.PYPY") +def test_init_py_from_cpp(): + # test dynamically added attr from C++ to Python counterpart + + # 1. on class not supporting dynamic attributes + with pytest.raises(AttributeError): + m.InitPyFromCpp1() + + with pytest.raises(AttributeError): + m.InitPyFromCpp2() + + # 2. on derived class of base not supporting dynamic attributes + class Derived1(m.InitPyFromCpp1): + ... + + with pytest.raises(AttributeError): + Derived1() + + class Derived2(m.InitPyFromCpp2): + ... + + assert Derived2().bar == 10.0 + + # 3. on class supporting dynamic attributes + # constructor will set the `bar` attribute to a temporary Python object + a = m.InitPyFromCppDynamic1() + with pytest.raises(AttributeError): + a.bar + + # works fine + assert m.InitPyFromCppDynamic2().bar == 10.0 + + # 4. on derived class of base supporting dynamic attributes + class DynamicDerived1(m.InitPyFromCppDynamic1): + ... + + # still the same issue + d = DynamicDerived1() + with pytest.raises(AttributeError): + d.bar + + # works fine + class DynamicDerived2(m.InitPyFromCppDynamic2): + ... + + assert DynamicDerived2().bar == 10.0 From 69998754b556a72865c28ce5d8161050ed2d511f Mon Sep 17 00:00:00 2001 From: Adrien DELSALLE Date: Tue, 9 Aug 2022 08:45:37 +0200 Subject: [PATCH 2/2] allocate using new operator instead of malloc --- include/pybind11/detail/init.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 5b6b01020b..f79ae45b7c 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -90,7 +90,7 @@ template < typename... Args, detail::enable_if_t::value && Preallocate, int> = 0> inline void construct_or_initialize(value_and_holder &v_h, Args &&...args) { - v_h.value_ptr() = malloc(sizeof(Class)); + v_h.value_ptr() = ::operator new(sizeof(Class)); register_instance(v_h.inst, v_h.value_ptr(), v_h.type); v_h.set_instance_registered(); @@ -102,7 +102,7 @@ template < typename... Args, detail::enable_if_t::value && Preallocate, int> = 0> inline void construct_or_initialize(value_and_holder &v_h, Args &&...args) { - v_h.value_ptr() = malloc(sizeof(Class)); + v_h.value_ptr() = ::operator new(sizeof(Class)); register_instance(v_h.inst, v_h.value_ptr(), v_h.type); v_h.set_instance_registered();