Allow prealloc and registration before calling C++ ctor

docs/advanced/classes.rst

@@ -1333,3 +1333,100 @@ You can do that using ``py::custom_type_setup``:
    cls.def_readwrite("value", &OwnsPythonObjects::value);
 
 .. versionadded:: 2.8
+
+Accessing Python object from C++
+================================
+
+In advanced cases, it's really handy to access the Python sibling of
+a C++ object to get/set attributes, do some heavy computations, just hide
+the implementation details, or even dynamically create a new attribute!
+
+One just need to rely on the `casting capabilities`_ of ``pybind11``:
+
+.. code-block:: cpp
+
+    // main.cpp
+    #include "pybind11/pybind11.h"
+
+    namespace py = pybind11;
+
+    class Base {
+    public:
+        Base() = default;
+        std::string get_foo() { return py::cast(this).attr("foo").cast<std::string>(); };
+        void set_bar() { py::cast(this).attr("bar") = 10.; };
+    };
+
+    PYBIND11_MODULE(test, m) {
+        py::class_<Base>(m, "Base")
+            .def(py::init<>())
+            .def("get_foo", &Base::get_foo)
+            .def("set_bar", &Base::set_bar);
+    }
+
+
+.. code-block:: python
+
+    # test.py
+    from test import Base
+
+
+    class Derived(Base):
+        def __init__(self):
+            Base.__init__(self)
+            self.foo = "hello"
+
+
+    b = Derived()
+    assert b.get_foo() == "hello"
+    assert not hasattr(b, "bar")
+
+    b.set_bar()
+    assert b.bar == 10.0
+
+
+However, there is a special case where such a behavior needs a hint to work as expected:
+when the C++ constructor is called, the C++ object is not yet allocated and registered,
+making impossible the casting operation.
+
+It's thus impossible to access the Python object from the C++ constructor one *as is*.
+
+Adding the ``py::preallocate()`` extra option to a constructor binding definition informs
+``pybind11`` to allocate the memory and register the object location just before calling the C++
+constructor, enabling the use of ``py::cast(this)``:
+
+
+.. code-block:: cpp
+
+    // main.cpp
+    #include "pybind11/pybind11.h"
+
+    namespace py = pybind11;
+
+    class Base {
+    public:
+        Base() { py::cast(this).attr("bar") = 10.; };
+    };
+
+    PYBIND11_MODULE(test, m) {
+        py::class_<Base>(m, "Base")
+            .def(py::init<>(), py::preallocate());
+    }
+
+
+.. code-block:: python
+
+    # test.py
+    from test import Base
+
+
+    class Derived(Base):
+        ...
+
+
+    b = Derived()
+    assert hasattr(b, "bar")
+    assert b.bar == 10.0
+
+
+.. _casting capabilities: https://pybind11.readthedocs.io/en/stable/advanced/pycpp/object.html?highlight=cast#casting-back-and-forth

include/pybind11/attr.h

@@ -72,6 +72,9 @@ struct keep_alive {};
 /// Annotation indicating that a class is involved in a multiple inheritance relationship
 struct multiple_inheritance {};
 
+/// Annotation indicating that a class should be preallocated and registered before construction
+struct preallocate {};
+
 /// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class
 struct dynamic_attr {};
 
@@ -438,6 +441,11 @@ struct process_attribute<is_operator> : process_attribute_default<is_operator> {
     static void init(const is_operator &, function_record *r) { r->is_operator = true; }
 };
 
+template <>
+struct process_attribute<preallocate> : process_attribute_default<preallocate> {
+    static void init(const preallocate &, function_record *) {}
+};
+
 template <>
 struct process_attribute<is_new_style_constructor>
     : process_attribute_default<is_new_style_constructor> {

include/pybind11/detail/common.h

@@ -885,6 +885,18 @@ using is_lambda = satisfies_none_of<remove_reference_t<T>,
                                     std::is_pointer,
                                     std::is_member_pointer>;
 
+/// Check if T is part of a template parameter pack
+template <typename T, typename... List>
+struct contains : std::true_type {};
+
+template <typename T, typename Head, typename... Rest>
+struct contains<T, Head, Rest...>
+    : std::conditional<std::is_same<T, Head>::value, std::true_type, contains<T, Rest...>>::type {
+};
+
+template <typename T>
+struct contains<T> : std::false_type {};
+
 // [workaround(intel)] Internal error on fold expression
 /// Apply a function over each element of a parameter pack
 #if defined(__cpp_fold_expressions) && !defined(__INTEL_COMPILER)

include/pybind11/detail/init.h

@@ -61,24 +61,53 @@ constexpr bool is_alias(void *) {
     return false;
 }
 
-// Constructs and returns a new object; if the given arguments don't map to a constructor, we fall
+// Constructs a new object inplace; if the given arguments don't map to a constructor, we fall
 // back to brace aggregate initiailization so that for aggregate initialization can be used with
 // py::init, e.g.  `py::init<int, int>` to initialize a `struct T { int a; int b; }`.  For
 // non-aggregate types, we need to use an ordinary T(...) constructor (invoking as `T{...}` usually
 // works, but will not do the expected thing when `T` has an `initializer_list<T>` constructor).
-template <typename Class,
-          typename... Args,
-          detail::enable_if_t<std::is_constructible<Class, Args...>::value, int> = 0>
-inline Class *construct_or_initialize(Args &&...args) {
-    return new Class(std::forward<Args>(args)...);
+template <
+    typename Class,
+    bool Preallocate,
+    typename... Args,
+    detail::enable_if_t<std::is_constructible<Class, Args...>::value && !Preallocate, int> = 0>
+inline void construct_or_initialize(value_and_holder &v_h, Args &&...args) {
+    v_h.value_ptr() = new Class(std::forward<Args>(args)...);
 }
-template <typename Class,
-          typename... Args,
-          detail::enable_if_t<!std::is_constructible<Class, Args...>::value, int> = 0>
-inline Class *construct_or_initialize(Args &&...args) {
-    return new Class{std::forward<Args>(args)...};
+template <
+    typename Class,
+    bool Preallocate,
+    typename... Args,
+    detail::enable_if_t<!std::is_constructible<Class, Args...>::value && !Preallocate, int> = 0>
+inline void construct_or_initialize(value_and_holder &v_h, Args &&...args) {
+    v_h.value_ptr() = new Class{std::forward<Args>(args)...};
+}
+// The preallocated variants are performing memory allocation and registration before actually
+// calling the constructor to allow casting the C++ pointer to its Python counterpart.
+template <
+    typename Class,
+    bool Preallocate,
+    typename... Args,
+    detail::enable_if_t<std::is_constructible<Class, Args...>::value && Preallocate, int> = 0>
+inline void construct_or_initialize(value_and_holder &v_h, Args &&...args) {
+    v_h.value_ptr() = malloc(sizeof(Class));
+    register_instance(v_h.inst, v_h.value_ptr(), v_h.type);
+    v_h.set_instance_registered();
+
+    new (v_h.value_ptr<Class>()) Class(std::forward<Args>(args)...);
+}
+template <
+    typename Class,
+    bool Preallocate,
+    typename... Args,
+    detail::enable_if_t<!std::is_constructible<Class, Args...>::value && Preallocate, int> = 0>
+inline void construct_or_initialize(value_and_holder &v_h, Args &&...args) {
+    v_h.value_ptr() = malloc(sizeof(Class));
+    register_instance(v_h.inst, v_h.value_ptr(), v_h.type);
+    v_h.set_instance_registered();
+
+    new (v_h.value_ptr<Class>()) Class{std::forward<Args>(args)...};
 }
-
 // Attempts to constructs an alias using a `Alias(Cpp &&)` constructor.  This allows types with
 // an alias to provide only a single Cpp factory function as long as the Alias can be
 // constructed from an rvalue reference of the base Cpp type.  This means that Alias classes
@@ -200,7 +229,8 @@ struct constructor {
         cl.def(
             "__init__",
             [](value_and_holder &v_h, Args... args) {
-                v_h.value_ptr() = construct_or_initialize<Cpp<Class>>(std::forward<Args>(args)...);
+                construct_or_initialize<Cpp<Class>, contains<preallocate, Extra...>::value>(
+                    v_h, std::forward<Args>(args)...);
             },
             is_new_style_constructor(),
             extra...);
@@ -215,11 +245,11 @@ struct constructor {
             "__init__",
             [](value_and_holder &v_h, Args... args) {
                 if (Py_TYPE(v_h.inst) == v_h.type->type) {
-                    v_h.value_ptr()
-                        = construct_or_initialize<Cpp<Class>>(std::forward<Args>(args)...);
+                    construct_or_initialize<Cpp<Class>, contains<preallocate, Extra...>::value>(
+                        v_h, std::forward<Args>(args)...);
                 } else {
-                    v_h.value_ptr()
-                        = construct_or_initialize<Alias<Class>>(std::forward<Args>(args)...);
+                    construct_or_initialize<Alias<Class>, contains<preallocate, Extra...>::value>(
+                        v_h, std::forward<Args>(args)...);
                 }
             },
             is_new_style_constructor(),
@@ -234,8 +264,8 @@ struct constructor {
         cl.def(
             "__init__",
             [](value_and_holder &v_h, Args... args) {
-                v_h.value_ptr()
-                    = construct_or_initialize<Alias<Class>>(std::forward<Args>(args)...);
+                construct_or_initialize<Alias<Class>, contains<preallocate, Extra...>::value>(
+                    v_h, std::forward<Args>(args)...);
             },
             is_new_style_constructor(),
             extra...);
@@ -253,8 +283,8 @@ struct alias_constructor {
         cl.def(
             "__init__",
             [](value_and_holder &v_h, Args... args) {
-                v_h.value_ptr()
-                    = construct_or_initialize<Alias<Class>>(std::forward<Args>(args)...);
+                construct_or_initialize<Alias<Class>, contains<preallocate, Extra...>::value>(
+                    v_h, std::forward<Args>(args)...);
             },
             is_new_style_constructor(),
             extra...);

tests/test_methods_and_attributes.cpp

@@ -177,6 +177,20 @@ struct RValueRefParam {
     std::size_t func4(std::string &&s) const & { return s.size(); }
 };
 
+// Test Python init from C++ constructor
+struct InitPyFromCpp1 {
+    InitPyFromCpp1() { py::cast(this).attr("bar") = 10.; };
+};
+struct InitPyFromCpp2 {
+    InitPyFromCpp2() { py::cast(this).attr("bar") = 10.; };
+};
+struct InitPyFromCppDynamic1 {
+    InitPyFromCppDynamic1() { py::cast(this).attr("bar") = 10.; };
+};
+struct InitPyFromCppDynamic2 {
+    InitPyFromCppDynamic2() { py::cast(this).attr("bar") = 10.; };
+};
+
 TEST_SUBMODULE(methods_and_attributes, m) {
     // test_methods_and_attributes
     py::class_<ExampleMandA> emna(m, "ExampleMandA");
@@ -456,4 +470,12 @@ TEST_SUBMODULE(methods_and_attributes, m) {
         .def("func2", &RValueRefParam::func2)
         .def("func3", &RValueRefParam::func3)
         .def("func4", &RValueRefParam::func4);
+
+    // Test Python init from C++ constructor
+    py::class_<InitPyFromCpp1>(m, "InitPyFromCpp1").def(py::init<>());
+    py::class_<InitPyFromCpp2>(m, "InitPyFromCpp2").def(py::init<>(), py::preallocate());
+    py::class_<InitPyFromCppDynamic1>(m, "InitPyFromCppDynamic1", py::dynamic_attr())
+        .def(py::init<>());
+    py::class_<InitPyFromCppDynamic2>(m, "InitPyFromCppDynamic2", py::dynamic_attr())
+        .def(py::init<>(), py::preallocate());
 }

tests/test_methods_and_attributes.py

@@ -525,3 +525,51 @@ def test_rvalue_ref_param():
     assert r.func2("1234") == 4
     assert r.func3("12345") == 5
     assert r.func4("123456") == 6
+
+
+@pytest.mark.xfail("env.PYPY")
+def test_init_py_from_cpp():
+    # test dynamically added attr from C++ to Python counterpart
+
+    # 1. on class not supporting dynamic attributes
+    with pytest.raises(AttributeError):
+        m.InitPyFromCpp1()
+
+    with pytest.raises(AttributeError):
+        m.InitPyFromCpp2()
+
+    # 2. on derived class of base not supporting dynamic attributes
+    class Derived1(m.InitPyFromCpp1):
+        ...
+
+    with pytest.raises(AttributeError):
+        Derived1()
+
+    class Derived2(m.InitPyFromCpp2):
+        ...
+
+    assert Derived2().bar == 10.0
+
+    # 3. on class supporting dynamic attributes
+    # constructor will set the `bar` attribute to a temporary Python object
+    a = m.InitPyFromCppDynamic1()
+    with pytest.raises(AttributeError):
+        a.bar
+
+    # works fine
+    assert m.InitPyFromCppDynamic2().bar == 10.0
+
+    # 4. on derived class of base supporting dynamic attributes
+    class DynamicDerived1(m.InitPyFromCppDynamic1):
+        ...
+
+    # still the same issue
+    d = DynamicDerived1()
+    with pytest.raises(AttributeError):
+        d.bar
+
+    # works fine
+    class DynamicDerived2(m.InitPyFromCppDynamic2):
+        ...
+
+    assert DynamicDerived2().bar == 10.0