diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4acbbf5e4..4bd01cb7fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: - 'pypy-3.8' - 'pypy-3.9' - 'pypy-3.10' + - 'graalpy-24.1' # Items in here will either be added to the build matrix (if not # present), or add new keys to an existing matrix element if all the @@ -67,6 +68,10 @@ jobs: # Extra ubuntu latest job - runs-on: ubuntu-latest python: '3.11' + exclude: + # The setup-python action currently doesn't have graalpy for windows + - python: 'graalpy-24.1' + runs-on: 'windows-2022' name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • x64 ${{ matrix.args }}" diff --git a/.github/workflows/emscripten.yaml b/.github/workflows/emscripten.yaml index 14b2b9dc7d..79c783e7fe 100644 --- a/.github/workflows/emscripten.yaml +++ b/.github/workflows/emscripten.yaml @@ -22,7 +22,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: pypa/cibuildwheel@v2.20 + - uses: pypa/cibuildwheel@v2.21 env: PYODIDE_BUILD_EXPORTS: whole_archive with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8190df441..88f82eea8f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,14 +25,14 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v18.1.8" + rev: "v19.1.3" hooks: - id: clang-format types_or: [c++, c, cuda] # Ruff, the Python auto-correcting linter/formatter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.3 + rev: v0.7.2 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -40,7 +40,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.11.2" + rev: "v1.13.0" hooks: - id: mypy args: [] @@ -62,7 +62,7 @@ repos: # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: "v4.6.0" + rev: "v5.0.0" hooks: - id: check-added-large-files - id: check-case-conflict @@ -79,7 +79,7 @@ repos: # Also code format the docs - repo: https://github.com/adamchainz/blacken-docs - rev: "1.18.0" + rev: "1.19.1" hooks: - id: blacken-docs additional_dependencies: @@ -108,7 +108,7 @@ repos: # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest - rev: "0.49" + rev: "0.50" hooks: - id: check-manifest # This is a slow hook, so only run this if --hook-stage manual is passed @@ -142,14 +142,14 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v3.2.7" + rev: "v3.3.1" hooks: - id: pylint files: ^pybind11 # Check schemas on some of our YAML files - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.2 + rev: 0.29.4 hooks: - id: check-readthedocs - id: check-github-workflows diff --git a/docs/advanced/deadlock.md b/docs/advanced/deadlock.md new file mode 100644 index 0000000000..f1bab5bdb0 --- /dev/null +++ b/docs/advanced/deadlock.md @@ -0,0 +1,391 @@ +# Double locking, deadlocking, GIL + +[TOC] + +## Introduction + +### Overview + +In concurrent programming with locks, *deadlocks* can arise when more than one +mutex is locked at the same time, and careful attention has to be paid to lock +ordering to avoid this. Here we will look at a common situation that occurs in +native extensions for CPython written in C++. + +### Deadlocks + +A deadlock can occur when more than one thread attempts to lock more than one +mutex, and two of the threads lock two of the mutexes in different orders. For +example, consider mutexes `mu1` and `mu2`, and threads T1 and T2, executing: + +| | T1 | T2 | +|--- | ------------------- | -------------------| +|1 | `mu1.lock()`{.good} | `mu2.lock()`{.good}| +|2 | `mu2.lock()`{.bad} | `mu1.lock()`{.bad} | +|3 | `/* work */` | `/* work */` | +|4 | `mu2.unlock()` | `mu1.unlock()` | +|5 | `mu1.unlock()` | `mu2.unlock()` | + +Now if T1 manages to lock `mu1` and T2 manages to lock `mu2` (as indicated in +green), then both threads will block while trying to lock the respective other +mutex (as indicated in red), but they are also unable to release the mutex that +they have locked (step 5). + +**The problem** is that it is possible for one thread to attempt to lock `mu1` +and then `mu2`, and for another thread to attempt to lock `mu2` and then `mu1`. +Note that it does not matter if either mutex is unlocked at any intermediate +point; what matters is only the order of any attempt to *lock* the mutexes. For +example, the following, more complex series of operations is just as prone to +deadlock: + +| | T1 | T2 | +|--- | ------------------- | -------------------| +|1 | `mu1.lock()`{.good} | `mu1.lock()`{.good}| +|2 | waiting for T2 | `mu2.lock()`{.good}| +|3 | waiting for T2 | `/* work */` | +|3 | waiting for T2 | `mu1.unlock()` | +|3 | `mu2.lock()`{.bad} | `/* work */` | +|3 | `/* work */` | `mu1.lock()`{.bad} | +|3 | `/* work */` | `/* work */` | +|4 | `mu2.unlock()` | `mu1.unlock()` | +|5 | `mu1.unlock()` | `mu2.unlock()` | + +When the mutexes involved in a locking sequence are known at compile-time, then +avoiding deadlocks is “merely” a matter of arranging the lock +operations carefully so as to only occur in one single, fixed order. However, it +is also possible for mutexes to only be determined at runtime. A typical example +of this is a database where each row has its own mutex. An operation that +modifies two rows in a single transaction (e.g. “transferring an amount +from one account to another”) must lock two row mutexes, but the locking +order cannot be established at compile time. In this case, a dynamic +“deadlock avoidance algorithm” is needed. (In C++, `std::lock` +provides such an algorithm. An algorithm might use a non-blocking `try_lock` +operation on a mutex, which can either succeed or fail to lock the mutex, but +returns without blocking.) + +Conceptually, one could also consider it a deadlock if _the same_ thread +attempts to lock a mutex that it has already locked (e.g. when some locked +operation accidentally recurses into itself): `mu.lock();`{.good} +`mu.lock();`{.bad} However, this is a slightly separate issue: Typical mutexes +are either of _recursive_ or _non-recursive_ kind. A recursive mutex allows +repeated locking and requires balanced unlocking. A non-recursive mutex can be +implemented more efficiently, and/but for efficiency reasons does not actually +guarantee a deadlock on second lock. Instead, the API simply forbids such use, +making it a precondition that the thread not already hold the mutex, with +undefined behaviour on violation. + +### “Once” initialization + +A common programming problem is to have an operation happen precisely once, even +if requested concurrently. While it is clear that we need to track in some +shared state somewhere whether the operation has already happened, it is worth +noting that this state only ever transitions, once, from `false` to `true`. This +is considerably simpler than a general shared state that can change values +arbitrarily. Next, we also need a mechanism for all but one thread to block +until the initialization has completed, which we can provide with a mutex. The +simplest solution just always locks the mutex: + +```c++ +// The "once" mechanism: +constinit absl::Mutex mu(absl::kConstInit); +constinit bool init_done = false; + +// The operation of interest: +void f(); + +void InitOnceNaive() { + absl::MutexLock lock(&mu); + if (!init_done) { + f(); + init_done = true; + } +} +``` + +This works, but the efficiency-minded reader will observe that once the +operation has completed, all future lock contention on the mutex is +unnecessary. This leads to the (in)famous “double-locking” +algorithm, which was historically hard to write correctly. The idea is to check +the boolean *before* locking the mutex, and avoid locking if the operation has +already completed. However, accessing shared state concurrently when at least +one access is a write is prone to causing a data race and needs to be done +according to an appropriate concurrent programming model. In C++ we use atomic +variables: + +```c++ +// The "once" mechanism: +constinit absl::Mutex mu(absl::kConstInit); +constinit std::atomic init_done = false; + +// The operation of interest: +void f(); + +void InitOnceWithFastPath() { + if (!init_done.load(std::memory_order_acquire)) { + absl::MutexLock lock(&mu); + if (!init_done.load(std::memory_order_relaxed)) { + f(); + init_done.store(true, std::memory_order_release); + } + } +} +``` + +Checking the flag now happens without holding the mutex lock, and if the +operation has already completed, we return immediately. After locking the mutex, +we need to check the flag again, since multiple threads can reach this point. + +*Atomic details.* Since the atomic flag variable is accessed concurrently, we +have to think about the memory order of the accesses. There are two separate +cases: The first, outer check outside the mutex lock, and the second, inner +check under the lock. The outer check and the flag update form an +acquire/release pair: *if* the load sees the value `true` (which must have been +written by the store operation), then it also sees everything that happened +before the store, namely the operation `f()`. By contrast, the inner check can +use relaxed memory ordering, since in that case the mutex operations provide the +necessary ordering: if the inner load sees the value `true`, it happened after +the `lock()`, which happened after the `unlock()`, which happened after the +store. + +The C++ standard library, and Abseil, provide a ready-made solution of this +algorithm called `std::call_once`/`absl::call_once`. (The interface is the same, +but the Abseil implementation is possibly better.) + +```c++ +// The "once" mechanism: +constinit absl::once_flag init_flag; + +// The operation of interest: +void f(); + +void InitOnceWithCallOnce() { + absl::call_once(once_flag, f); +} +``` + +Even though conceptually this is performing the same algorithm, this +implementation has some considerable advantages: The `once_flag` type is a small +and trivial, integer-like type and is trivially destructible. Not only does it +take up less space than a mutex, it also generates less code since it does not +have to run a destructor, which would need to be added to the program's global +destructor list. + +The final clou comes with the C++ semantics of a `static` variable declared at +block scope: According to [[stmt.dcl]](https://eel.is/c++draft/stmt.dcl#3): + +> Dynamic initialization of a block variable with static storage duration or +> thread storage duration is performed the first time control passes through its +> declaration; such a variable is considered initialized upon the completion of +> its initialization. [...] If control enters the declaration concurrently while +> the variable is being initialized, the concurrent execution shall wait for +> completion of the initialization. + +This is saying that the initialization of a local, `static` variable precisely +has the “once” semantics that we have been discussing. We can +therefore write the above example as follows: + +```c++ +// The operation of interest: +void f(); + +void InitOnceWithStatic() { + static int unused = (f(), 0); +} +``` + +This approach is by far the simplest and easiest, but the big difference is that +the mutex (or mutex-like object) in this implementation is no longer visible or +in the user’s control. This is perfectly fine if the initializer is +simple, but if the initializer itself attempts to lock any other mutex +(including by initializing another static variable!), then we have no control +over the lock ordering! + +Finally, you may have noticed the `constinit`s around the earlier code. Both +`constinit` and `constexpr` specifiers on a declaration mean that the variable +is *constant-initialized*, which means that no initialization is performed at +runtime (the initial value is already known at compile time). This in turn means +that a static variable guard mutex may not be needed, and static initialization +never blocks. The difference between the two is that a `constexpr`-specified +variable is also `const`, and a variable cannot be `constexpr` if it has a +non-trivial destructor. Such a destructor also means that the guard mutex is +needed after all, since the destructor must be registered to run at exit, +conditionally on initialization having happened. + +## Python, CPython, GIL + +With CPython, a Python program can call into native code. To this end, the +native code registers callback functions with the Python runtime via the CPython +API. In order to ensure that the internal state of the Python runtime remains +consistent, there is a single, shared mutex called the “global interpreter +lock”, or GIL for short. Upon entry of one of the user-provided callback +functions, the GIL is locked (or “held”), so that no other mutations +of the Python runtime state can occur until the native callback returns. + +Many native extensions do not interact with the Python runtime for at least some +part of them, and so it is common for native extensions to _release_ the GIL, do +some work, and then reacquire the GIL before returning. Similarly, when code is +generally not holding the GIL but needs to interact with the runtime briefly, it +will first reacquire the GIL. The GIL is reentrant, and constructions to acquire +and subsequently release the GIL are common, and often don't worry about whether +the GIL is already held. + +If the native code is written in C++ and contains local, `static` variables, +then we are now dealing with at least _two_ mutexes: the static variable guard +mutex, and the GIL from CPython. + +A common problem in such code is an operation with “only once” +semantics that also ends up requiring the GIL to be held at some point. As per +the above description of “once”-style techniques, one might find a +static variable: + +```c++ +// CPython callback, assumes that the GIL is held on entry. +PyObject* InvokeWidget(PyObject* self) { + static PyObject* impl = CreateWidget(); + return PyObject_CallOneArg(impl, self); +} +``` + +This seems reasonable, but bear in mind that there are two mutexes (the "guard +mutex" and "the GIL"), and we must think about the lock order. Otherwise, if the +callback is called from multiple threads, a deadlock may ensue. + +Let us consider what we can see here: On entry, the GIL is already locked, and +we are locking the guard mutex. This is one lock order. Inside the initializer +`CreateWidget`, with both mutexes already locked, the function can freely access +the Python runtime. + +However, it is entirely possible that `CreateWidget` will want to release the +GIL at one point and reacquire it later: + +```c++ +// Assumes that the GIL is held on entry. +// Ensures that the GIL is held on exit. +PyObject* CreateWidget() { + // ... + Py_BEGIN_ALLOW_THREADS // releases GIL + // expensive work, not accessing the Python runtime + Py_END_ALLOW_THREADS // acquires GIL, #! + // ... + return result; +} +``` + +Now we have a second lock order: the guard mutex is locked, and then the GIL is +locked (at `#!`). To see how this deadlocks, consider threads T1 and T2 both +having the runtime attempt to call `InvokeWidget`. T1 locks the GIL and +proceeds, locking the guard mutex and calling `CreateWidget`; T2 is blocked +waiting for the GIL. Then T1 releases the GIL to do “expensive +work”, and T2 awakes and locks the GIL. Now T2 is blocked trying to +acquire the guard mutex, but T1 is blocked reacquiring the GIL (at `#!`). + +In other words: if we want to support “once-called” functions that +can arbitrarily release and reacquire the GIL, as is very common, then the only +lock order that we can ensure is: guard mutex first, GIL second. + +To implement this, we must rewrite our code. Naively, we could always release +the GIL before a `static` variable with blocking initializer: + +```c++ +// CPython callback, assumes that the GIL is held on entry. +PyObject* InvokeWidget(PyObject* self) { + Py_BEGIN_ALLOW_THREADS // releases GIL + static PyObject* impl = CreateWidget(); + Py_END_ALLOW_THREADS // acquires GIL + + return PyObject_CallOneArg(impl, self); +} +``` + +But similar to the `InitOnceNaive` example above, this code cycles the GIL +(possibly descheduling the thread) even when the static variable has already +been initialized. If we want to avoid this, we need to abandon the use of a +static variable, since we do not control the guard mutex well enough. Instead, +we use an operation whose mutex locking is under our control, such as +`call_once`. For example: + +```c++ +// CPython callback, assumes that the GIL is held on entry. +PyObject* InvokeWidget(PyObject* self) { + static constinit PyObject* impl = nullptr; + static constinit std::atomic init_done = false; + static constinit absl::once_flag init_flag; + + if (!init_done.load(std::memory_order_acquire)) { + Py_BEGIN_ALLOW_THREADS // releases GIL + absl::call_once(init_flag, [&]() { + PyGILState_STATE s = PyGILState_Ensure(); // acquires GIL + impl = CreateWidget(); + PyGILState_Release(s); // releases GIL + init_done.store(true, std::memory_order_release); + }); + Py_END_ALLOW_THREADS // acquires GIL + } + + return PyObject_CallOneArg(impl, self); +} +``` + +The lock order is now always guard mutex first, GIL second. Unfortunately we +have to duplicate the “double-checked done flag”, effectively +leading to triple checking, because the flag state inside the `absl::once_flag` +is not accessible to the user. In other words, we cannot ask `init_flag` whether +it has been used yet. + +However, we can perform one last, minor optimisation: since we assume that the +GIL is held on entry, and again when the initializing operation returns, the GIL +actually serializes access to our done flag variable, which therefore does not +need to be atomic. (The difference to the previous, atomic code may be small, +depending on the architecture. For example, on x86-64, acquire/release on a bool +is nearly free ([demo](https://godbolt.org/z/P9vYWf4fE)).) + +```c++ +// CPython callback, assumes that the GIL is held on entry, and indeed anywhere +// directly in this function (i.e. the GIL can be released inside CreateWidget, +// but must be reaqcuired when that call returns). +PyObject* InvokeWidget(PyObject* self) { + static constinit PyObject* impl = nullptr; + static constinit bool init_done = false; // guarded by GIL + static constinit absl::once_flag init_flag; + + if (!init_done) { + Py_BEGIN_ALLOW_THREADS // releases GIL + // (multiple threads may enter here) + absl::call_once(init_flag, [&]() { + // (only one thread enters here) + PyGILState_STATE s = PyGILState_Ensure(); // acquires GIL + impl = CreateWidget(); + init_done = true; // (GIL is held) + PyGILState_Release(s); // releases GIL + }); + + Py_END_ALLOW_THREADS // acquires GIL + } + + return PyObject_CallOneArg(impl, self); +} +``` + +## Debugging tips + +* Build with symbols. +* Ctrl-C sends `SIGINT`, Ctrl-\\ + sends `SIGQUIT`. Both have their uses. +* Useful `gdb` commands: + * `py-bt` prints a Python backtrace if you are in a Python frame. + * `thread apply all bt 10` prints the top-10 frames for each thread. A + full backtrace can be prohibitively expensive, and the top few frames + are often good enough. + * `p PyGILState_Check()` shows whether a thread is holding the GIL. For + all threads, run `thread apply all p PyGILState_Check()` to find out + which thread is holding the GIL. + * The `static` variable guard mutex is accessed with functions like + `cxa_guard_acquire` (though this depends on ABI details and can vary). + The guard mutex itself contains information about which thread is + currently holding it. + +## Links + +* Article on + [double-checked locking](https://preshing.com/20130930/double-checked-locking-is-fixed-in-cpp11/) +* [The Deadlock Empire](https://deadlockempire.github.io/), hands-on exercises + to construct deadlocks diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index ddd7f39370..a256da54a9 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -62,7 +62,11 @@ will acquire the GIL before calling the Python callback. Similarly, the back into Python. When writing C++ code that is called from other C++ code, if that code accesses -Python state, it must explicitly acquire and release the GIL. +Python state, it must explicitly acquire and release the GIL. A separate +document on deadlocks [#f8]_ elaborates on a particularly subtle interaction +with C++'s block-scope static variable initializer guard mutexes. + +.. [#f8] See docs/advanced/deadlock.md The classes :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be used to acquire and release the global interpreter lock in the body of a C++ @@ -142,6 +146,9 @@ following checklist. destructors can sometimes get invoked in weird and unexpected circumstances as a result of exceptions. +- C++ static block-scope variable initialization that calls back into Python can + cause deadlocks; see [#f8]_ for a detailed discussion. + - You should try running your code in a debug build. That will enable additional assertions within pybind11 that will throw exceptions on certain GIL handling errors (reference counting operations). diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 17c26aaa93..5cef2b81ab 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -24,7 +24,8 @@ changes are that: function is not available anymore. Due to NumPy changes, you may experience difficulties updating to NumPy 2. -Please see the [NumPy 2 migration guide](https://numpy.org/devdocs/numpy_2_0_migration_guide.html) for details. +Please see the `NumPy 2 migration guide `_ +for details. For example, a more direct change could be that the default integer ``"int_"`` (and ``"uint"``) is now ``ssize_t`` and not ``long`` (affects 64bit windows). diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 0c862e4bec..f6a7e83be8 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -343,7 +343,7 @@ class type_caster { #else // Alternate approach for CPython: this does the same as the above, but optimized // using the CPython API so as to avoid an unneeded attribute lookup. - else if (auto *tp_as_number = src.ptr()->ob_type->tp_as_number) { + else if (auto *tp_as_number = Py_TYPE(src.ptr())->tp_as_number) { if (PYBIND11_NB_BOOL(tp_as_number)) { res = (*PYBIND11_NB_BOOL(tp_as_number))(src.ptr()); } @@ -1564,15 +1564,24 @@ struct function_call { handle init_self; }; +// See PR #5396 for the discussion that led to this +template +struct is_same_or_base_of : std::is_same {}; + +// Only evaluate is_base_of if Derived is complete. +// is_base_of raises a compiler error if Derived is incomplete. +template +struct is_same_or_base_of + : any_of, std::is_base_of> {}; + /// Helper class which loads arguments for C++ functions called from Python template class argument_loader { using indices = make_index_sequence; - template - using argument_is_args = std::is_same, args>; + using argument_is_args = is_same_or_base_of>; template - using argument_is_kwargs = std::is_same, kwargs>; + using argument_is_kwargs = is_same_or_base_of>; // Get kwargs argument position, or -1 if not present: static constexpr auto kwargs_pos = constexpr_last(); diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 6c353eb09c..88e6d60a98 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -583,9 +583,9 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla return -1; } std::memset(view, 0, sizeof(Py_buffer)); - buffer_info *info = nullptr; + std::unique_ptr info = nullptr; try { - info = tinfo->get_buffer(obj, tinfo->get_buffer_data); + info.reset(tinfo->get_buffer(obj, tinfo->get_buffer_data)); } catch (...) { try_translate_exceptions(); raise_from(PyExc_BufferError, "Error getting buffer"); @@ -596,29 +596,72 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla } if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { - delete info; // view->obj = nullptr; // Was just memset to 0, so not necessary set_error(PyExc_BufferError, "Writable buffer requested for readonly storage"); return -1; } - view->obj = obj; - view->ndim = 1; - view->internal = info; - view->buf = info->ptr; + + // Fill in all the information, and then downgrade as requested by the caller, or raise an + // error if that's not possible. view->itemsize = info->itemsize; view->len = view->itemsize; for (auto s : info->shape) { view->len *= s; } + view->ndim = static_cast(info->ndim); + view->shape = info->shape.data(); + view->strides = info->strides.data(); view->readonly = static_cast(info->readonly); if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { view->format = const_cast(info->format.c_str()); } - if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { - view->ndim = (int) info->ndim; - view->strides = info->strides.data(); - view->shape = info->shape.data(); + + // Note, all contiguity flags imply PyBUF_STRIDES and lower. + if ((flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) { + if (PyBuffer_IsContiguous(view, 'C') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, + "C-contiguous buffer requested for discontiguous storage"); + return -1; + } + } else if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) { + if (PyBuffer_IsContiguous(view, 'F') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, + "Fortran-contiguous buffer requested for discontiguous storage"); + return -1; + } + } else if ((flags & PyBUF_ANY_CONTIGUOUS) == PyBUF_ANY_CONTIGUOUS) { + if (PyBuffer_IsContiguous(view, 'A') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, "Contiguous buffer requested for discontiguous storage"); + return -1; + } + + } else if ((flags & PyBUF_STRIDES) != PyBUF_STRIDES) { + // If no strides are requested, the buffer must be C-contiguous. + // https://docs.python.org/3/c-api/buffer.html#contiguity-requests + if (PyBuffer_IsContiguous(view, 'C') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, + "C-contiguous buffer requested for discontiguous storage"); + return -1; + } + + view->strides = nullptr; + + // Since this is a contiguous buffer, it can also pretend to be 1D. + if ((flags & PyBUF_ND) != PyBUF_ND) { + view->shape = nullptr; + view->ndim = 0; + } } + + // Set these after all checks so they don't leak out into the caller, and can be automatically + // cleaned up on error. + view->buf = info->ptr; + view->internal = info.release(); + view->obj = obj; Py_INCREF(view->obj); return 0; } diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index ac73dfe0c4..2f97279158 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -169,14 +169,6 @@ # endif #endif -#if !defined(PYBIND11_EXPORT_EXCEPTION) -# if defined(__apple_build_version__) -# define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT -# else -# define PYBIND11_EXPORT_EXCEPTION -# endif -#endif - // For CUDA, GCC7, GCC8: // PYBIND11_NOINLINE_FORCED is incompatible with `-Wattributes -Werror`. // When defining PYBIND11_NOINLINE_FORCED, it is best to also use `-Wno-attributes`. @@ -255,7 +247,7 @@ # define PYBIND11_INTERNAL_NUMPY_1_ONLY_DETECTED #endif -#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +#if (defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) # define PYBIND11_SIMPLE_GIL_MANAGEMENT #endif @@ -277,6 +269,17 @@ # endif #endif +// For libc++, the exceptions should be exported, +// otherwise, the exception translation would be incorrect. +// IMPORTANT: This code block must stay BELOW the #include above (see PR #5390). +#if !defined(PYBIND11_EXPORT_EXCEPTION) +# if defined(_LIBCPP_EXCEPTION) +# define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT +# else +# define PYBIND11_EXPORT_EXCEPTION +# endif +#endif + // Must be after including or one of the other headers specified by the standard #if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L # define PYBIND11_HAS_U8STRING @@ -335,6 +338,20 @@ #define PYBIND11_CONCAT(first, second) first##second #define PYBIND11_ENSURE_INTERNALS_READY pybind11::detail::get_internals(); +#if !defined(GRAALVM_PYTHON) +# define PYBIND11_PYCFUNCTION_GET_DOC(func) ((func)->m_ml->ml_doc) +# define PYBIND11_PYCFUNCTION_SET_DOC(func, doc) \ + do { \ + (func)->m_ml->ml_doc = (doc); \ + } while (0) +#else +# define PYBIND11_PYCFUNCTION_GET_DOC(func) (GraalPyCFunction_GetDoc((PyObject *) (func))) +# define PYBIND11_PYCFUNCTION_SET_DOC(func, doc) \ + do { \ + GraalPyCFunction_SetDoc((PyObject *) (func), (doc)); \ + } while (0) +#endif + #define PYBIND11_CHECK_PYTHON_VERSION \ { \ const char *compiled_ver \ @@ -1076,14 +1093,14 @@ struct overload_cast_impl { } template - constexpr auto operator()(Return (Class::*pmf)(Args...), - std::false_type = {}) const noexcept -> decltype(pmf) { + constexpr auto operator()(Return (Class::*pmf)(Args...), std::false_type = {}) const noexcept + -> decltype(pmf) { return pmf; } template - constexpr auto operator()(Return (Class::*pmf)(Args...) const, - std::true_type) const noexcept -> decltype(pmf) { + constexpr auto operator()(Return (Class::*pmf)(Args...) const, std::true_type) const noexcept + -> decltype(pmf) { return pmf; } }; diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index 7d546311e7..635614b0d6 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -156,9 +156,8 @@ constexpr auto concat(const descr &d, const Args &...args) { } #else template -constexpr auto concat(const descr &d, - const Args &...args) -> decltype(std::declval>() - + concat(args...)) { +constexpr auto concat(const descr &d, const Args &...args) + -> decltype(std::declval>() + concat(args...)) { return d + const_name(", ") + concat(args...); } #endif diff --git a/include/pybind11/detail/exception_translation.h b/include/pybind11/detail/exception_translation.h index 2764180bb0..22ae8a1c94 100644 --- a/include/pybind11/detail/exception_translation.h +++ b/include/pybind11/detail/exception_translation.h @@ -50,17 +50,17 @@ inline void try_translate_exceptions() { - delegate translation to the next translator by throwing a new type of exception. */ - bool handled = with_internals([&](internals &internals) { - auto &local_exception_translators = get_local_internals().registered_exception_translators; - if (detail::apply_exception_translators(local_exception_translators)) { - return true; - } - auto &exception_translators = internals.registered_exception_translators; - if (detail::apply_exception_translators(exception_translators)) { - return true; - } - return false; - }); + bool handled = with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &local_exception_translators) { + if (detail::apply_exception_translators(local_exception_translators)) { + return true; + } + if (detail::apply_exception_translators(exception_translators)) { + return true; + } + return false; + }); if (!handled) { set_error(PyExc_SystemError, "Exception escaped from default exception translator!"); diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 74b8f6a40c..bb101a725a 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -40,7 +40,11 @@ # if PY_VERSION_HEX >= 0x030C0000 || defined(_MSC_VER) // Version bump for Python 3.12+, before first 3.12 beta release. // Version bump for MSVC piggy-backed on PR #4779. See comments there. -# define PYBIND11_INTERNALS_VERSION 5 +# ifdef Py_GIL_DISABLED +# define PYBIND11_INTERNALS_VERSION 6 +# else +# define PYBIND11_INTERNALS_VERSION 5 +# endif # else # define PYBIND11_INTERNALS_VERSION 4 # endif @@ -178,6 +182,7 @@ static_assert(sizeof(instance_map_shard) % 64 == 0, struct internals { #ifdef Py_GIL_DISABLED pymutex mutex; + pymutex exception_translator_mutex; #endif // std::type_index -> pybind11's type information type_map registered_types_cpp; @@ -389,7 +394,7 @@ inline void translate_local_exception(std::exception_ptr p) { inline object get_python_state_dict() { object state_dict; -#if PYBIND11_INTERNALS_VERSION <= 4 || defined(PYPY_VERSION) +#if PYBIND11_INTERNALS_VERSION <= 4 || defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) state_dict = reinterpret_borrow(PyEval_GetBuiltins()); #else # if PY_VERSION_HEX < 0x03090000 @@ -583,6 +588,19 @@ inline auto with_internals(const F &cb) -> decltype(cb(get_internals())) { return cb(internals); } +template +inline auto with_exception_translators(const F &cb) + -> decltype(cb(get_internals().registered_exception_translators, + get_local_internals().registered_exception_translators)) { + auto &internals = get_internals(); +#ifdef Py_GIL_DISABLED + std::unique_lock lock((internals).exception_translator_mutex); +#endif + auto &local_internals = get_local_internals(); + return cb(internals.registered_exception_translators, + local_internals.registered_exception_translators); +} + inline std::uint64_t mix64(std::uint64_t z) { // David Stafford's variant 13 of the MurmurHash3 finalizer popularized // by the SplitMix PRNG. @@ -593,8 +611,8 @@ inline std::uint64_t mix64(std::uint64_t z) { } template -inline auto with_instance_map(const void *ptr, - const F &cb) -> decltype(cb(std::declval())) { +inline auto with_instance_map(const void *ptr, const F &cb) + -> decltype(cb(std::declval())) { auto &internals = get_internals(); #ifdef Py_GIL_DISABLED @@ -649,7 +667,8 @@ const char *c_str(Args &&...args) { } inline const char *get_function_record_capsule_name() { -#if PYBIND11_INTERNALS_VERSION > 4 + // On GraalPy, pointer equality of the names is currently not guaranteed +#if PYBIND11_INTERNALS_VERSION > 4 && !defined(GRAALVM_PYTHON) return get_internals().function_record_capsule_name.c_str(); #else return nullptr; diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index e40e44ba6c..d5d86dc6c1 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -117,7 +117,6 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector(t->tp_bases)) { check.push_back((PyTypeObject *) parent.ptr()); } - auto const &type_dict = get_internals().registered_types_py; for (size_t i = 0; i < check.size(); i++) { auto *type = check[i]; @@ -176,13 +175,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector &all_type_info(PyTypeObject *type) { - auto ins = all_type_info_get_cache(type); - if (ins.second) { - // New cache entry: populate it - all_type_info_populate(type, ins.first->second); - } - - return ins.first->second; + return all_type_info_get_cache(type).first->second; } /** @@ -459,7 +452,7 @@ PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_i } inline PyThreadState *get_thread_state_unchecked() { -#if defined(PYPY_VERSION) +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) return PyThreadState_GET(); #elif PY_VERSION_HEX < 0x030D0000 return _PyThreadState_UncheckedGet(); @@ -1161,14 +1154,14 @@ class type_caster_base : public type_caster_generic { does not have a private operator new implementation. A comma operator is used in the decltype argument to apply SFINAE to the public copy/move constructors.*/ template ::value>> - static auto make_copy_constructor(const T *) -> decltype(new T(std::declval()), - Constructor{}) { + static auto make_copy_constructor(const T *) + -> decltype(new T(std::declval()), Constructor{}) { return [](const void *arg) -> void * { return new T(*reinterpret_cast(arg)); }; } template ::value>> - static auto make_move_constructor(const T *) -> decltype(new T(std::declval()), - Constructor{}) { + static auto make_move_constructor(const T *) + -> decltype(new T(std::declval()), Constructor{}) { return [](const void *arg) -> void * { return new T(std::move(*const_cast(reinterpret_cast(arg)))); }; diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 0a9d7c2522..9b5d9e89b5 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -124,9 +124,9 @@ struct eigen_tensor_helper< template struct get_tensor_descriptor { static constexpr auto details - = const_name(", flags.writeable", "") - + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>( - ", flags.c_contiguous", ", flags.f_contiguous"); + = const_name(", flags.writeable", "") + const_name + < static_cast(Type::Layout) + == static_cast(Eigen::RowMajor) > (", flags.c_contiguous", ", flags.f_contiguous"); static constexpr auto value = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + eigen_tensor_helper>::dimensions_descriptor diff --git a/include/pybind11/eval.h b/include/pybind11/eval.h index 74d9b96b86..3ed1b5a4a9 100644 --- a/include/pybind11/eval.h +++ b/include/pybind11/eval.h @@ -94,18 +94,18 @@ void exec(const char (&s)[N], object global = globals(), object local = object() eval(s, std::move(global), std::move(local)); } -#if defined(PYPY_VERSION) +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) template object eval_file(str, object, object) { - pybind11_fail("eval_file not supported in PyPy3. Use eval"); + pybind11_fail("eval_file not supported in this interpreter. Use eval"); } template object eval_file(str, object) { - pybind11_fail("eval_file not supported in PyPy3. Use eval"); + pybind11_fail("eval_file not supported in this interpreter. Use eval"); } template object eval_file(str) { - pybind11_fail("eval_file not supported in PyPy3. Use eval"); + pybind11_fail("eval_file not supported in this interpreter. Use eval"); } #else template diff --git a/include/pybind11/gil_safe_call_once.h b/include/pybind11/gil_safe_call_once.h index 5f9e1b03c6..44e68f0294 100644 --- a/include/pybind11/gil_safe_call_once.h +++ b/include/pybind11/gil_safe_call_once.h @@ -46,6 +46,8 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) // get processed only when it is the main thread's turn again and it is running // normal Python code. However, this will be unnoticeable for quick call-once // functions, which is usually the case. +// +// For in-depth background, see docs/advanced/deadlock.md template class gil_safe_call_once_and_store { public: diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index c52df969e6..b4f93f1a6a 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -573,8 +573,7 @@ class cpp_function : public function { // chain. chain_start = rec; rec->next = chain; - auto rec_capsule - = reinterpret_borrow(((PyCFunctionObject *) m_ptr)->m_self); + auto rec_capsule = reinterpret_borrow(PyCFunction_GET_SELF(m_ptr)); rec_capsule.set_pointer(unique_rec.release()); guarded_strdup.release(); } else { @@ -634,12 +633,11 @@ class cpp_function : public function { } } - /* Install docstring */ auto *func = (PyCFunctionObject *) m_ptr; - std::free(const_cast(func->m_ml->ml_doc)); // Install docstring if it's non-empty (when at least one option is enabled) - func->m_ml->ml_doc - = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str()); + auto *doc = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str()); + std::free(const_cast(PYBIND11_PYCFUNCTION_GET_DOC(func))); + PYBIND11_PYCFUNCTION_SET_DOC(func, doc); if (rec->is_method) { m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr()); @@ -2328,13 +2326,20 @@ keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret) { inline std::pair all_type_info_get_cache(PyTypeObject *type) { auto res = with_internals([type](internals &internals) { - return internals - .registered_types_py + auto ins = internals + .registered_types_py #ifdef __cpp_lib_unordered_map_try_emplace - .try_emplace(type); + .try_emplace(type); #else - .emplace(type, std::vector()); + .emplace(type, std::vector()); #endif + if (ins.second) { + // For free-threading mode, this call must be under + // the with_internals() mutex lock, to avoid that other threads + // continue running with the empty ins.first->second. + all_type_info_populate(type, ins.first->second); + } + return ins; }); if (res.second) { // New cache entry created; set up a weak reference to automatically remove it if the type @@ -2586,10 +2591,12 @@ void implicitly_convertible() { } inline void register_exception_translator(ExceptionTranslator &&translator) { - detail::with_internals([&](detail::internals &internals) { - internals.registered_exception_translators.push_front( - std::forward(translator)); - }); + detail::with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &local_exception_translators) { + (void) local_exception_translators; + exception_translators.push_front(std::forward(translator)); + }); } /** @@ -2599,11 +2606,12 @@ inline void register_exception_translator(ExceptionTranslator &&translator) { * the exception. */ inline void register_local_exception_translator(ExceptionTranslator &&translator) { - detail::with_internals([&](detail::internals &internals) { - (void) internals; - detail::get_local_internals().registered_exception_translators.push_front( - std::forward(translator)); - }); + detail::with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &local_exception_translators) { + (void) exception_translators; + local_exception_translators.push_front(std::forward(translator)); + }); } /** @@ -2777,8 +2785,8 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char * } /* Don't call dispatch code if invoked from overridden function. - Unfortunately this doesn't work on PyPy. */ -#if !defined(PYPY_VERSION) + Unfortunately this doesn't work on PyPy and GraalPy. */ +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) # if PY_VERSION_HEX >= 0x03090000 PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get()); if (frame != nullptr) { diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 1e76d7bc13..027e36098b 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -643,7 +643,7 @@ struct error_fetch_and_normalize { bool have_trace = false; if (m_trace) { -#if !defined(PYPY_VERSION) +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) auto *tb = reinterpret_cast(m_trace.ptr()); // Get the deepest trace possible. @@ -1356,7 +1356,7 @@ inline bool PyUnicode_Check_Permissive(PyObject *o) { # define PYBIND11_STR_CHECK_FUN PyUnicode_Check #endif -inline bool PyStaticMethod_Check(PyObject *o) { return o->ob_type == &PyStaticMethod_Type; } +inline bool PyStaticMethod_Check(PyObject *o) { return Py_TYPE(o) == &PyStaticMethod_Type; } class kwargs_proxy : public handle { public: @@ -1470,11 +1470,17 @@ class iterator : public object { PYBIND11_OBJECT_DEFAULT(iterator, object, PyIter_Check) iterator &operator++() { + init(); advance(); return *this; } iterator operator++(int) { + // Note: We must call init() first so that rv.value is + // the same as this->value just before calling advance(). + // Otherwise, dereferencing the returned iterator may call + // advance() again and return the 3rd item instead of the 1st. + init(); auto rv = *this; advance(); return rv; @@ -1482,15 +1488,12 @@ class iterator : public object { // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 reference operator*() const { - if (m_ptr && !value.ptr()) { - auto &self = const_cast(*this); - self.advance(); - } + init(); return value; } pointer operator->() const { - operator*(); + init(); return &value; } @@ -1513,6 +1516,13 @@ class iterator : public object { friend bool operator!=(const iterator &a, const iterator &b) { return a->ptr() != b->ptr(); } private: + void init() const { + if (m_ptr && !value.ptr()) { + auto &self = const_cast(*this); + self.advance(); + } + } + void advance() { value = reinterpret_steal(PyIter_Next(m_ptr)); if (value.ptr() == nullptr && PyErr_Occurred()) { diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index fcb48dea33..af3a47f39c 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -694,6 +694,40 @@ struct ItemsViewImpl : public detail::items_view { Map ↦ }; +inline str format_message_key_error_key_object(handle py_key) { + str message = "pybind11::bind_map key"; + if (!py_key) { + return message; + } + try { + message = str(py_key); + } catch (const std::exception &) { + try { + message = repr(py_key); + } catch (const std::exception &) { + return message; + } + } + const ssize_t cut_length = 100; + if (len(message) > 2 * cut_length + 3) { + return str(message[slice(0, cut_length, 1)]) + str("✄✄✄") + + str(message[slice(-cut_length, static_cast(len(message)), 1)]); + } + return message; +} + +template +str format_message_key_error(const KeyType &key) { + object py_key; + try { + py_key = cast(key); + } catch (const std::exception &) { + do { // Trick to avoid "empty catch" warning/error. + } while (false); + } + return format_message_key_error_key_object(py_key); +} + PYBIND11_NAMESPACE_END(detail) template , typename... Args> @@ -785,7 +819,8 @@ class_ bind_map(handle scope, const std::string &name, Args && [](Map &m, const KeyType &k) -> MappedType & { auto it = m.find(k); if (it == m.end()) { - throw key_error(); + set_error(PyExc_KeyError, detail::format_message_key_error(k)); + throw error_already_set(); } return it->second; }, @@ -808,7 +843,8 @@ class_ bind_map(handle scope, const std::string &name, Args && cl.def("__delitem__", [](Map &m, const KeyType &k) { auto it = m.find(k); if (it == m.end()) { - throw key_error(); + set_error(PyExc_KeyError, detail::format_message_key_error(k)); + throw error_already_set(); } m.erase(it); }); diff --git a/tests/conftest.py b/tests/conftest.py index c40b11221f..9e7ca88120 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,8 +28,8 @@ @pytest.fixture(scope="session", autouse=True) def use_multiprocessing_forkserver_on_linux(): - if sys.platform != "linux": - # The default on Windows and macOS is "spawn": If it's not broken, don't fix it. + if sys.platform != "linux" or sys.implementation.name == "graalpy": + # The default on Windows, macOS and GraalPy is "spawn": If it's not broken, don't fix it. return # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 diff --git a/tests/custom_exceptions.py b/tests/custom_exceptions.py new file mode 100644 index 0000000000..b1a092d761 --- /dev/null +++ b/tests/custom_exceptions.py @@ -0,0 +1,10 @@ +from __future__ import annotations + + +class PythonMyException7(Exception): + def __init__(self, message): + self.message = message + super().__init__(message) + + def __str__(self): + return "[PythonMyException7]: " + self.message.a diff --git a/tests/env.py b/tests/env.py index 9f5347f2e5..b513e455d1 100644 --- a/tests/env.py +++ b/tests/env.py @@ -12,6 +12,7 @@ CPYTHON = platform.python_implementation() == "CPython" PYPY = platform.python_implementation() == "PyPy" +GRAALPY = sys.implementation.name == "graalpy" PY_GIL_DISABLED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index 3d2d84e77a..818d53a548 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -128,4 +128,9 @@ PYBIND11_MODULE(pybind11_tests, m, py::mod_gil_not_used()) { for (const auto &initializer : initializers()) { initializer(m); } + + py::class_(m, "TestContext") + .def(py::init<>(&TestContext::createNewContextForInit)) + .def("__enter__", &TestContext::contextEnter) + .def("__exit__", &TestContext::contextExit); } diff --git a/tests/pybind11_tests.h b/tests/pybind11_tests.h index 7be58feb6c..0eb0398df0 100644 --- a/tests/pybind11_tests.h +++ b/tests/pybind11_tests.h @@ -96,3 +96,24 @@ void ignoreOldStyleInitWarnings(F &&body) { )", py::dict(py::arg("body") = py::cpp_function(body))); } + +// See PR #5419 for background. +class TestContext { +public: + TestContext() = delete; + TestContext(const TestContext &) = delete; + TestContext(TestContext &&) = delete; + static TestContext *createNewContextForInit() { return new TestContext("new-context"); } + + pybind11::object contextEnter() { + py::object contextObj = py::cast(*this); + return contextObj; + } + void contextExit(const pybind11::object & /*excType*/, + const pybind11::object & /*excVal*/, + const pybind11::object & /*excTb*/) {} + +private: + explicit TestContext(const std::string &context) : context(context) {} + std::string context; +}; diff --git a/tests/requirements.txt b/tests/requirements.txt index f4f838ec0a..688ff6fe89 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,11 +2,12 @@ build~=1.0; python_version>="3.8" numpy~=1.23.0; python_version=="3.8" and platform_python_implementation=="PyPy" numpy~=1.25.0; python_version=="3.9" and platform_python_implementation=='PyPy' -numpy~=1.21.5; platform_python_implementation!="PyPy" and python_version>="3.8" and python_version<"3.10" -numpy~=1.22.2; platform_python_implementation!="PyPy" and python_version=="3.10" -numpy~=1.26.0; platform_python_implementation!="PyPy" and python_version>="3.11" and python_version<"3.13" +numpy~=1.26.0; platform_python_implementation=="GraalVM" and sys_platform=="linux" +numpy~=1.21.5; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.8" and python_version<"3.10" +numpy~=1.22.2; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version=="3.10" +numpy~=1.26.0; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.11" and python_version<"3.13" pytest~=7.0 pytest-timeout -scipy~=1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10" -scipy~=1.8.0; platform_python_implementation!="PyPy" and python_version=="3.10" and sys_platform!='win32' -scipy~=1.11.1; platform_python_implementation!="PyPy" and python_version>="3.11" and python_version<"3.13" and sys_platform!='win32' +scipy~=1.5.4; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version<"3.10" +scipy~=1.8.0; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version=="3.10" and sys_platform!='win32' +scipy~=1.11.1; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.11" and python_version<"3.13" and sys_platform!='win32' diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index a6c527c108..ac4489f70c 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -167,6 +167,125 @@ TEST_SUBMODULE(buffers, m) { sizeof(float)}); }); + // A matrix that uses Fortran storage order. + class FortranMatrix : public Matrix { + public: + FortranMatrix(py::ssize_t rows, py::ssize_t cols) : Matrix(cols, rows) { + print_created(this, + std::to_string(rows) + "x" + std::to_string(cols) + " Fortran matrix"); + } + + float operator()(py::ssize_t i, py::ssize_t j) const { return Matrix::operator()(j, i); } + + float &operator()(py::ssize_t i, py::ssize_t j) { return Matrix::operator()(j, i); } + + using Matrix::data; + + py::ssize_t rows() const { return Matrix::cols(); } + py::ssize_t cols() const { return Matrix::rows(); } + }; + py::class_(m, "FortranMatrix", py::buffer_protocol()) + .def(py::init()) + + .def("rows", &FortranMatrix::rows) + .def("cols", &FortranMatrix::cols) + + /// Bare bones interface + .def("__getitem__", + [](const FortranMatrix &m, std::pair i) { + if (i.first >= m.rows() || i.second >= m.cols()) { + throw py::index_error(); + } + return m(i.first, i.second); + }) + .def("__setitem__", + [](FortranMatrix &m, std::pair i, float v) { + if (i.first >= m.rows() || i.second >= m.cols()) { + throw py::index_error(); + } + m(i.first, i.second) = v; + }) + /// Provide buffer access + .def_buffer([](FortranMatrix &m) -> py::buffer_info { + return py::buffer_info(m.data(), /* Pointer to buffer */ + {m.rows(), m.cols()}, /* Buffer dimensions */ + /* Strides (in bytes) for each index */ + {sizeof(float), sizeof(float) * size_t(m.rows())}); + }); + + // A matrix that uses a discontiguous underlying memory block. + class DiscontiguousMatrix : public Matrix { + public: + DiscontiguousMatrix(py::ssize_t rows, + py::ssize_t cols, + py::ssize_t row_factor, + py::ssize_t col_factor) + : Matrix(rows * row_factor, cols * col_factor), m_row_factor(row_factor), + m_col_factor(col_factor) { + print_created(this, + std::to_string(rows) + "(*" + std::to_string(row_factor) + ")x" + + std::to_string(cols) + "(*" + std::to_string(col_factor) + + ") matrix"); + } + + ~DiscontiguousMatrix() { + print_destroyed(this, + std::to_string(rows() / m_row_factor) + "(*" + + std::to_string(m_row_factor) + ")x" + + std::to_string(cols() / m_col_factor) + "(*" + + std::to_string(m_col_factor) + ") matrix"); + } + + float operator()(py::ssize_t i, py::ssize_t j) const { + return Matrix::operator()(i * m_row_factor, j * m_col_factor); + } + + float &operator()(py::ssize_t i, py::ssize_t j) { + return Matrix::operator()(i * m_row_factor, j * m_col_factor); + } + + using Matrix::data; + + py::ssize_t rows() const { return Matrix::rows() / m_row_factor; } + py::ssize_t cols() const { return Matrix::cols() / m_col_factor; } + py::ssize_t row_factor() const { return m_row_factor; } + py::ssize_t col_factor() const { return m_col_factor; } + + private: + py::ssize_t m_row_factor; + py::ssize_t m_col_factor; + }; + py::class_(m, "DiscontiguousMatrix", py::buffer_protocol()) + .def(py::init()) + + .def("rows", &DiscontiguousMatrix::rows) + .def("cols", &DiscontiguousMatrix::cols) + + /// Bare bones interface + .def("__getitem__", + [](const DiscontiguousMatrix &m, std::pair i) { + if (i.first >= m.rows() || i.second >= m.cols()) { + throw py::index_error(); + } + return m(i.first, i.second); + }) + .def("__setitem__", + [](DiscontiguousMatrix &m, std::pair i, float v) { + if (i.first >= m.rows() || i.second >= m.cols()) { + throw py::index_error(); + } + m(i.first, i.second) = v; + }) + /// Provide buffer access + .def_buffer([](DiscontiguousMatrix &m) -> py::buffer_info { + return py::buffer_info(m.data(), /* Pointer to buffer */ + {m.rows(), m.cols()}, /* Buffer dimensions */ + /* Strides (in bytes) for each index */ + {size_t(m.col_factor()) * sizeof(float) * size_t(m.cols()) + * size_t(m.row_factor()), + size_t(m.col_factor()) * sizeof(float)}); + }); + class BrokenMatrix : public Matrix { public: BrokenMatrix(py::ssize_t rows, py::ssize_t cols) : Matrix(rows, cols) {} @@ -268,4 +387,56 @@ TEST_SUBMODULE(buffers, m) { }); m.def("get_buffer_info", [](const py::buffer &buffer) { return buffer.request(); }); + + // Expose Py_buffer for testing. + m.attr("PyBUF_FORMAT") = PyBUF_FORMAT; + m.attr("PyBUF_SIMPLE") = PyBUF_SIMPLE; + m.attr("PyBUF_ND") = PyBUF_ND; + m.attr("PyBUF_STRIDES") = PyBUF_STRIDES; + m.attr("PyBUF_INDIRECT") = PyBUF_INDIRECT; + m.attr("PyBUF_C_CONTIGUOUS") = PyBUF_C_CONTIGUOUS; + m.attr("PyBUF_F_CONTIGUOUS") = PyBUF_F_CONTIGUOUS; + m.attr("PyBUF_ANY_CONTIGUOUS") = PyBUF_ANY_CONTIGUOUS; + + m.def("get_py_buffer", [](const py::object &object, int flags) { + Py_buffer buffer; + memset(&buffer, 0, sizeof(Py_buffer)); + if (PyObject_GetBuffer(object.ptr(), &buffer, flags) == -1) { + throw py::error_already_set(); + } + + auto SimpleNamespace = py::module_::import("types").attr("SimpleNamespace"); + py::object result = SimpleNamespace("len"_a = buffer.len, + "readonly"_a = buffer.readonly, + "itemsize"_a = buffer.itemsize, + "format"_a = buffer.format, + "ndim"_a = buffer.ndim, + "shape"_a = py::none(), + "strides"_a = py::none(), + "suboffsets"_a = py::none()); + if (buffer.shape != nullptr) { + py::list l; + for (auto i = 0; i < buffer.ndim; i++) { + l.append(buffer.shape[i]); + } + py::setattr(result, "shape", l); + } + if (buffer.strides != nullptr) { + py::list l; + for (auto i = 0; i < buffer.ndim; i++) { + l.append(buffer.strides[i]); + } + py::setattr(result, "strides", l); + } + if (buffer.suboffsets != nullptr) { + py::list l; + for (auto i = 0; i < buffer.ndim; i++) { + l.append(buffer.suboffsets[i]); + } + py::setattr(result, "suboffsets", l); + } + + PyBuffer_Release(&buffer); + return result; + }); } diff --git a/tests/test_buffers.py b/tests/test_buffers.py index 535f33c2de..2612edb270 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -82,6 +82,8 @@ def test_from_python(): for j in range(m4.cols()): assert m3[i, j] == m4[i, j] + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") cstats = ConstructorStats.get(m.Matrix) assert cstats.alive() == 1 del m3, m4 @@ -118,6 +120,8 @@ def test_to_python(): mat2[2, 3] = 5 assert mat2[2, 3] == 5 + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") cstats = ConstructorStats.get(m.Matrix) assert cstats.alive() == 1 del mat @@ -235,3 +239,163 @@ def test_buffer_exception(): memoryview(m.BrokenMatrix(1, 1)) assert isinstance(excinfo.value.__cause__, RuntimeError) assert "for context" in str(excinfo.value.__cause__) + + +@pytest.mark.parametrize("type", ["pybind11", "numpy"]) +def test_c_contiguous_to_pybuffer(type): + if type == "pybind11": + mat = m.Matrix(5, 4) + elif type == "numpy": + mat = np.empty((5, 4), dtype=np.float32) + else: + raise ValueError(f"Unknown parametrization {type}") + + info = m.get_py_buffer(mat, m.PyBUF_SIMPLE) + assert info.format is None + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 0 # See discussion on PR #5407. + assert info.shape is None + assert info.strides is None + assert info.suboffsets is None + assert not info.readonly + info = m.get_py_buffer(mat, m.PyBUF_SIMPLE | m.PyBUF_FORMAT) + assert info.format == "f" + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 0 # See discussion on PR #5407. + assert info.shape is None + assert info.strides is None + assert info.suboffsets is None + assert not info.readonly + info = m.get_py_buffer(mat, m.PyBUF_ND) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides is None + assert info.suboffsets is None + assert not info.readonly + info = m.get_py_buffer(mat, m.PyBUF_STRIDES) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides == [4 * info.itemsize, info.itemsize] + assert info.suboffsets is None + assert not info.readonly + info = m.get_py_buffer(mat, m.PyBUF_INDIRECT) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides == [4 * info.itemsize, info.itemsize] + assert info.suboffsets is None # Should be filled in here, but we don't use it. + assert not info.readonly + + +@pytest.mark.parametrize("type", ["pybind11", "numpy"]) +def test_fortran_contiguous_to_pybuffer(type): + if type == "pybind11": + mat = m.FortranMatrix(5, 4) + elif type == "numpy": + mat = np.empty((5, 4), dtype=np.float32, order="F") + else: + raise ValueError(f"Unknown parametrization {type}") + + # A Fortran-shaped buffer can only be accessed at PyBUF_STRIDES level or higher. + info = m.get_py_buffer(mat, m.PyBUF_STRIDES) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides == [info.itemsize, 5 * info.itemsize] + assert info.suboffsets is None + assert not info.readonly + info = m.get_py_buffer(mat, m.PyBUF_INDIRECT) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides == [info.itemsize, 5 * info.itemsize] + assert info.suboffsets is None # Should be filled in here, but we don't use it. + assert not info.readonly + + +@pytest.mark.parametrize("type", ["pybind11", "numpy"]) +def test_discontiguous_to_pybuffer(type): + if type == "pybind11": + mat = m.DiscontiguousMatrix(5, 4, 2, 3) + elif type == "numpy": + mat = np.empty((5 * 2, 4 * 3), dtype=np.float32)[::2, ::3] + else: + raise ValueError(f"Unknown parametrization {type}") + + info = m.get_py_buffer(mat, m.PyBUF_STRIDES) + assert info.itemsize == ctypes.sizeof(ctypes.c_float) + assert info.len == 5 * 4 * info.itemsize + assert info.ndim == 2 + assert info.shape == [5, 4] + assert info.strides == [2 * 4 * 3 * info.itemsize, 3 * info.itemsize] + assert info.suboffsets is None + assert not info.readonly + + +@pytest.mark.parametrize("type", ["pybind11", "numpy"]) +def test_to_pybuffer_contiguity(type): + def check_strides(mat): + # The full block is memset to 0, so fill it with non-zero in real spots. + expected = np.arange(1, 5 * 4 + 1).reshape((5, 4)) + for i in range(5): + for j in range(4): + mat[i, j] = expected[i, j] + # If all strides are correct, the exposed buffer should match the input. + np.testing.assert_array_equal(np.array(mat), expected) + + if type == "pybind11": + cmat = m.Matrix(5, 4) # C contiguous. + fmat = m.FortranMatrix(5, 4) # Fortran contiguous. + dmat = m.DiscontiguousMatrix(5, 4, 2, 3) # Not contiguous. + expected_exception = BufferError + elif type == "numpy": + cmat = np.empty((5, 4), dtype=np.float32) # C contiguous. + fmat = np.empty((5, 4), dtype=np.float32, order="F") # Fortran contiguous. + dmat = np.empty((5 * 2, 4 * 3), dtype=np.float32)[::2, ::3] # Not contiguous. + # NumPy incorrectly raises ValueError; when the minimum NumPy requirement is + # above the version that fixes https://github.com/numpy/numpy/issues/3634 then + # BufferError can be used everywhere. + expected_exception = (BufferError, ValueError) + else: + raise ValueError(f"Unknown parametrization {type}") + + check_strides(cmat) + # Should work in C-contiguous mode, but not Fortran order. + m.get_py_buffer(cmat, m.PyBUF_C_CONTIGUOUS) + m.get_py_buffer(cmat, m.PyBUF_ANY_CONTIGUOUS) + with pytest.raises(expected_exception): + m.get_py_buffer(cmat, m.PyBUF_F_CONTIGUOUS) + + check_strides(fmat) + # These flags imply C-contiguity, so won't work. + with pytest.raises(expected_exception): + m.get_py_buffer(fmat, m.PyBUF_SIMPLE) + with pytest.raises(expected_exception): + m.get_py_buffer(fmat, m.PyBUF_ND) + # Should work in Fortran-contiguous mode, but not C order. + with pytest.raises(expected_exception): + m.get_py_buffer(fmat, m.PyBUF_C_CONTIGUOUS) + m.get_py_buffer(fmat, m.PyBUF_ANY_CONTIGUOUS) + m.get_py_buffer(fmat, m.PyBUF_F_CONTIGUOUS) + + check_strides(dmat) + # Should never work. + with pytest.raises(expected_exception): + m.get_py_buffer(dmat, m.PyBUF_SIMPLE) + with pytest.raises(expected_exception): + m.get_py_buffer(dmat, m.PyBUF_ND) + with pytest.raises(expected_exception): + m.get_py_buffer(dmat, m.PyBUF_C_CONTIGUOUS) + with pytest.raises(expected_exception): + m.get_py_buffer(dmat, m.PyBUF_ANY_CONTIGUOUS) + with pytest.raises(expected_exception): + m.get_py_buffer(dmat, m.PyBUF_F_CONTIGUOUS) diff --git a/tests/test_call_policies.cpp b/tests/test_call_policies.cpp index 92924cb452..9140f7e9f2 100644 --- a/tests/test_call_policies.cpp +++ b/tests/test_call_policies.cpp @@ -95,8 +95,8 @@ TEST_SUBMODULE(call_policies, m) { }, py::call_guard()); -#if !defined(PYPY_VERSION) - // `py::call_guard()` should work in PyPy as well, +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) + // `py::call_guard()` should work in PyPy/GraalPy as well, // but it's unclear how to test it without `PyGILState_GetThisThreadState`. auto report_gil_status = []() { auto is_gil_held = false; diff --git a/tests/test_call_policies.py b/tests/test_call_policies.py index 91670deb37..11aab9fd9c 100644 --- a/tests/test_call_policies.py +++ b/tests/test_call_policies.py @@ -8,6 +8,7 @@ @pytest.mark.xfail("env.PYPY", reason="sometimes comes out 1 off on PyPy", strict=False) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_keep_alive_argument(capture): n_inst = ConstructorStats.detail_reg_inst() with capture: @@ -60,6 +61,7 @@ def test_keep_alive_argument(capture): assert str(excinfo.value) == "Could not activate keep_alive!" +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_keep_alive_return_value(capture): n_inst = ConstructorStats.detail_reg_inst() with capture: @@ -118,6 +120,7 @@ def test_keep_alive_return_value(capture): # https://foss.heptapod.net/pypy/pypy/-/issues/2447 @pytest.mark.xfail("env.PYPY", reason="_PyObject_GetDictPtr is unimplemented") +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_alive_gc(capture): n_inst = ConstructorStats.detail_reg_inst() p = m.ParentGC() @@ -137,6 +140,7 @@ def test_alive_gc(capture): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_alive_gc_derived(capture): class Derived(m.Parent): pass @@ -159,6 +163,7 @@ class Derived(m.Parent): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_alive_gc_multi_derived(capture): class Derived(m.Parent, m.Child): def __init__(self): @@ -185,6 +190,7 @@ def __init__(self): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_return_none(capture): n_inst = ConstructorStats.detail_reg_inst() with capture: @@ -212,6 +218,7 @@ def test_return_none(capture): assert capture == "Releasing parent." +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_keep_alive_constructor(capture): n_inst = ConstructorStats.detail_reg_inst() diff --git a/tests/test_callbacks.cpp b/tests/test_callbacks.cpp index e303e76567..f28f0a9e28 100644 --- a/tests/test_callbacks.cpp +++ b/tests/test_callbacks.cpp @@ -270,7 +270,7 @@ TEST_SUBMODULE(callbacks, m) { m.add_object("custom_function", PyCFunction_New(custom_def, rec_capsule.ptr())); // This test requires a new ABI version to pass -#if PYBIND11_INTERNALS_VERSION > 4 +#if PYBIND11_INTERNALS_VERSION > 4 && !defined(GRAALVM_PYTHON) // rec_capsule with nullptr name py::capsule rec_capsule2(std::malloc(1), [](void *data) { std::free(data); }); m.add_object("custom_function2", PyCFunction_New(custom_def, rec_capsule2.ptr())); diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index db6d8dece0..00b9015d0a 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -90,6 +90,7 @@ def f(*args, **kwargs): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_lambda_closure_cleanup(): m.test_lambda_closure_cleanup() cstats = m.payload_cstats() @@ -98,6 +99,7 @@ def test_lambda_closure_cleanup(): assert cstats.move_constructions >= 1 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_cpp_callable_cleanup(): alive_counts = m.test_cpp_callable_cleanup() assert alive_counts == [0, 1, 2, 1, 2, 1, 0] diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 9001d86b19..cb84c327a0 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -52,8 +52,24 @@ void bind_empty0(py::module_ &m) { } } // namespace pr4220_tripped_over_this + +namespace pr5396_forward_declared_class { +class ForwardClass; +class Args : public py::args {}; +} // namespace pr5396_forward_declared_class + } // namespace test_class +static_assert(py::detail::is_same_or_base_of::value, ""); +static_assert( + py::detail::is_same_or_base_of::value, + ""); +static_assert(!py::detail::is_same_or_base_of< + py::args, + test_class::pr5396_forward_declared_class::ForwardClass>::value, + ""); + TEST_SUBMODULE(class_, m) { m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); diff --git a/tests/test_class.py b/tests/test_class.py index 9b2b1d8346..01963d0122 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys from unittest import mock import pytest @@ -27,6 +28,9 @@ def test_instance(msg): instance = m.NoConstructor.new_instance() + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + cstats = ConstructorStats.get(m.NoConstructor) assert cstats.alive() == 1 del instance @@ -35,6 +39,10 @@ def test_instance(msg): def test_instance_new(): instance = m.NoConstructorNew() # .__new__(m.NoConstructor.__class__) + + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + cstats = ConstructorStats.get(m.NoConstructorNew) assert cstats.alive() == 1 del instance @@ -361,7 +369,7 @@ def test_brace_initialization(): assert b.vec == [123, 456] -@pytest.mark.xfail("env.PYPY") +@pytest.mark.xfail("env.PYPY or env.GRAALPY") def test_class_refcount(): """Instances must correctly increase/decrease the reference count of their types (#1029)""" from sys import getrefcount @@ -501,3 +509,31 @@ def test_pr4220_tripped_over_this(): m.Empty0().get_msg() == "This is really only meant to exercise successful compilation." ) + + +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") +def test_all_type_info_multithreaded(): + # See PR #5419 for background. + import threading + + from pybind11_tests import TestContext + + class Context(TestContext): + pass + + num_runs = 10 + num_threads = 4 + barrier = threading.Barrier(num_threads) + + def func(): + barrier.wait() + with Context(): + pass + + for _ in range(num_runs): + threads = [threading.Thread(target=func) for _ in range(num_threads)] + for thread in threads: + thread.start() + + for thread in threads: + thread.join() diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index f28bde08e5..ec365c0f67 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -55,8 +55,10 @@ possibly_uninitialized(PYTHON_MODULE_EXTENSION Python_INTERPRETER_ID) pybind11_add_build_test(subdirectory_function) pybind11_add_build_test(subdirectory_target) -if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy") - message(STATUS "Skipping embed test on PyPy") +if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" + OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy") + message(STATUS "Skipping embed test on PyPy or GraalPy") else() pybind11_add_build_test(subdirectory_embed) endif() @@ -66,10 +68,14 @@ if(PYBIND11_INSTALL) mock_install ${CMAKE_COMMAND} "-DCMAKE_INSTALL_PREFIX=${pybind11_BINARY_DIR}/mock_install" -P "${pybind11_BINARY_DIR}/cmake_install.cmake") - pybind11_add_build_test(installed_function INSTALL) + if(NOT "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy") + pybind11_add_build_test(installed_function INSTALL) + endif() pybind11_add_build_test(installed_target INSTALL) - if(NOT ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" - )) + if(NOT + ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" + OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy")) pybind11_add_build_test(installed_embed INSTALL) endif() endif() diff --git a/tests/test_copy_move.py b/tests/test_copy_move.py index ee046305f5..3a3f293414 100644 --- a/tests/test_copy_move.py +++ b/tests/test_copy_move.py @@ -2,6 +2,7 @@ import pytest +import env # noqa: F401 from pybind11_tests import copy_move_policies as m @@ -17,6 +18,7 @@ def test_lacking_move_ctor(): assert "is neither movable nor copyable!" in str(excinfo.value) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_move_and_copy_casts(): """Cast some values in C++ via custom type casters and count the number of moves/copies.""" @@ -44,6 +46,7 @@ def test_move_and_copy_casts(): assert c_m.alive() + c_mc.alive() + c_c.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_move_and_copy_loads(): """Call some functions that load arguments via custom type casters and count the number of moves/copies.""" @@ -77,6 +80,7 @@ def test_move_and_copy_loads(): @pytest.mark.skipif(not m.has_optional, reason="no ") +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_move_and_copy_load_optional(): """Tests move/copy loads of std::optional arguments""" diff --git a/tests/test_cpp_conduit.py b/tests/test_cpp_conduit.py index 51fcf69368..eb300587fa 100644 --- a/tests/test_cpp_conduit.py +++ b/tests/test_cpp_conduit.py @@ -7,6 +7,7 @@ import home_planet_very_lonely_traveler import pytest +import env from pybind11_tests import cpp_conduit as home_planet @@ -27,7 +28,10 @@ def test_call_cpp_conduit_success(): home_planet.cpp_type_info_capsule_Traveler, b"raw_pointer_ephemeral", ) - assert cap.__class__.__name__ == "PyCapsule" + assert cap.__class__.__name__ == "PyCapsule" or ( + # Note: this will become unnecessary in the next GraalPy release + env.GRAALPY and cap.__class__.__name__ == "capsule" + ) def test_call_cpp_conduit_platform_abi_id_mismatch(): diff --git a/tests/test_custom_type_casters.py b/tests/test_custom_type_casters.py index 689b1e9cb6..dcf3ca734a 100644 --- a/tests/test_custom_type_casters.py +++ b/tests/test_custom_type_casters.py @@ -2,6 +2,7 @@ import pytest +import env # noqa: F401 from pybind11_tests import custom_type_casters as m @@ -94,6 +95,7 @@ def test_noconvert_args(msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_custom_caster_destruction(): """Tests that returning a pointer to a type that gets converted with a custom type caster gets destroyed when the function has py::return_value_policy::take_ownership policy applied. diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index ca3340bd53..bb2865cade 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -34,7 +34,7 @@ def add_ref(obj): # PyPy does not seem to reliably garbage collect. -@pytest.mark.skipif("env.PYPY") +@pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_self_cycle(gc_tester): obj = m.OwnsPythonObjects() obj.value = obj @@ -42,7 +42,7 @@ def test_self_cycle(gc_tester): # PyPy does not seem to reliably garbage collect. -@pytest.mark.skipif("env.PYPY") +@pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_indirect_cycle(gc_tester): obj = m.OwnsPythonObjects() obj_list = [obj] diff --git a/tests/test_eigen_matrix.py b/tests/test_eigen_matrix.py index e1d7433f15..5093ce7d88 100644 --- a/tests/test_eigen_matrix.py +++ b/tests/test_eigen_matrix.py @@ -2,6 +2,7 @@ import pytest +import env # noqa: F401 from pybind11_tests import ConstructorStats np = pytest.importorskip("numpy") @@ -394,6 +395,7 @@ def test_eigen_return_references(): np.testing.assert_array_equal(a_copy5, c5want) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def assert_keeps_alive(cl, method, *args): cstats = ConstructorStats.get(cl) start_with = cstats.alive() @@ -409,6 +411,7 @@ def assert_keeps_alive(cl, method, *args): assert cstats.alive() == start_with +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_eigen_keepalive(): a = m.ReturnTester() cstats = ConstructorStats.get(m.ReturnTester) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py index a2b99d9d7d..0860c1dad7 100644 --- a/tests/test_eigen_tensor.py +++ b/tests/test_eigen_tensor.py @@ -4,6 +4,8 @@ import pytest +import env # noqa: F401 + np = pytest.importorskip("numpy") eigen_tensor = pytest.importorskip("pybind11_tests.eigen_tensor") submodules = [eigen_tensor.c_style, eigen_tensor.f_style] @@ -61,6 +63,7 @@ def assert_equal_tensor_ref(mat, writeable=True, modified=None): @pytest.mark.parametrize("m", submodules) @pytest.mark.parametrize("member_name", ["member", "member_view"]) +@pytest.mark.skipif("env.GRAALPY", reason="Different refcounting mechanism") def test_reference_internal(m, member_name): if not hasattr(sys, "getrefcount"): pytest.skip("No reference counting") diff --git a/tests/test_embed/CMakeLists.txt b/tests/test_embed/CMakeLists.txt index 9b539cd42d..f646458d1e 100644 --- a/tests/test_embed/CMakeLists.txt +++ b/tests/test_embed/CMakeLists.txt @@ -1,8 +1,10 @@ possibly_uninitialized(PYTHON_MODULE_EXTENSION Python_INTERPRETER_ID) -if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy") - message(STATUS "Skipping embed test on PyPy") - add_custom_target(cpptest) # Dummy target on PyPy. Embedding is not supported. +if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" + OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy") + message(STATUS "Skipping embed test on PyPy or GraalPy") + add_custom_target(cpptest) # Dummy target on PyPy or GraalPy. Embedding is not supported. set(_suppress_unused_variable_warning "${DOWNLOAD_CATCH}") return() endif() diff --git a/tests/test_enum.py b/tests/test_enum.py index 9914b90013..03cd1c1a64 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -3,9 +3,11 @@ import pytest +import env # noqa: F401 from pybind11_tests import enums as m +@pytest.mark.xfail("env.GRAALPY", reason="TODO should get fixed on GraalPy side") def test_unscoped_enum(): assert str(m.UnscopedEnum.EOne) == "UnscopedEnum.EOne" assert str(m.UnscopedEnum.ETwo) == "UnscopedEnum.ETwo" @@ -193,6 +195,7 @@ def test_implicit_conversion(): assert repr(x) == "{: 3, : 4}" +@pytest.mark.xfail("env.GRAALPY", reason="TODO should get fixed on GraalPy side") def test_binary_operators(): assert int(m.Flags.Read) == 4 assert int(m.Flags.Write) == 2 diff --git a/tests/test_eval.py b/tests/test_eval.py index 45b68ece72..8ac1907c7a 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -19,7 +19,7 @@ def test_evals(capture): assert m.test_eval_failure() -@pytest.mark.xfail("env.PYPY", raises=RuntimeError) +@pytest.mark.xfail("env.PYPY or env.GRAALPY", raises=RuntimeError) def test_eval_file(): filename = os.path.join(os.path.dirname(__file__), "test_eval_call.py") assert m.test_eval_file(filename) diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index c1d05bb241..0a970065b4 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -111,6 +111,16 @@ struct PythonAlreadySetInDestructor { py::str s; }; +struct CustomData { + explicit CustomData(const std::string &a) : a(a) {} + std::string a; +}; + +struct MyException7 { + explicit MyException7(const CustomData &message) : message(message) {} + CustomData message; +}; + TEST_SUBMODULE(exceptions, m) { m.def("throw_std_exception", []() { throw std::runtime_error("This exception was intentionally thrown."); }); @@ -385,4 +395,33 @@ TEST_SUBMODULE(exceptions, m) { // m.def("pass_exception_void", [](const py::exception&) {}); // Does not compile. m.def("return_exception_void", []() { return py::exception(); }); + + m.def("throws7", []() { + auto data = CustomData("abc"); + throw MyException7(data); + }); + + py::class_(m, "CustomData", py::module_local()) + .def(py::init()) + .def_readwrite("a", &CustomData::a); + + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store + PythonMyException7_storage; + PythonMyException7_storage.call_once_and_store_result([&]() { + auto mod = py::module_::import("custom_exceptions"); + py::object obj = mod.attr("PythonMyException7"); + return obj; + }); + + py::register_local_exception_translator([](std::exception_ptr p) { + try { + if (p) { + std::rethrow_exception(p); + } + } catch (const MyException7 &e) { + auto exc_type = PythonMyException7_storage.get_stored(); + py::object exc_inst = exc_type(e.message); + PyErr_SetObject(PyExc_Exception, exc_inst.ptr()); + } + }); } diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index ba5063a749..47214a7029 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -3,6 +3,7 @@ import sys import pytest +from custom_exceptions import PythonMyException7 import env import pybind11_cross_module_tests as cm @@ -75,7 +76,7 @@ def test_cross_module_exceptions(msg): # TODO: FIXME @pytest.mark.xfail( - "env.MACOS and (env.PYPY or pybind11_tests.compiler_info.startswith('Homebrew Clang')) or sys.platform.startswith('emscripten')", + "env.MACOS and env.PYPY", raises=RuntimeError, reason="See Issue #2847, PR #2999, PR #4324", ) @@ -195,7 +196,12 @@ def test_custom(msg): raise RuntimeError("Exception error: caught child from parent") from err assert msg(excinfo.value) == "this is a helper-defined translated exception" + with pytest.raises(PythonMyException7) as excinfo: + m.throws7() + assert msg(excinfo.value) == "[PythonMyException7]: abc" + +@pytest.mark.xfail("env.GRAALPY", reason="TODO should get fixed on GraalPy side") def test_nested_throws(capture): """Tests nested (e.g. C++ -> Python -> C++) exception handling""" @@ -364,6 +370,7 @@ def _test_flaky_exception_failure_point_init_py_3_12(): "env.PYPY and sys.version_info[:2] < (3, 12)", reason="PyErr_NormalizeException Segmentation fault", ) +@pytest.mark.xfail("env.GRAALPY", reason="TODO should be fixed on GraalPy side") def test_flaky_exception_failure_point_init(): if sys.version_info[:2] < (3, 12): _test_flaky_exception_failure_point_init_before_py_3_12() @@ -371,6 +378,7 @@ def test_flaky_exception_failure_point_init(): _test_flaky_exception_failure_point_init_py_3_12() +@pytest.mark.xfail("env.GRAALPY", reason="TODO should be fixed on GraalPy side") def test_flaky_exception_failure_point_str(): what, py_err_set_after_what = m.error_already_set_what( FlakyException, ("failure_point_str",) diff --git a/tests/test_factory_constructors.py b/tests/test_factory_constructors.py index 0ddad5e323..1d3a9bcddd 100644 --- a/tests/test_factory_constructors.py +++ b/tests/test_factory_constructors.py @@ -4,11 +4,13 @@ import pytest +import env # noqa: F401 from pybind11_tests import ConstructorStats from pybind11_tests import factory_constructors as m from pybind11_tests.factory_constructors import tag +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_init_factory_basic(): """Tests py::init_factory() wrapper around various ways of returning the object""" @@ -102,6 +104,7 @@ def test_init_factory_signature(msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_init_factory_casting(): """Tests py::init_factory() wrapper with various upcasting and downcasting returns""" @@ -150,6 +153,7 @@ def test_init_factory_casting(): ] +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_init_factory_alias(): """Tests py::init_factory() wrapper with value conversions and alias types""" @@ -220,6 +224,7 @@ def get(self): ] +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_init_factory_dual(): """Tests init factory functions with dual main/alias factory functions""" from pybind11_tests.factory_constructors import TestFactory7 @@ -302,6 +307,7 @@ def get(self): ] +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_no_placement_new(capture): """Prior to 2.2, `py::init<...>` relied on the type supporting placement new; this tests a class without placement new support.""" @@ -350,6 +356,7 @@ def strip_comments(s): return re.sub(r"\s+#.*", "", s) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_a(capture, msg): """When the constructor is overloaded, previous overloads can require a preallocated value. This test makes sure that such preallocated values only happen when they might be necessary, @@ -372,6 +379,7 @@ def test_reallocation_a(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_b(capture, msg): with capture: create_and_destroy(1.5) @@ -388,6 +396,7 @@ def test_reallocation_b(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_c(capture, msg): with capture: create_and_destroy(2, 3) @@ -402,6 +411,7 @@ def test_reallocation_c(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_d(capture, msg): with capture: create_and_destroy(2.5, 3) @@ -417,6 +427,7 @@ def test_reallocation_d(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_e(capture, msg): with capture: create_and_destroy(3.5, 4.5) @@ -432,6 +443,7 @@ def test_reallocation_e(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_f(capture, msg): with capture: create_and_destroy(4, 0.5) @@ -448,6 +460,7 @@ def test_reallocation_f(capture, msg): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_reallocation_g(capture, msg): with capture: create_and_destroy(5, "hi") diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index eab92093c8..257e66612f 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -211,6 +211,10 @@ def _run_in_threads(test_fn, num_threads, parallel): @pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +@pytest.mark.skipif( + "env.GRAALPY", + reason="GraalPy transiently complains about unfinished threads at process exit", +) def test_run_in_process_one_thread(test_fn): """Makes sure there is no GIL deadlock when running in a thread. @@ -221,6 +225,10 @@ def test_run_in_process_one_thread(test_fn): @pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +@pytest.mark.skipif( + "env.GRAALPY", + reason="GraalPy transiently complains about unfinished threads at process exit", +) def test_run_in_process_multiple_threads_parallel(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel. @@ -231,6 +239,10 @@ def test_run_in_process_multiple_threads_parallel(test_fn): @pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +@pytest.mark.skipif( + "env.GRAALPY", + reason="GraalPy transiently complains about unfinished threads at process exit", +) def test_run_in_process_multiple_threads_sequential(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially. @@ -241,6 +253,10 @@ def test_run_in_process_multiple_threads_sequential(test_fn): @pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +@pytest.mark.skipif( + "env.GRAALPY", + reason="GraalPy transiently complains about unfinished threads at process exit", +) def test_run_in_process_direct(test_fn): """Makes sure there is no GIL deadlock when using processes. diff --git a/tests/test_iostream.py b/tests/test_iostream.py index c3d987787a..606028d6fe 100644 --- a/tests/test_iostream.py +++ b/tests/test_iostream.py @@ -6,8 +6,14 @@ import pytest +import env # noqa: F401 from pybind11_tests import iostream as m +pytestmark = pytest.mark.skipif( + "env.GRAALPY", + reason="Delayed prints from finalizers from other tests can end up in the output", +) + def test_captured(capsys): msg = "I've been redirected to Python, I hope!" diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index bc76ec7c2d..09036ccd51 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -14,6 +14,26 @@ #include +// Classes needed for subclass test. +class ArgsSubclass : public py::args { + using py::args::args; +}; +class KWArgsSubclass : public py::kwargs { + using py::kwargs::kwargs; +}; +namespace pybind11 { +namespace detail { +template <> +struct handle_type_name { + static constexpr auto name = const_name("*Args"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("**KWArgs"); +}; +} // namespace detail +} // namespace pybind11 + TEST_SUBMODULE(kwargs_and_defaults, m) { auto kw_func = [](int x, int y) { return "x=" + std::to_string(x) + ", y=" + std::to_string(y); }; @@ -322,4 +342,10 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { py::pos_only{}, py::arg("i"), py::arg("j")); + + // Test support for args and kwargs subclasses + m.def("args_kwargs_subclass_function", + [](const ArgsSubclass &args, const KWArgsSubclass &kwargs) { + return py::make_tuple(args, kwargs); + }); } diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index b9b1a7ea87..e3e9a0a0d1 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -2,6 +2,7 @@ import pytest +import env # noqa: F401 from pybind11_tests import kwargs_and_defaults as m @@ -17,6 +18,10 @@ def test_function_signatures(doc): assert ( doc(m.args_kwargs_function) == "args_kwargs_function(*args, **kwargs) -> tuple" ) + assert ( + doc(m.args_kwargs_subclass_function) + == "args_kwargs_subclass_function(*Args, **KWArgs) -> tuple" + ) assert ( doc(m.KWClass.foo0) == "foo0(self: m.kwargs_and_defaults.KWClass, arg0: int, arg1: float) -> None" @@ -98,6 +103,7 @@ def test_arg_and_kwargs(): args = "a1", "a2" kwargs = {"arg3": "a3", "arg4": 4} assert m.args_kwargs_function(*args, **kwargs) == (args, kwargs) + assert m.args_kwargs_subclass_function(*args, **kwargs) == (args, kwargs) def test_mixed_args_and_kwargs(msg): @@ -378,6 +384,7 @@ def test_signatures(): ) +@pytest.mark.skipif("env.GRAALPY", reason="Different refcounting mechanism") def test_args_refcount(): """Issue/PR #1216 - py::args elements get double-inc_ref()ed when combined with regular arguments""" @@ -413,6 +420,12 @@ def test_args_refcount(): ) assert refcount(myval) == expected + assert m.args_kwargs_subclass_function(7, 8, myval, a=1, b=myval) == ( + (7, 8, myval), + {"a": 1, "b": myval}, + ) + assert refcount(myval) == expected + exp3 = refcount(myval, myval, myval) assert m.args_refcount(myval, myval, myval) == (exp3, exp3, exp3) assert refcount(myval) == expected diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index dfa31f5465..91c7b7751e 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -4,7 +4,7 @@ import pytest -import env # noqa: F401 +import env from pybind11_tests import ConstructorStats from pybind11_tests import methods_and_attributes as m @@ -68,6 +68,9 @@ def test_methods_and_attributes(): instance1.value = 100 assert str(instance1) == "ExampleMandA[value=100]" + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + cstats = ConstructorStats.get(m.ExampleMandA) assert cstats.alive() == 2 del instance1, instance2 @@ -316,6 +319,8 @@ def test_dynamic_attributes(): instance.__dict__ = [] assert str(excinfo.value) == "__dict__ must be set to a dictionary, not a 'list'" + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") cstats = ConstructorStats.get(m.DynamicClass) assert cstats.alive() == 1 del instance @@ -337,6 +342,7 @@ class PythonDerivedDynamicClass(m.DynamicClass): # https://foss.heptapod.net/pypy/pypy/-/issues/2447 @pytest.mark.xfail("env.PYPY") +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_cyclic_gc(): # One object references itself instance = m.DynamicClass() diff --git a/tests/test_modules.py b/tests/test_modules.py index 95835e14ea..436271a701 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -39,6 +39,9 @@ def test_reference_internal(): assert str(b.get_a2()) == "A[43]" assert str(b.a2) == "A[43]" + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + astats, bstats = ConstructorStats.get(ms.A), ConstructorStats.get(ms.B) assert astats.alive() == 2 assert bstats.alive() == 1 @@ -97,9 +100,9 @@ def test_def_submodule_failures(): sm = m.def_submodule(m, b"ScratchSubModuleName") # Using bytes to show it works. assert sm.__name__ == m.__name__ + "." + "ScratchSubModuleName" malformed_utf8 = b"\x80" - if env.PYPY: + if env.PYPY or env.GRAALPY: # It is not worth the effort finding a trigger for a failure when running with PyPy. - pytest.skip("Sufficiently exercised on platforms other than PyPy.") + pytest.skip("Sufficiently exercised on platforms other than PyPy/GraalPy.") else: # Meant to trigger PyModule_GetName() failure: sm_name_orig = sm.__name__ diff --git a/tests/test_multiple_inheritance.py b/tests/test_multiple_inheritance.py index d445824b54..6f5a656f5d 100644 --- a/tests/test_multiple_inheritance.py +++ b/tests/test_multiple_inheritance.py @@ -2,7 +2,7 @@ import pytest -import env # noqa: F401 +import env from pybind11_tests import ConstructorStats from pybind11_tests import multiple_inheritance as m @@ -279,8 +279,9 @@ def test_mi_unaligned_base(): c = m.I801C() d = m.I801D() - # + 4 below because we have the two instances, and each instance has offset base I801B2 - assert ConstructorStats.detail_reg_inst() == n_inst + 4 + if not env.GRAALPY: + # + 4 below because we have the two instances, and each instance has offset base I801B2 + assert ConstructorStats.detail_reg_inst() == n_inst + 4 b1c = m.i801b1_c(c) assert b1c is c b2c = m.i801b2_c(c) @@ -290,6 +291,9 @@ def test_mi_unaligned_base(): b2d = m.i801b2_d(d) assert b2d is d + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + assert ConstructorStats.detail_reg_inst() == n_inst + 4 # no extra instances del c, b1c, b2c assert ConstructorStats.detail_reg_inst() == n_inst + 2 @@ -312,7 +316,8 @@ def test_mi_base_return(): assert d1.a == 1 assert d1.b == 2 - assert ConstructorStats.detail_reg_inst() == n_inst + 4 + if not env.GRAALPY: + assert ConstructorStats.detail_reg_inst() == n_inst + 4 c2 = m.i801c_b2() assert type(c2) is m.I801C @@ -324,12 +329,13 @@ def test_mi_base_return(): assert d2.a == 1 assert d2.b == 2 - assert ConstructorStats.detail_reg_inst() == n_inst + 8 + if not env.GRAALPY: + assert ConstructorStats.detail_reg_inst() == n_inst + 8 - del c2 - assert ConstructorStats.detail_reg_inst() == n_inst + 6 - del c1, d1, d2 - assert ConstructorStats.detail_reg_inst() == n_inst + del c2 + assert ConstructorStats.detail_reg_inst() == n_inst + 6 + del c1, d1, d2 + assert ConstructorStats.detail_reg_inst() == n_inst # Returning an unregistered derived type with a registered base; we won't # pick up the derived type, obviously, but should still work (as an object diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index bc7b3d5557..6e8bde826f 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -242,6 +242,7 @@ def assert_references(a, b, base=None): assert_references(a1m, a2, a1) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_numpy_view(capture): with capture: ac = m.ArrayClass() @@ -465,7 +466,7 @@ def test_array_resize(): assert b.shape == (8, 8) -@pytest.mark.xfail("env.PYPY") +@pytest.mark.xfail("env.PYPY or env.GRAALPY") def test_array_create_and_resize(): a = m.create_and_resize(2) assert a.size == 4 diff --git a/tests/test_opaque_types.py b/tests/test_opaque_types.py index 3420864367..56c9b5db19 100644 --- a/tests/test_opaque_types.py +++ b/tests/test_opaque_types.py @@ -2,6 +2,7 @@ import pytest +import env from pybind11_tests import ConstructorStats, UserType from pybind11_tests import opaque_types as m @@ -30,7 +31,9 @@ def test_pointers(msg): living_before = ConstructorStats.get(UserType).alive() assert m.get_void_ptr_value(m.return_void_ptr()) == 0x1234 assert m.get_void_ptr_value(UserType()) # Should also work for other C++ types - assert ConstructorStats.get(UserType).alive() == living_before + + if not env.GRAALPY: + assert ConstructorStats.get(UserType).alive() == living_before with pytest.raises(TypeError) as excinfo: m.get_void_ptr_value([1, 2, 3]) # This should not work diff --git a/tests/test_operator_overloading.py b/tests/test_operator_overloading.py index b6760902dc..47949042dd 100644 --- a/tests/test_operator_overloading.py +++ b/tests/test_operator_overloading.py @@ -2,10 +2,12 @@ import pytest +import env from pybind11_tests import ConstructorStats from pybind11_tests import operators as m +@pytest.mark.xfail("env.GRAALPY", reason="TODO should get fixed on GraalPy side") def test_operator_overloading(): v1 = m.Vector2(1, 2) v2 = m.Vector(3, -1) @@ -49,6 +51,9 @@ def test_operator_overloading(): v2 /= v1 assert str(v2) == "[2.000000, 8.000000]" + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + cstats = ConstructorStats.get(m.Vector2) assert cstats.alive() == 3 del v1 @@ -83,6 +88,7 @@ def test_operator_overloading(): assert cstats.move_assignments == 0 +@pytest.mark.xfail("env.GRAALPY", reason="TODO should get fixed on GraalPy side") def test_operators_notimplemented(): """#393: need to return NotSupported to ensure correct arithmetic operator behavior""" diff --git a/tests/test_pickling.py b/tests/test_pickling.py index ad67a1df98..3e2e453969 100644 --- a/tests/test_pickling.py +++ b/tests/test_pickling.py @@ -20,7 +20,7 @@ def test_pickle_simple_callable(): # all C Python versions. with pytest.raises(TypeError) as excinfo: pickle.dumps(m.simple_callable) - assert re.search("can.*t pickle .*PyCapsule.* object", str(excinfo.value)) + assert re.search("can.*t pickle .*[Cc]apsule.* object", str(excinfo.value)) @pytest.mark.parametrize("cls_name", ["Pickleable", "PickleableNew"]) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 19f65ce7eb..8df4cdd3f6 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -150,6 +150,18 @@ TEST_SUBMODULE(pytypes, m) { m.def("get_iterator", [] { return py::iterator(); }); // test_iterable m.def("get_iterable", [] { return py::iterable(); }); + m.def("get_first_item_from_iterable", [](const py::iterable &iter) { + // This tests the postfix increment operator + py::iterator it = iter.begin(); + py::iterator it2 = it++; + return *it2; + }); + m.def("get_second_item_from_iterable", [](const py::iterable &iter) { + // This tests the prefix increment operator + py::iterator it = iter.begin(); + ++it; + return *it; + }); m.def("get_frozenset_from_iterable", [](const py::iterable &iter) { return py::frozenset(iter); }); m.def("get_list_from_iterable", [](const py::iterable &iter) { return py::list(iter); }); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 6f015eec83..9fd24b34f1 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -52,6 +52,11 @@ def test_from_iterable(pytype, from_iter_func): def test_iterable(doc): assert doc(m.get_iterable) == "get_iterable() -> Iterable" + lins = [1, 2, 3] + i = m.get_first_item_from_iterable(lins) + assert i == 1 + i = m.get_second_item_from_iterable(lins) + assert i == 2 def test_float(doc): @@ -262,6 +267,7 @@ def __repr__(self): m.str_from_std_string_input, ], ) +@pytest.mark.xfail("env.GRAALPY", reason="TODO should be fixed on GraalPy side") def test_surrogate_pairs_unicode_error(func): input_str = "\ud83d\ude4f".encode("utf-8", "surrogatepass") with pytest.raises(UnicodeDecodeError): @@ -420,6 +426,7 @@ def test_accessor_moves(): pytest.skip("Not defined: PYBIND11_HANDLE_REF_DEBUG") +@pytest.mark.xfail("env.GRAALPY", reason="TODO should be fixed on GraalPy side") def test_constructors(): """C++ default and converting constructors are equivalent to type calls in Python""" types = [bytes, bytearray, str, bool, int, float, tuple, list, dict, set] @@ -712,6 +719,7 @@ def test_pass_bytes_or_unicode_to_string_types(): m.pass_to_pybind11_str(malformed_utf8) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") @pytest.mark.parametrize( ("create_weakref", "create_weakref_with_callback"), [ @@ -765,7 +773,10 @@ def callback(_): ob = C() # Should raise TypeError on CPython - with pytest.raises(TypeError) if not env.PYPY else contextlib.nullcontext(): + cm = pytest.raises(TypeError) + if env.PYPY or env.GRAALPY: + cm = contextlib.nullcontext() + with cm: _ = create_weakref(ob, callback) if has_callback else create_weakref(ob) diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index f609f553de..20abd29d59 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -3,6 +3,7 @@ import pytest from pytest import approx # noqa: PT013 +import env from pybind11_tests import ConstructorStats from pybind11_tests import sequences_and_iterators as m @@ -110,7 +111,8 @@ def test_sequence(): cstats = ConstructorStats.get(m.Sequence) s = m.Sequence(5) - assert cstats.values() == ["of size", "5"] + if not env.GRAALPY: + assert cstats.values() == ["of size", "5"] assert "Sequence" in repr(s) assert len(s) == 5 @@ -123,16 +125,19 @@ def test_sequence(): assert s[3] == approx(56.78, rel=1e-05) rev = reversed(s) - assert cstats.values() == ["of size", "5"] + if not env.GRAALPY: + assert cstats.values() == ["of size", "5"] rev2 = s[::-1] - assert cstats.values() == ["of size", "5"] + if not env.GRAALPY: + assert cstats.values() == ["of size", "5"] it = iter(m.Sequence(0)) for _ in range(3): # __next__ must continue to raise StopIteration with pytest.raises(StopIteration): next(it) - assert cstats.values() == ["of size", "0"] + if not env.GRAALPY: + assert cstats.values() == ["of size", "0"] expected = [0, 56.78, 0, 0, 12.34] assert rev == approx(expected, rel=1e-05) @@ -140,10 +145,14 @@ def test_sequence(): assert rev == rev2 rev[0::2] = m.Sequence([2.0, 2.0, 2.0]) - assert cstats.values() == ["of size", "3", "from std::vector"] + if not env.GRAALPY: + assert cstats.values() == ["of size", "3", "from std::vector"] assert rev == approx([2, 56.78, 2, 0, 2], rel=1e-05) + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + assert cstats.alive() == 4 del it assert cstats.alive() == 3 diff --git a/tests/test_smart_ptr.py b/tests/test_smart_ptr.py index bf0ae4aeb1..9729004ffc 100644 --- a/tests/test_smart_ptr.py +++ b/tests/test_smart_ptr.py @@ -2,10 +2,13 @@ import pytest +import env # noqa: F401 + m = pytest.importorskip("pybind11_tests.smart_ptr") from pybind11_tests import ConstructorStats # noqa: E402 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_smart_ptr(capture): # Object1 for i, o in enumerate( @@ -118,6 +121,7 @@ def test_smart_ptr_refcounting(): assert m.test_object1_refcounting() +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_unique_nodelete(): o = m.MyObject4(23) assert o.value == 23 @@ -129,6 +133,7 @@ def test_unique_nodelete(): assert cstats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_unique_nodelete4a(): o = m.MyObject4a(23) assert o.value == 23 @@ -140,6 +145,7 @@ def test_unique_nodelete4a(): assert cstats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_unique_deleter(): m.MyObject4a(0) o = m.MyObject4b(23) @@ -156,6 +162,7 @@ def test_unique_deleter(): assert cstats4b.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_large_holder(): o = m.MyObject5(5) assert o.value == 5 @@ -165,6 +172,7 @@ def test_large_holder(): assert cstats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_shared_ptr_and_references(): s = m.SharedPtrRef() stats = ConstructorStats.get(m.A) @@ -196,6 +204,7 @@ def test_shared_ptr_and_references(): assert stats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_shared_ptr_from_this_and_references(): s = m.SharedFromThisRef() stats = ConstructorStats.get(m.B) @@ -242,6 +251,7 @@ def test_shared_ptr_from_this_and_references(): assert y is z +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_move_only_holder(): a = m.TypeWithMoveOnlyHolder.make() b = m.TypeWithMoveOnlyHolder.make_as_object() @@ -253,6 +263,7 @@ def test_move_only_holder(): assert stats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_holder_with_addressof_operator(): # this test must not throw exception from c++ a = m.TypeForHolderWithAddressOf.make() @@ -283,6 +294,7 @@ def test_holder_with_addressof_operator(): assert stats.alive() == 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_move_only_holder_with_addressof_operator(): a = m.TypeForMoveOnlyHolderWithAddressOf.make() a.print_object() diff --git a/tests/test_stl.py b/tests/test_stl.py index 6f548789ed..d1a9ff08b0 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -2,6 +2,7 @@ import pytest +import env # noqa: F401 from pybind11_tests import ConstructorStats, UserType from pybind11_tests import stl as m @@ -362,6 +363,7 @@ def test_function_with_string_and_vector_string_arg(): assert m.func_with_string_or_vector_string_arg_overload("A") == 3 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_stl_ownership(): cstats = ConstructorStats.get(m.Placeholder) assert cstats.alive() == 0 diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index 09e784e2eb..9856ba462b 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -302,6 +302,25 @@ def test_map_delitem(): assert list(mm) == ["b"] assert list(mm.items()) == [("b", 2.5)] + with pytest.raises(KeyError) as excinfo: + mm["a_long_key"] + assert "a_long_key" in str(excinfo.value) + + with pytest.raises(KeyError) as excinfo: + del mm["a_long_key"] + assert "a_long_key" in str(excinfo.value) + + cut_length = 100 + k_very_long = "ab" * cut_length + "xyz" + with pytest.raises(KeyError) as excinfo: + mm[k_very_long] + assert k_very_long in str(excinfo.value) + k_very_long += "@" + with pytest.raises(KeyError) as excinfo: + mm[k_very_long] + k_repr = k_very_long[:cut_length] + "✄✄✄" + k_very_long[-cut_length:] + assert k_repr in str(excinfo.value) + um = m.UnorderedMapStringDouble() um["ua"] = 1.1 um["ub"] = 2.6 diff --git a/tests/test_type_caster_pyobject_ptr.cpp b/tests/test_type_caster_pyobject_ptr.cpp index a45c08b648..758c4ee09c 100644 --- a/tests/test_type_caster_pyobject_ptr.cpp +++ b/tests/test_type_caster_pyobject_ptr.cpp @@ -37,7 +37,8 @@ struct WithPyObjectPtrReturnTrampoline : WithPyObjectPtrReturn { std::string call_return_pyobject_ptr(const WithPyObjectPtrReturn *base_class_ptr) { PyObject *returned_obj = base_class_ptr->return_pyobject_ptr(); -#if !defined(PYPY_VERSION) // It is not worth the trouble doing something special for PyPy. +// It is not worth the trouble doing something special for PyPy/GraalPy +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) if (Py_REFCNT(returned_obj) != 1) { py::pybind11_fail(__FILE__ ":" PYBIND11_TOSTRING(__LINE__)); } diff --git a/tests/test_virtual_functions.py b/tests/test_virtual_functions.py index bc0177787a..617c87b8e0 100644 --- a/tests/test_virtual_functions.py +++ b/tests/test_virtual_functions.py @@ -4,7 +4,7 @@ import pytest -import env # noqa: F401 +import env m = pytest.importorskip("pybind11_tests.virtual_functions") from pybind11_tests import ConstructorStats # noqa: E402 @@ -82,6 +82,9 @@ def get_string2(self): """ ) + if env.GRAALPY: + pytest.skip("ConstructorStats is incompatible with GraalPy.") + cstats = ConstructorStats.get(m.ExampleVirt) assert cstats.alive() == 3 del ex12, ex12p, ex12p2 @@ -91,6 +94,7 @@ def get_string2(self): assert cstats.move_constructions >= 0 +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_alias_delay_initialization1(capture): """`A` only initializes its trampoline class when we inherit from it @@ -130,6 +134,7 @@ def f(self): ) +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") def test_alias_delay_initialization2(capture): """`A2`, unlike the above, is configured to always initialize the alias @@ -188,7 +193,7 @@ def f(self): # PyPy: Reference count > 1 causes call with noncopyable instance # to fail in ncv1.print_nc() -@pytest.mark.xfail("env.PYPY") +@pytest.mark.xfail("env.PYPY or env.GRAALPY") @pytest.mark.skipif( not hasattr(m, "NCVirt"), reason="NCVirt does not work on Intel/PGI/NVCC compilers" ) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index d3e938fab0..8bac211e14 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -274,10 +274,6 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::embed) endif() - if(MSVC) - target_link_libraries(${target_name} PRIVATE pybind11::windows_extras) - endif() - # -fvisibility=hidden is required to allow multiple modules compiled against # different pybind versions to work properly, and for some features (e.g. # py::module_local). We force it on everything inside the `pybind11`