From 1dd3f63f7c999c6c0ff4bb1d92497e8a87707c5d Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Wed, 26 May 2021 23:05:55 -0500 Subject: [PATCH 01/32] Fixed async-std parity with tokio when the rust future panics --- pytests/test_async_std_asyncio.rs | 23 +++++++++++++++++++++++ pytests/tokio_asyncio/mod.rs | 24 ++++++++++++++++++++++++ src/async_std.rs | 21 ++++++++++++--------- src/generic.rs | 10 ++++++---- src/tokio.rs | 2 +- 5 files changed, 66 insertions(+), 14 deletions(-) diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index f2b8cad..ae1e055 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -74,6 +74,29 @@ fn test_init_twice() -> PyResult<()> { common::test_init_twice() } +#[pyo3_asyncio::async_std::test] +async fn test_panic() -> PyResult<()> { + let fut = Python::with_gil(|py| -> PyResult<_> { + pyo3_asyncio::into_future( + pyo3_asyncio::async_std::into_coroutine(py, async { + panic!("this panic was intentional!") + })? + .as_ref(py), + ) + })?; + + match fut.await { + Ok(_) => panic!("coroutine should panic"), + Err(e) => Python::with_gil(|py| { + if e.is_instance::(py) { + Ok(()) + } else { + panic!("expected RustPanic err") + } + }), + } +} + #[pyo3_asyncio::async_std::main] async fn main() -> pyo3::PyResult<()> { pyo3_asyncio::testing::main().await diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index c1b95cf..8f00b70 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use futures::prelude::*; use pyo3::{prelude::*, wrap_pyfunction}; use crate::common; @@ -82,3 +83,26 @@ fn test_init_tokio_twice() -> PyResult<()> { Ok(()) } + +#[pyo3_asyncio::tokio::test] +async fn test_panic() -> PyResult<()> { + let fut = Python::with_gil(|py| -> PyResult<_> { + pyo3_asyncio::into_future( + pyo3_asyncio::tokio::into_coroutine(py, async { + panic!("this panic was intentional!") + })? + .as_ref(py), + ) + })?; + + match fut.await { + Ok(_) => panic!("coroutine should panic"), + Err(e) => Python::with_gil(|py| { + if e.is_instance::(py) { + Ok(()) + } else { + panic!("expected RustPanic err") + } + }), + } +} diff --git a/src/async_std.rs b/src/async_std.rs index 822b349..143d883 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -1,6 +1,7 @@ -use std::future::Future; +use std::{any::Any, future::Future, panic::AssertUnwindSafe}; use async_std::task; +use futures::prelude::*; use pyo3::prelude::*; use crate::generic::{self, JoinError, Runtime}; @@ -23,27 +24,29 @@ pub use pyo3_asyncio_macros::async_std_main as main; #[cfg(all(feature = "attributes", feature = "testing"))] pub use pyo3_asyncio_macros::async_std_test as test; -struct AsyncStdJoinError; +pub struct AsyncStdJoinErr(Box); -impl JoinError for AsyncStdJoinError { +impl JoinError for AsyncStdJoinErr { fn is_panic(&self) -> bool { - todo!() + true } } -struct AsyncStdRuntime; +pub struct AsyncStdRuntime; impl Runtime for AsyncStdRuntime { - type JoinError = AsyncStdJoinError; - type JoinHandle = task::JoinHandle>; + type JoinError = AsyncStdJoinErr; + type JoinHandle = task::JoinHandle>; fn spawn(fut: F) -> Self::JoinHandle where F: Future + Send + 'static, { task::spawn(async move { - fut.await; - Ok(()) + AssertUnwindSafe(fut) + .catch_unwind() + .await + .map_err(|e| AsyncStdJoinErr(e)) }) } } diff --git a/src/generic.rs b/src/generic.rs index 09526e0..4ba8072 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -1,6 +1,6 @@ use std::future::Future; -use pyo3::{exceptions::PyException, prelude::*}; +use pyo3::{create_exception, exceptions::PyException, prelude::*}; use crate::{dump_err, get_event_loop, CALL_SOON, CREATE_FUTURE, EXPECT_INIT}; @@ -126,6 +126,8 @@ fn set_result(py: Python, future: &PyAny, result: PyResult) -> PyResul Ok(()) } +create_exception!(pyo3_asyncio, RustPanic, PyException); + /// Convert a Rust Future into a Python coroutine with a generic runtime /// /// # Arguments @@ -198,8 +200,8 @@ where F: Future> + Send + 'static, { let future_rx = CREATE_FUTURE.get().expect(EXPECT_INIT).call0(py)?; - let future_tx1 = future_rx.clone(); - let future_tx2 = future_rx.clone(); + let future_tx1: PyObject = future_rx.clone(); + let future_tx2: PyObject = future_rx.clone(); R::spawn(async move { if let Err(e) = R::spawn(async move { @@ -222,7 +224,7 @@ where if set_result( py, future_tx2.as_ref(py), - Err(PyException::new_err("rust future panicked")), + Err(RustPanic::new_err("Rust future panicked")), ) .map_err(dump_err(py)) .is_err() diff --git a/src/tokio.rs b/src/tokio.rs index 1eb7c88..810d934 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -40,7 +40,7 @@ impl generic::JoinError for task::JoinError { } } -struct TokioRuntime; +pub struct TokioRuntime; impl generic::Runtime for TokioRuntime { type JoinError = task::JoinError; From 91896b82ab3d8b9c32f1d6c1a3b0cebcde4dbaad Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Wed, 26 May 2021 23:16:05 -0500 Subject: [PATCH 02/32] Fixed several warnings after async-std panic parity fix --- pytests/test_async_std_asyncio.rs | 2 +- pytests/tokio_asyncio/mod.rs | 3 +-- src/async_std.rs | 4 ++-- src/err.rs | 9 +++++++++ src/generic.rs | 6 ++---- src/lib.rs | 3 +++ src/tokio.rs | 2 +- 7 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 src/err.rs diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index ae1e055..9ae0926 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -88,7 +88,7 @@ async fn test_panic() -> PyResult<()> { match fut.await { Ok(_) => panic!("coroutine should panic"), Err(e) => Python::with_gil(|py| { - if e.is_instance::(py) { + if e.is_instance::(py) { Ok(()) } else { panic!("expected RustPanic err") diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index 8f00b70..91a6d23 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -1,6 +1,5 @@ use std::time::Duration; -use futures::prelude::*; use pyo3::{prelude::*, wrap_pyfunction}; use crate::common; @@ -98,7 +97,7 @@ async fn test_panic() -> PyResult<()> { match fut.await { Ok(_) => panic!("coroutine should panic"), Err(e) => Python::with_gil(|py| { - if e.is_instance::(py) { + if e.is_instance::(py) { Ok(()) } else { panic!("expected RustPanic err") diff --git a/src/async_std.rs b/src/async_std.rs index 143d883..f384e99 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -24,7 +24,7 @@ pub use pyo3_asyncio_macros::async_std_main as main; #[cfg(all(feature = "attributes", feature = "testing"))] pub use pyo3_asyncio_macros::async_std_test as test; -pub struct AsyncStdJoinErr(Box); +struct AsyncStdJoinErr(Box); impl JoinError for AsyncStdJoinErr { fn is_panic(&self) -> bool { @@ -32,7 +32,7 @@ impl JoinError for AsyncStdJoinErr { } } -pub struct AsyncStdRuntime; +struct AsyncStdRuntime; impl Runtime for AsyncStdRuntime { type JoinError = AsyncStdJoinErr; diff --git a/src/err.rs b/src/err.rs new file mode 100644 index 0000000..95e5d15 --- /dev/null +++ b/src/err.rs @@ -0,0 +1,9 @@ +// FIXME - is there a way to document custom PyO3 exceptions? +#[allow(missing_docs)] +mod exceptions { + use pyo3::{create_exception, exceptions::PyException}; + + create_exception!(pyo3_asyncio, RustPanic, PyException); +} + +pub use exceptions::RustPanic; diff --git a/src/generic.rs b/src/generic.rs index 4ba8072..d91bb58 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -1,8 +1,8 @@ use std::future::Future; -use pyo3::{create_exception, exceptions::PyException, prelude::*}; +use pyo3::prelude::*; -use crate::{dump_err, get_event_loop, CALL_SOON, CREATE_FUTURE, EXPECT_INIT}; +use crate::{dump_err, err::RustPanic, get_event_loop, CALL_SOON, CREATE_FUTURE, EXPECT_INIT}; /// Generic utilities for a JoinError pub trait JoinError { @@ -126,8 +126,6 @@ fn set_result(py: Python, future: &PyAny, result: PyResult) -> PyResul Ok(()) } -create_exception!(pyo3_asyncio, RustPanic, PyException); - /// Convert a Rust Future into a Python coroutine with a generic runtime /// /// # Arguments diff --git a/src/lib.rs b/src/lib.rs index 6e16100..33eee1c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -103,6 +103,9 @@ pub mod async_std; #[doc(inline)] pub mod tokio; +/// Errors and exceptions related to PyO3 Asyncio +pub mod err; + /// Generic implementations of PyO3 Asyncio utilities that can be used for any Rust runtime pub mod generic; diff --git a/src/tokio.rs b/src/tokio.rs index 810d934..1eb7c88 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -40,7 +40,7 @@ impl generic::JoinError for task::JoinError { } } -pub struct TokioRuntime; +struct TokioRuntime; impl generic::Runtime for TokioRuntime { type JoinError = task::JoinError; From bc0f5e87b54332dd4df5003f0d6f0cc4a83eb132 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Fri, 2 Jul 2021 13:44:22 -0500 Subject: [PATCH 03/32] Changed conversions to accept any event loop, library no longer caches single event loop --- README.md | 14 ++- examples/async_std.rs | 7 +- examples/tokio.rs | 5 +- examples/tokio_current_thread.rs | 5 +- examples/tokio_multi_thread.rs | 5 +- pyo3-asyncio-macros/src/lib.rs | 10 +- pytests/common/mod.rs | 13 ++- pytests/test_async_std_asyncio.rs | 33 ++++--- pytests/test_async_std_run_forever.rs | 11 ++- pytests/tokio_asyncio/mod.rs | 35 ++++--- pytests/tokio_run_forever/mod.rs | 11 ++- src/async_std.rs | 86 +++++++++++++---- src/generic.rs | 126 +++++++++++++++++------- src/lib.rs | 132 ++++++++++++++------------ src/testing.rs | 12 +-- src/tokio.rs | 98 +++++++++++++++---- 16 files changed, 420 insertions(+), 183 deletions(-) diff --git a/README.md b/README.md index 4102f2f..b8310f1 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,10 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) + pyo3_asyncio::into_future( + pyo3_asyncio::async_std::task_event_loop().unwrap().as_ref(py), + asyncio.call_method1("sleep", (1.into_py(py),))? + ) })?; fut.await?; @@ -77,7 +80,10 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) + pyo3_asyncio::into_future( + pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), + asyncio.call_method1("sleep", (1.into_py(py),))? + ) })?; fut.await?; @@ -127,7 +133,7 @@ use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] fn rust_sleep(py: Python) -> PyResult { - pyo3_asyncio::async_std::into_coroutine(py, async { + pyo3_asyncio::async_std::into_coroutine(pyo3_asyncio::get_event_loop(py)?, async { async_std::task::sleep(std::time::Duration::from_secs(1)).await; Ok(Python::with_gil(|py| py.None())) }) @@ -153,7 +159,7 @@ use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] fn rust_sleep(py: Python) -> PyResult { - pyo3_asyncio::tokio::into_coroutine(py, async { + pyo3_asyncio::tokio::into_coroutine(pyo3_asyncio::get_event_loop(py)?, async { tokio::time::sleep(std::time::Duration::from_secs(1)).await; Ok(Python::with_gil(|py| py.None())) }) diff --git a/examples/async_std.rs b/examples/async_std.rs index 35d72f3..75f17d5 100644 --- a/examples/async_std.rs +++ b/examples/async_std.rs @@ -6,7 +6,12 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) + pyo3_asyncio::into_future( + pyo3_asyncio::async_std::task_event_loop() + .unwrap() + .as_ref(py), + asyncio.call_method1("sleep", (1.into_py(py),))?, + ) })?; println!("sleeping for 1s"); diff --git a/examples/tokio.rs b/examples/tokio.rs index 8f5f49a..a4b6981 100644 --- a/examples/tokio.rs +++ b/examples/tokio.rs @@ -6,7 +6,10 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) + pyo3_asyncio::into_future( + pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), + asyncio.call_method1("sleep", (1.into_py(py),))?, + ) })?; println!("sleeping for 1s"); diff --git a/examples/tokio_current_thread.rs b/examples/tokio_current_thread.rs index 84b56a8..435e602 100644 --- a/examples/tokio_current_thread.rs +++ b/examples/tokio_current_thread.rs @@ -6,7 +6,10 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) + pyo3_asyncio::into_future( + pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), + asyncio.call_method1("sleep", (1.into_py(py),))?, + ) })?; println!("sleeping for 1s"); diff --git a/examples/tokio_multi_thread.rs b/examples/tokio_multi_thread.rs index 82556f0..2ff2d59 100644 --- a/examples/tokio_multi_thread.rs +++ b/examples/tokio_multi_thread.rs @@ -6,7 +6,10 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) + pyo3_asyncio::into_future( + pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), + asyncio.call_method1("sleep", (1.into_py(py),))?, + ) })?; println!("sleeping for 1s"); diff --git a/pyo3-asyncio-macros/src/lib.rs b/pyo3-asyncio-macros/src/lib.rs index 5f02273..fd4498a 100644 --- a/pyo3-asyncio-macros/src/lib.rs +++ b/pyo3-asyncio-macros/src/lib.rs @@ -146,8 +146,10 @@ pub fn async_std_test(_attr: TokenStream, item: TokenStream) -> TokenStream { #body } + let event_loop = pyo3_asyncio::async_std::task_event_loop().unwrap(); + Box::pin(pyo3_asyncio::async_std::re_exports::spawn_blocking(move || { - #name() + #name(event_loop) })) } } @@ -222,8 +224,10 @@ pub fn tokio_test(_attr: TokenStream, item: TokenStream) -> TokenStream { #body } - Box::pin(async { - match pyo3_asyncio::tokio::get_runtime().spawn_blocking(&#name).await { + let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); + + Box::pin(async move { + match pyo3_asyncio::tokio::get_runtime().spawn_blocking(move || #name(event_loop)).await { Ok(result) => result, Err(e) => { assert!(e.is_panic()); diff --git a/pytests/common/mod.rs b/pytests/common/mod.rs index c4ffb48..2cc6791 100644 --- a/pytests/common/mod.rs +++ b/pytests/common/mod.rs @@ -12,12 +12,15 @@ async def sleep_for_1s(sleep_for): await sleep_for(1) "#; -pub(super) async fn test_into_future() -> PyResult<()> { +pub(super) async fn test_into_future(event_loop: PyObject) -> PyResult<()> { let fut = Python::with_gil(|py| { let test_mod = PyModule::from_code(py, TEST_MOD, "test_rust_coroutine/test_mod.py", "test_mod")?; - pyo3_asyncio::into_future(test_mod.call_method1("py_sleep", (1.into_py(py),))?) + pyo3_asyncio::into_future( + event_loop.as_ref(py), + test_mod.call_method1("py_sleep", (1.into_py(py),))?, + ) })?; fut.await?; @@ -30,13 +33,13 @@ pub(super) fn test_blocking_sleep() -> PyResult<()> { Ok(()) } -pub(super) async fn test_other_awaitables() -> PyResult<()> { +pub(super) async fn test_other_awaitables(event_loop: PyObject) -> PyResult<()> { let fut = Python::with_gil(|py| { let functools = py.import("functools")?; let time = py.import("time")?; // spawn a blocking sleep in the threadpool executor - returns a task, not a coroutine - let task = pyo3_asyncio::get_event_loop(py).call_method1( + let task = event_loop.as_ref(py).call_method1( "run_in_executor", ( py.None(), @@ -44,7 +47,7 @@ pub(super) async fn test_other_awaitables() -> PyResult<()> { ), )?; - pyo3_asyncio::into_future(task) + pyo3_asyncio::into_future(event_loop.as_ref(py), task) })?; fut.await?; diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index 96ac0cc..193c567 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -9,7 +9,7 @@ use pyo3::{prelude::*, wrap_pyfunction}; fn sleep_for(py: Python, secs: &PyAny) -> PyResult { let secs = secs.extract()?; - pyo3_asyncio::async_std::into_coroutine(py, async move { + pyo3_asyncio::async_std::into_coroutine(pyo3_asyncio::get_event_loop(py)?, async move { task::sleep(Duration::from_secs(secs)).await; Python::with_gil(|py| Ok(py.None())) }) @@ -30,6 +30,9 @@ async fn test_into_coroutine() -> PyResult<()> { )?; pyo3_asyncio::into_future( + pyo3_asyncio::async_std::task_event_loop() + .unwrap() + .as_ref(py), test_mod.call_method1("sleep_for_1s", (sleeper_mod.getattr("sleep_for")?,))?, ) })?; @@ -47,7 +50,12 @@ async fn test_async_sleep() -> PyResult<()> { task::sleep(Duration::from_secs(1)).await; Python::with_gil(|py| { - pyo3_asyncio::into_future(asyncio.as_ref(py).call_method1("sleep", (1.0,))?) + pyo3_asyncio::into_future( + pyo3_asyncio::async_std::task_event_loop() + .unwrap() + .as_ref(py), + asyncio.as_ref(py).call_method1("sleep", (1.0,))?, + ) })? .await?; @@ -55,22 +63,22 @@ async fn test_async_sleep() -> PyResult<()> { } #[pyo3_asyncio::async_std::test] -fn test_blocking_sleep() -> PyResult<()> { +fn test_blocking_sleep(_event_loop: PyObject) -> PyResult<()> { common::test_blocking_sleep() } #[pyo3_asyncio::async_std::test] async fn test_into_future() -> PyResult<()> { - common::test_into_future().await + common::test_into_future(pyo3_asyncio::async_std::task_event_loop().unwrap()).await } #[pyo3_asyncio::async_std::test] async fn test_other_awaitables() -> PyResult<()> { - common::test_other_awaitables().await + common::test_other_awaitables(pyo3_asyncio::async_std::task_event_loop().unwrap()).await } #[pyo3_asyncio::async_std::test] -fn test_init_twice() -> PyResult<()> { +fn test_init_twice(_event_loop: PyObject) -> PyResult<()> { common::test_init_twice() } @@ -84,12 +92,15 @@ async fn test_local_coroutine() -> PyResult<()> { Python::with_gil(|py| { let non_send_secs = Rc::new(1); - let py_future = pyo3_asyncio::async_std::local_future_into_py(py, async move { - async_std::task::sleep(Duration::from_secs(*non_send_secs)).await; - Ok(Python::with_gil(|py| py.None())) - })?; + let event_loop = pyo3_asyncio::async_std::task_event_loop().unwrap(); + + let py_future = + pyo3_asyncio::async_std::local_future_into_py(event_loop.as_ref(py), async move { + async_std::task::sleep(Duration::from_secs(*non_send_secs)).await; + Ok(Python::with_gil(|py| py.None())) + })?; - pyo3_asyncio::into_future(py_future) + pyo3_asyncio::into_future(event_loop.as_ref(py), py_future.as_ref(py)) })? .await?; diff --git a/pytests/test_async_std_run_forever.rs b/pytests/test_async_std_run_forever.rs index dd2d620..ce7206d 100644 --- a/pytests/test_async_std_run_forever.rs +++ b/pytests/test_async_std_run_forever.rs @@ -13,16 +13,21 @@ fn dump_err(py: Python<'_>) -> impl FnOnce(PyErr) + '_ { fn main() { Python::with_gil(|py| { pyo3_asyncio::with_runtime(py, || { + let event_loop = PyObject::from(pyo3_asyncio::get_event_loop(py).unwrap()); + async_std::task::spawn(async move { async_std::task::sleep(Duration::from_secs(1)).await; Python::with_gil(|py| { - let event_loop = pyo3_asyncio::get_event_loop(py); - event_loop + .as_ref(py) .call_method1( "call_soon_threadsafe", - (event_loop.getattr("stop").map_err(dump_err(py)).unwrap(),), + (event_loop + .as_ref(py) + .getattr("stop") + .map_err(dump_err(py)) + .unwrap(),), ) .map_err(dump_err(py)) .unwrap(); diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index 8953d23..7845d76 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -8,7 +8,7 @@ use crate::common; fn sleep_for(py: Python, secs: &PyAny) -> PyResult { let secs = secs.extract()?; - pyo3_asyncio::tokio::into_coroutine(py, async move { + pyo3_asyncio::tokio::into_coroutine(pyo3_asyncio::get_event_loop(py)?, async move { tokio::time::sleep(Duration::from_secs(secs)).await; Python::with_gil(|py| Ok(py.None())) }) @@ -16,6 +16,8 @@ fn sleep_for(py: Python, secs: &PyAny) -> PyResult { #[pyo3_asyncio::tokio::test] async fn test_into_coroutine() -> PyResult<()> { + let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); + let fut = Python::with_gil(|py| { let sleeper_mod = PyModule::new(py, "rust_sleeper")?; @@ -29,6 +31,7 @@ async fn test_into_coroutine() -> PyResult<()> { )?; pyo3_asyncio::into_future( + event_loop.as_ref(py), test_mod.call_method1("sleep_for_1s", (sleeper_mod.getattr("sleep_for")?,))?, ) })?; @@ -40,13 +43,18 @@ async fn test_into_coroutine() -> PyResult<()> { #[pyo3_asyncio::tokio::test] async fn test_async_sleep() -> PyResult<()> { + let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); + let asyncio = Python::with_gil(|py| py.import("asyncio").map(|asyncio| PyObject::from(asyncio)))?; tokio::time::sleep(Duration::from_secs(1)).await; Python::with_gil(|py| { - pyo3_asyncio::into_future(asyncio.as_ref(py).call_method1("sleep", (1.0,))?) + pyo3_asyncio::into_future( + event_loop.as_ref(py), + asyncio.as_ref(py).call_method1("sleep", (1.0,))?, + ) })? .await?; @@ -54,27 +62,27 @@ async fn test_async_sleep() -> PyResult<()> { } #[pyo3_asyncio::tokio::test] -fn test_blocking_sleep() -> PyResult<()> { +fn test_blocking_sleep(_event_loop: PyObject) -> PyResult<()> { common::test_blocking_sleep() } #[pyo3_asyncio::tokio::test] async fn test_into_future() -> PyResult<()> { - common::test_into_future().await + common::test_into_future(pyo3_asyncio::tokio::task_event_loop().unwrap()).await } #[pyo3_asyncio::tokio::test] async fn test_other_awaitables() -> PyResult<()> { - common::test_other_awaitables().await + common::test_other_awaitables(pyo3_asyncio::tokio::task_event_loop().unwrap()).await } #[pyo3_asyncio::tokio::test] -fn test_init_twice() -> PyResult<()> { +fn test_init_twice(_event_loop: PyObject) -> PyResult<()> { common::test_init_twice() } #[pyo3_asyncio::tokio::test] -fn test_init_tokio_twice() -> PyResult<()> { +fn test_init_tokio_twice(_event_loop: PyObject) -> PyResult<()> { // tokio has already been initialized in test main. call these functions to // make sure they don't cause problems with the other tests. pyo3_asyncio::tokio::init_multi_thread_once(); @@ -84,17 +92,18 @@ fn test_init_tokio_twice() -> PyResult<()> { } #[pyo3_asyncio::tokio::test] -fn test_local_set_coroutine() -> PyResult<()> { +fn test_local_set_coroutine(event_loop: PyObject) -> PyResult<()> { tokio::task::LocalSet::new().block_on(pyo3_asyncio::tokio::get_runtime(), async { Python::with_gil(|py| { let non_send_secs = Rc::new(1); - let py_future = pyo3_asyncio::tokio::local_future_into_py(py, async move { - tokio::time::sleep(Duration::from_secs(*non_send_secs)).await; - Ok(Python::with_gil(|py| py.None())) - })?; + let py_future = + pyo3_asyncio::tokio::local_future_into_py(event_loop.as_ref(py), async move { + tokio::time::sleep(Duration::from_secs(*non_send_secs)).await; + Ok(Python::with_gil(|py| py.None())) + })?; - pyo3_asyncio::into_future(py_future) + pyo3_asyncio::into_future(event_loop.as_ref(py), py_future.as_ref(py)) })? .await?; diff --git a/pytests/tokio_run_forever/mod.rs b/pytests/tokio_run_forever/mod.rs index 28fda8c..60f79fe 100644 --- a/pytests/tokio_run_forever/mod.rs +++ b/pytests/tokio_run_forever/mod.rs @@ -13,16 +13,21 @@ fn dump_err(py: Python<'_>) -> impl FnOnce(PyErr) + '_ { pub(super) fn test_main() { Python::with_gil(|py| { pyo3_asyncio::with_runtime(py, || { + let event_loop = PyObject::from(pyo3_asyncio::get_event_loop(py).unwrap()); + pyo3_asyncio::tokio::get_runtime().spawn(async move { tokio::time::sleep(Duration::from_secs(1)).await; Python::with_gil(|py| { - let event_loop = pyo3_asyncio::get_event_loop(py); - event_loop + .as_ref(py) .call_method1( "call_soon_threadsafe", - (event_loop.getattr("stop").map_err(dump_err(py)).unwrap(),), + (event_loop + .as_ref(py) + .getattr("stop") + .map_err(dump_err(py)) + .unwrap(),), ) .map_err(dump_err(py)) .unwrap(); diff --git a/src/async_std.rs b/src/async_std.rs index 6459031..3ddb061 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -1,6 +1,7 @@ -use std::future::Future; +use std::{future::Future, pin::Pin}; use async_std::task; +use once_cell::unsync::OnceCell; use pyo3::prelude::*; use crate::generic::{self, JoinError, Runtime, SpawnLocalExt}; @@ -31,12 +32,27 @@ impl JoinError for AsyncStdJoinError { } } +async_std::task_local! { + static EVENT_LOOP: OnceCell = OnceCell::new() +} + struct AsyncStdRuntime; impl Runtime for AsyncStdRuntime { type JoinError = AsyncStdJoinError; type JoinHandle = task::JoinHandle>; + fn scope(event_loop: PyObject, fut: F) -> Pin + Send>> + where + F: Future + Send + 'static, + { + EVENT_LOOP.with(|c| c.set(event_loop).unwrap()); + Box::pin(fut) + } + fn get_task_event_loop() -> Option { + EVENT_LOOP.with(|c| c.get().map(|event_loop| event_loop.clone())) + } + fn spawn(fut: F) -> Self::JoinHandle where F: Future + Send + 'static, @@ -49,6 +65,14 @@ impl Runtime for AsyncStdRuntime { } impl SpawnLocalExt for AsyncStdRuntime { + fn scope_local(event_loop: PyObject, fut: F) -> Pin>> + where + F: Future + 'static, + { + EVENT_LOOP.with(|c| c.set(event_loop).unwrap()); + Box::pin(fut) + } + fn spawn_local(fut: F) -> Self::JoinHandle where F: Future + 'static, @@ -60,6 +84,25 @@ impl SpawnLocalExt for AsyncStdRuntime { } } +pub async fn scope(event_loop: PyObject, fut: F) -> R +where + F: Future + Send + 'static, +{ + AsyncStdRuntime::scope(event_loop, fut).await +} + +pub async fn scope_local(event_loop: PyObject, fut: F) -> R +where + F: Future + 'static, +{ + AsyncStdRuntime::scope_local(event_loop, fut).await +} + +/// Get the task local event loop for the current async_std task +pub fn task_event_loop() -> Option { + AsyncStdRuntime::get_task_event_loop() +} + /// Run the event loop until the given Future completes /// /// The event loop runs until the given future is complete. @@ -117,17 +160,20 @@ where /// fn sleep_for(py: Python, secs: &PyAny) -> PyResult { /// let secs = secs.extract()?; /// -/// pyo3_asyncio::async_std::into_coroutine(py, async move { -/// async_std::task::sleep(Duration::from_secs(secs)).await; -/// Python::with_gil(|py| Ok(py.None())) -/// }) +/// pyo3_asyncio::async_std::into_coroutine( +/// pyo3_asyncio::get_event_loop(py)?, +/// async move { +/// async_std::task::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// } +/// ) /// } /// ``` -pub fn into_coroutine(py: Python, fut: F) -> PyResult +pub fn into_coroutine(event_loop: &PyAny, fut: F) -> PyResult where F: Future> + Send + 'static, { - generic::into_coroutine::(py, fut) + generic::into_coroutine::(event_loop, fut) } /// Convert a `!Send` Rust Future into a Python awaitable @@ -145,22 +191,28 @@ where /// /// /// Awaitable non-send sleep function /// #[pyfunction] -/// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { -/// // Rc is non-send so it cannot be passed into pyo3_asyncio::tokio::into_coroutine +/// fn sleep_for(py: Python, secs: u64) -> PyResult { +/// // Rc is non-send so it cannot be passed into pyo3_asyncio::async_std::into_coroutine /// let secs = Rc::new(secs); /// -/// pyo3_asyncio::async_std::local_future_into_py(py, async move { -/// async_std::task::sleep(Duration::from_secs(*secs)).await; -/// Python::with_gil(|py| Ok(py.None())) -/// }) +/// pyo3_asyncio::async_std::local_future_into_py( +/// pyo3_asyncio::async_std::task_event_loop().unwrap().as_ref(py), +/// async move { +/// async_std::task::sleep(Duration::from_secs(*secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// } +/// ) /// } /// /// # #[cfg(all(feature = "async-std-runtime", feature = "attributes"))] /// #[pyo3_asyncio::async_std::main] /// async fn main() -> PyResult<()> { /// Python::with_gil(|py| { -/// let py_future = sleep_for(py, 1)?; -/// pyo3_asyncio::into_future(py_future) +/// let py_future = sleep_for(py, 1)?; +/// pyo3_asyncio::into_future( +/// pyo3_asyncio::async_std::task_event_loop().unwrap().as_ref(py), +/// py_future.as_ref(py) +/// ) /// })? /// .await?; /// @@ -169,9 +221,9 @@ where /// # #[cfg(not(all(feature = "async-std-runtime", feature = "attributes")))] /// # fn main() {} /// ``` -pub fn local_future_into_py(py: Python, fut: F) -> PyResult<&PyAny> +pub fn local_future_into_py(event_loop: &PyAny, fut: F) -> PyResult where F: Future> + 'static, { - generic::local_future_into_py::(py, fut) + generic::local_future_into_py::(event_loop, fut) } diff --git a/src/generic.rs b/src/generic.rs index e40126b..5b73c33 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -1,8 +1,8 @@ -use std::future::Future; +use std::{future::Future, pin::Pin}; use pyo3::{exceptions::PyException, prelude::*}; -use crate::{dump_err, get_event_loop, CALL_SOON, CREATE_FUTURE, EXPECT_INIT}; +use crate::{call_soon_threadsafe, create_future, dump_err, get_event_loop}; /// Generic utilities for a JoinError pub trait JoinError { @@ -17,6 +17,13 @@ pub trait Runtime { /// A future that completes with the result of the spawned task type JoinHandle: Future> + Send; + /// Set the task local event loop for the given future + fn scope(event_loop: PyObject, fut: F) -> Pin + Send>> + where + F: Future + Send + 'static; + /// Get the task local event loop for the current task + fn get_task_event_loop() -> Option; + /// Spawn a future onto this runtime's event loop fn spawn(fut: F) -> Self::JoinHandle where @@ -25,6 +32,11 @@ pub trait Runtime { /// Extension trait for async/await runtimes that support spawning local tasks pub trait SpawnLocalExt: Runtime { + /// Set the task local event loop for the given !Send future + fn scope_local(event_loop: PyObject, fut: F) -> Pin>> + where + F: Future + 'static; + /// Spawn a !Send future onto this runtime's event loop fn spawn_local(fut: F) -> Self::JoinHandle where @@ -70,6 +82,16 @@ pub trait SpawnLocalExt: Runtime { /// # impl Runtime for MyCustomRuntime { /// # type JoinError = MyCustomJoinError; /// # type JoinHandle = MyCustomJoinHandle; +/// # +/// # fn scope(_event_loop: PyObject, fut: F) -> Pin + Send>> +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # fn get_task_event_loop() -> Option { +/// # unreachable!() +/// # } /// # /// # fn spawn(fut: F) -> Self::JoinHandle /// # where @@ -103,31 +125,27 @@ where R: Runtime, F: Future> + Send + 'static, { - let coro = into_coroutine::(py, async move { + let event_loop = get_event_loop(py)?; + + let coro = into_coroutine::(event_loop, async move { fut.await?; Ok(Python::with_gil(|py| py.None())) })?; - get_event_loop(py).call_method1("run_until_complete", (coro,))?; + event_loop.call_method1("run_until_complete", (coro,))?; Ok(()) } -fn set_result(py: Python, future: &PyAny, result: PyResult) -> PyResult<()> { +fn set_result(event_loop: &PyAny, future: &PyAny, result: PyResult) -> PyResult<()> { match result { Ok(val) => { let set_result = future.getattr("set_result")?; - CALL_SOON - .get() - .expect(EXPECT_INIT) - .call1(py, (set_result, val))?; + call_soon_threadsafe(event_loop, (set_result, val))?; } Err(err) => { let set_exception = future.getattr("set_exception")?; - CALL_SOON - .get() - .expect(EXPECT_INIT) - .call1(py, (set_exception, err))?; + call_soon_threadsafe(event_loop, (set_exception, err))?; } } @@ -176,6 +194,16 @@ fn set_result(py: Python, future: &PyAny, result: PyResult) -> PyResul /// # impl Runtime for MyCustomRuntime { /// # type JoinError = MyCustomJoinError; /// # type JoinHandle = MyCustomJoinHandle; +/// # +/// # fn scope(_event_loop: PyObject, fut: F) -> Pin + Send>> +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # fn get_task_event_loop() -> Option { +/// # unreachable!() +/// # } /// # /// # fn spawn(fut: F) -> Self::JoinHandle /// # where @@ -194,27 +222,34 @@ fn set_result(py: Python, future: &PyAny, result: PyResult) -> PyResul /// fn sleep_for(py: Python, secs: &PyAny) -> PyResult { /// let secs = secs.extract()?; /// -/// pyo3_asyncio::generic::into_coroutine::(py, async move { -/// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; -/// Python::with_gil(|py| Ok(py.None())) -/// }) +/// pyo3_asyncio::generic::into_coroutine::( +/// pyo3_asyncio::get_event_loop(py)?, +/// async move { +/// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// } +/// ) /// } /// ``` -pub fn into_coroutine(py: Python, fut: F) -> PyResult +pub fn into_coroutine(event_loop: &PyAny, fut: F) -> PyResult where R: Runtime, F: Future> + Send + 'static, { - let future_rx = CREATE_FUTURE.get().expect(EXPECT_INIT).call0(py)?; + let future_rx: PyObject = create_future(event_loop)?.into(); let future_tx1 = future_rx.clone(); let future_tx2 = future_rx.clone(); + let event_loop = PyObject::from(event_loop); + R::spawn(async move { + let event_loop2 = event_loop.clone(); + if let Err(e) = R::spawn(async move { - let result = fut.await; + let result = R::scope(event_loop2.clone(), fut).await; Python::with_gil(move |py| { - if set_result(py, future_tx1.as_ref(py), result) + if set_result(event_loop2.as_ref(py), future_tx1.as_ref(py), result) .map_err(dump_err(py)) .is_err() { @@ -228,7 +263,7 @@ where if e.is_panic() { Python::with_gil(move |py| { if set_result( - py, + event_loop.as_ref(py), future_tx2.as_ref(py), Err(PyException::new_err("rust future panicked")), ) @@ -287,6 +322,16 @@ where /// # impl Runtime for MyCustomRuntime { /// # type JoinError = MyCustomJoinError; /// # type JoinHandle = MyCustomJoinHandle; +/// # +/// # fn scope(_event_loop: PyObject, fut: F) -> Pin + Send>> +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # fn get_task_event_loop() -> Option { +/// # unreachable!() +/// # } /// # /// # fn spawn(fut: F) -> Self::JoinHandle /// # where @@ -297,6 +342,13 @@ where /// # } /// # /// # impl SpawnLocalExt for MyCustomRuntime { +/// # fn scope_local(_event_loop: PyObject, fut: F) -> Pin>> +/// # where +/// # F: Future + 'static +/// # { +/// # unreachable!() +/// # } +/// # /// # fn spawn_local(fut: F) -> Self::JoinHandle /// # where /// # F: Future + 'static @@ -311,32 +363,38 @@ where /// /// /// Awaitable sleep function /// #[pyfunction] -/// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { -/// pyo3_asyncio::generic::local_future_into_py::(py, async move { -/// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; -/// Python::with_gil(|py| Ok(py.None())) -/// }) +/// fn sleep_for(py: Python, secs: u64) -> PyResult { +/// pyo3_asyncio::generic::local_future_into_py::( +/// pyo3_asyncio::get_event_loop(py)?, +/// async move { +/// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// } +/// ) /// } /// ``` -pub fn local_future_into_py(py: Python, fut: F) -> PyResult<&PyAny> +pub fn local_future_into_py(event_loop: &PyAny, fut: F) -> PyResult where R: SpawnLocalExt, F: Future> + 'static, { - let future_rx = CREATE_FUTURE.get().expect(EXPECT_INIT).as_ref(py).call0()?; - let future_tx1 = PyObject::from(future_rx); + let future_rx = PyObject::from(create_future(event_loop)?); + let future_tx1 = future_rx.clone(); let future_tx2 = future_tx1.clone(); + let event_loop = PyObject::from(event_loop); + R::spawn_local(async move { + let event_loop2 = event_loop.clone(); + if let Err(e) = R::spawn_local(async move { - let result = fut.await; + let result = R::scope_local(event_loop2.clone(), fut).await; Python::with_gil(move |py| { - if set_result(py, future_tx1.as_ref(py), result) + if set_result(event_loop2.as_ref(py), future_tx1.as_ref(py), result) .map_err(dump_err(py)) .is_err() { - // Cancelled } }); @@ -346,7 +404,7 @@ where if e.is_panic() { Python::with_gil(move |py| { if set_result( - py, + event_loop.as_ref(py), future_tx2.as_ref(py), Err(PyException::new_err("rust future panicked")), ) diff --git a/src/lib.rs b/src/lib.rs index 26768cd..8fad0b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -110,7 +110,7 @@ use std::future::Future; use futures::channel::oneshot; use once_cell::sync::OnceCell; -use pyo3::{exceptions::PyKeyboardInterrupt, prelude::*, PyNativeType}; +use pyo3::{exceptions::PyKeyboardInterrupt, prelude::*, types::PyTuple}; /// Re-exported for #[test] attributes #[cfg(all(feature = "attributes", feature = "testing"))] @@ -146,13 +146,19 @@ const EXPECT_INIT: &str = "PyO3 Asyncio has not been initialized"; static ASYNCIO: OnceCell = OnceCell::new(); static ENSURE_FUTURE: OnceCell = OnceCell::new(); -static EVENT_LOOP: OnceCell = OnceCell::new(); static EXECUTOR: OnceCell = OnceCell::new(); -static CALL_SOON: OnceCell = OnceCell::new(); -static CREATE_FUTURE: OnceCell = OnceCell::new(); -fn ensure_future(py: Python) -> &PyAny { - ENSURE_FUTURE.get().expect(EXPECT_INIT).as_ref(py) +fn ensure_future<'p>(py: Python<'p>, awaitable: &'p PyAny) -> PyResult<&'p PyAny> { + ENSURE_FUTURE + .get_or_try_init(|| -> PyResult { + Ok(asyncio(py)?.getattr("ensure_future")?.into()) + })? + .as_ref(py) + .call1((awaitable,)) +} + +fn create_future<'p>(event_loop: &'p PyAny) -> PyResult<&'p PyAny> { + event_loop.call_method0("create_future") } #[allow(clippy::needless_doctest_main)] @@ -201,32 +207,31 @@ where /// - Calling `try_init` a second time returns `Ok(())` and does nothing. /// > In future versions this may return an `Err`. pub fn try_init(py: Python) -> PyResult<()> { - EVENT_LOOP.get_or_try_init(|| -> PyResult { + ASYNCIO.get_or_try_init(|| -> PyResult { let asyncio = py.import("asyncio")?; - let ensure_future = asyncio.getattr("ensure_future")?; let event_loop = asyncio.call_method0("get_event_loop")?; let executor = py .import("concurrent.futures.thread")? .getattr("ThreadPoolExecutor")? .call0()?; event_loop.call_method1("set_default_executor", (executor,))?; - let call_soon = event_loop.getattr("call_soon_threadsafe")?; - let create_future = event_loop.getattr("create_future")?; - ASYNCIO.get_or_init(|| asyncio.into()); - ENSURE_FUTURE.get_or_init(|| ensure_future.into()); EXECUTOR.get_or_init(|| executor.into()); - CALL_SOON.get_or_init(|| call_soon.into()); - CREATE_FUTURE.get_or_init(|| create_future.into()); - Ok(event_loop.into()) + Ok(asyncio.into()) })?; Ok(()) } +fn asyncio(py: Python) -> PyResult<&PyAny> { + ASYNCIO + .get_or_try_init(|| Ok(py.import("asyncio")?.into())) + .map(|asyncio| asyncio.as_ref(py)) +} + /// Get a reference to the Python Event Loop from Rust -pub fn get_event_loop(py: Python) -> &PyAny { - EVENT_LOOP.get().expect(EXPECT_INIT).as_ref(py) +pub fn get_event_loop(py: Python) -> PyResult<&PyAny> { + asyncio(py)?.call_method0("get_event_loop") } /// Run the event loop forever @@ -243,40 +248,44 @@ pub fn get_event_loop(py: Python) -> &PyAny { /// # Examples /// /// ``` -/// # use std::time::Duration; -/// # use pyo3::prelude::*; -/// # Python::with_gil(|py| { -/// # pyo3_asyncio::with_runtime(py, || { -/// // Wait 1 second, then stop the event loop /// # #[cfg(feature = "async-std-runtime")] -/// async_std::task::spawn(async move { -/// async_std::task::sleep(Duration::from_secs(1)).await; +/// fn main() { +/// use std::time::Duration; +/// use pyo3::prelude::*; +/// /// Python::with_gil(|py| { -/// let event_loop = pyo3_asyncio::get_event_loop(py); -/// -/// event_loop -/// .call_method1( -/// "call_soon_threadsafe", -/// (event_loop -/// .getattr("stop") -/// .map_err(|e| e.print_and_set_sys_last_vars(py)) -/// .unwrap(),), -/// ) -/// .map_err(|e| e.print_and_set_sys_last_vars(py)) -/// .unwrap(); +/// pyo3_asyncio::with_runtime(py, || -> PyResult<()> { +/// let event_loop = PyObject::from(pyo3_asyncio::get_event_loop(py)?); +/// // Wait 1 second, then stop the event loop +/// async_std::task::spawn(async move { +/// async_std::task::sleep(Duration::from_secs(1)).await; +/// Python::with_gil(|py| { +/// event_loop +/// .as_ref(py) +/// .call_method1( +/// "call_soon_threadsafe", +/// (event_loop +/// .as_ref(py) +/// .getattr("stop") +/// .map_err(|e| e.print_and_set_sys_last_vars(py)) +/// .unwrap(),), +/// ) +/// .map_err(|e| e.print_and_set_sys_last_vars(py)) +/// .unwrap(); +/// }) +/// }); +/// +/// pyo3_asyncio::run_forever(py)?; +/// Ok(()) +/// }) +/// .map_err(|e| e.print_and_set_sys_last_vars(py)) +/// .unwrap(); /// }) -/// }); -/// -/// // block until stop is called -/// # #[cfg(feature = "async-std-runtime")] -/// pyo3_asyncio::run_forever(py)?; -/// # Ok(()) -/// # }) -/// # .map_err(|e| e.print_and_set_sys_last_vars(py)) -/// # .unwrap(); -/// # }) +/// } +/// # #[cfg(not(feature = "async-std-runtime"))] +/// fn main() {} pub fn run_forever(py: Python) -> PyResult<()> { - if let Err(e) = get_event_loop(py).call_method0("run_forever") { + if let Err(e) = get_event_loop(py)?.call_method0("run_forever") { if e.is_instance::(py) { Ok(()) } else { @@ -295,8 +304,9 @@ pub fn try_close(py: Python) -> PyResult<()> { .expect(EXPECT_INIT) .call_method0(py, "shutdown")?; - get_event_loop(py).call_method0("stop")?; - get_event_loop(py).call_method0("close")?; + let event_loop = get_event_loop(py)?; + event_loop.call_method0("stop")?; + event_loop.call_method0("close")?; Ok(()) } @@ -342,7 +352,7 @@ impl PyEnsureFuture { #[call] pub fn __call__(&mut self) -> PyResult<()> { Python::with_gil(|py| { - let task = ensure_future(py).call1((self.awaitable.as_ref(py),))?; + let task = ensure_future(py, self.awaitable.as_ref(py))?; let on_complete = PyTaskCompleter { tx: self.tx.take() }; task.call_method1("add_done_callback", (on_complete,))?; @@ -351,6 +361,11 @@ impl PyEnsureFuture { } } +fn call_soon_threadsafe(event_loop: &PyAny, args: impl IntoPy>) -> PyResult<()> { + event_loop.call_method1("call_soon_threadsafe", args)?; + Ok(()) +} + /// Convert a Python `awaitable` into a Rust Future /// /// This function converts the `awaitable` into a Python Task using `run_coroutine_threadsafe`. A @@ -390,6 +405,7 @@ impl PyEnsureFuture { /// /// Python::with_gil(|py| { /// pyo3_asyncio::into_future( +/// pyo3_asyncio::get_event_loop(py)?, /// test_mod /// .call_method1(py, "py_sleep", (seconds.into_py(py),))? /// .as_ref(py), @@ -399,12 +415,14 @@ impl PyEnsureFuture { /// Ok(()) /// } /// ``` -pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { - let py = awaitable.py(); +pub fn into_future( + event_loop: &PyAny, + awaitable: &PyAny, +) -> PyResult> + Send> { let (tx, rx) = oneshot::channel(); - CALL_SOON.get().expect(EXPECT_INIT).call1( - py, + call_soon_threadsafe( + event_loop, (PyEnsureFuture { awaitable: awaitable.into(), tx: Some(tx), @@ -416,11 +434,7 @@ pub fn into_future(awaitable: &PyAny) -> PyResult item, Err(_) => Python::with_gil(|py| { Err(PyErr::from_instance( - ASYNCIO - .get() - .expect(EXPECT_INIT) - .call_method0(py, "CancelledError")? - .as_ref(py), + asyncio(py)?.call_method0("CancelledError")?, )) }), } diff --git a/src/testing.rs b/src/testing.rs index 0b43ef3..07fbf8e 100644 --- a/src/testing.rs +++ b/src/testing.rs @@ -95,7 +95,7 @@ //! } //! //! #[pyo3_asyncio::async_std::test] -//! fn test_blocking_sleep() -> PyResult<()> { +//! fn test_blocking_sleep(_event_loop: PyObject) -> PyResult<()> { //! thread::sleep(Duration::from_secs(1)); //! Ok(()) //! } @@ -125,7 +125,7 @@ //! } //! //! #[pyo3_asyncio::tokio::test] -//! fn test_blocking_sleep() -> PyResult<()> { +//! fn test_blocking_sleep(_event_loop: PyObject) -> PyResult<()> { //! thread::sleep(Duration::from_secs(1)); //! Ok(()) //! } @@ -163,7 +163,7 @@ //! } //! # #[cfg(feature = "async-std-runtime")] //! #[pyo3_asyncio::async_std::test] -//! fn test_async_std_sync_test_compiles() -> PyResult<()> { +//! fn test_async_std_sync_test_compiles(_event_loop: PyObject) -> PyResult<()> { //! Ok(()) //! } //! @@ -174,7 +174,7 @@ //! } //! # #[cfg(feature = "tokio-runtime")] //! #[pyo3_asyncio::tokio::test] -//! fn test_tokio_sync_test_compiles() -> PyResult<()> { +//! fn test_tokio_sync_test_compiles(_event_loop: PyObject) -> PyResult<()> { //! Ok(()) //! } //! } @@ -341,7 +341,7 @@ mod tests { } #[cfg(feature = "async-std-runtime")] #[pyo3_asyncio::async_std::test] - fn test_async_std_sync_test_compiles() -> PyResult<()> { + fn test_async_std_sync_test_compiles(_event_loop: PyObject) -> PyResult<()> { Ok(()) } @@ -352,7 +352,7 @@ mod tests { } #[cfg(feature = "tokio-runtime")] #[pyo3_asyncio::tokio::test] - fn test_tokio_sync_test_compiles() -> PyResult<()> { + fn test_tokio_sync_test_compiles(_event_loop: PyObject) -> PyResult<()> { Ok(()) } } diff --git a/src/tokio.rs b/src/tokio.rs index c629918..ea96d01 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -1,14 +1,14 @@ -use std::{future::Future, thread}; +use std::{future::Future, pin::Pin, thread}; use ::tokio::{ runtime::{Builder, Runtime}, task, }; use futures::future::pending; -use once_cell::sync::OnceCell; +use once_cell::{sync::OnceCell, unsync::OnceCell as UnsyncOnceCell}; use pyo3::prelude::*; -use crate::generic; +use crate::generic::{self, Runtime as GenericRuntime, SpawnLocalExt}; /// attributes /// re-exports for macros @@ -42,10 +42,28 @@ impl generic::JoinError for task::JoinError { struct TokioRuntime; -impl generic::Runtime for TokioRuntime { +tokio::task_local! { + static EVENT_LOOP: UnsyncOnceCell; +} + +impl GenericRuntime for TokioRuntime { type JoinError = task::JoinError; type JoinHandle = task::JoinHandle<()>; + fn scope(event_loop: PyObject, fut: F) -> Pin + Send>> + where + F: Future + Send + 'static, + { + let cell = UnsyncOnceCell::new(); + cell.set(event_loop).unwrap(); + + Box::pin(EVENT_LOOP.scope(cell, fut)) + } + + fn get_task_event_loop() -> Option { + EVENT_LOOP.with(|c| c.get().map(|event_loop| event_loop.clone())) + } + fn spawn(fut: F) -> Self::JoinHandle where F: Future + Send + 'static, @@ -56,7 +74,17 @@ impl generic::Runtime for TokioRuntime { } } -impl generic::SpawnLocalExt for TokioRuntime { +impl SpawnLocalExt for TokioRuntime { + fn scope_local(event_loop: PyObject, fut: F) -> Pin>> + where + F: Future + 'static, + { + let cell = UnsyncOnceCell::new(); + cell.set(event_loop).unwrap(); + + Box::pin(EVENT_LOOP.scope(cell, fut)) + } + fn spawn_local(fut: F) -> Self::JoinHandle where F: Future + 'static, @@ -65,6 +93,25 @@ impl generic::SpawnLocalExt for TokioRuntime { } } +pub async fn scope(event_loop: PyObject, fut: F) -> R +where + F: Future + Send + 'static, +{ + TokioRuntime::scope(event_loop, fut).await +} + +pub async fn scope_local(event_loop: PyObject, fut: F) -> R +where + F: Future + 'static, +{ + TokioRuntime::scope_local(event_loop, fut).await +} + +/// Get the task local event loop for the current tokio task +pub fn task_event_loop() -> Option { + TokioRuntime::get_task_event_loop() +} + /// Initialize the Tokio Runtime with a custom build pub fn init(runtime: Runtime) { TOKIO_RUNTIME @@ -204,17 +251,17 @@ where /// fn sleep_for(py: Python, secs: &PyAny) -> PyResult { /// let secs = secs.extract()?; /// -/// pyo3_asyncio::tokio::into_coroutine(py, async move { +/// pyo3_asyncio::tokio::into_coroutine(pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), async move { /// tokio::time::sleep(Duration::from_secs(secs)).await; /// Python::with_gil(|py| Ok(py.None())) /// }) /// } /// ``` -pub fn into_coroutine(py: Python, fut: F) -> PyResult +pub fn into_coroutine(event_loop: &PyAny, fut: F) -> PyResult where F: Future> + Send + 'static, { - generic::into_coroutine::(py, fut) + generic::into_coroutine::(event_loop, fut) } /// Convert a `!Send` Rust Future into a Python awaitable @@ -232,11 +279,12 @@ where /// /// /// Awaitable non-send sleep function /// #[pyfunction] -/// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { +/// fn sleep_for(py: Python, secs: u64) -> PyResult { /// // Rc is non-send so it cannot be passed into pyo3_asyncio::tokio::into_coroutine /// let secs = Rc::new(secs); +/// let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); /// -/// pyo3_asyncio::tokio::local_future_into_py(py, async move { +/// pyo3_asyncio::tokio::local_future_into_py(event_loop.as_ref(py), async move { /// tokio::time::sleep(Duration::from_secs(*secs)).await; /// Python::with_gil(|py| Ok(py.None())) /// }) @@ -245,28 +293,36 @@ where /// # #[cfg(all(feature = "tokio-runtime", feature = "attributes"))] /// #[pyo3_asyncio::tokio::main] /// async fn main() -> PyResult<()> { +/// let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); +/// /// // the main coroutine is running in a Send context, so we cannot use LocalSet here. Instead /// // we use spawn_blocking in order to use LocalSet::block_on -/// tokio::task::spawn_blocking(|| { +/// tokio::task::spawn_blocking(move || { /// // LocalSet allows us to work with !Send futures within tokio. Without it, any calls to /// // pyo3_asyncio::tokio::local_future_into_py will panic. -/// tokio::task::LocalSet::new().block_on(pyo3_asyncio::tokio::get_runtime(), async { -/// Python::with_gil(|py| { -/// let py_future = sleep_for(py, 1)?; -/// pyo3_asyncio::into_future(py_future) -/// })? -/// .await?; +/// tokio::task::LocalSet::new().block_on( +/// pyo3_asyncio::tokio::get_runtime(), +/// pyo3_asyncio::tokio::scope_local(event_loop, async { +/// Python::with_gil(|py| { +/// let py_future = sleep_for(py, 1)?; +/// pyo3_asyncio::into_future( +/// pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), +/// py_future.as_ref(py) +/// ) +/// })? +/// .await?; /// -/// Ok(()) -/// }) +/// Ok(()) +/// }) +/// ) /// }).await.unwrap() /// } /// # #[cfg(not(all(feature = "tokio-runtime", feature = "attributes")))] /// # fn main() {} /// ``` -pub fn local_future_into_py<'p, F>(py: Python<'p>, fut: F) -> PyResult<&'p PyAny> +pub fn local_future_into_py<'p, F>(event_loop: &'p PyAny, fut: F) -> PyResult where F: Future> + 'static, { - generic::local_future_into_py::(py, fut) + generic::local_future_into_py::(event_loop, fut) } From 8a7aec8921815cc9f04c90ad58269205a113be82 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Fri, 2 Jul 2021 13:45:10 -0500 Subject: [PATCH 04/32] Removed unnecessary lifetime on create_future --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 8fad0b4..041ead0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -157,7 +157,7 @@ fn ensure_future<'p>(py: Python<'p>, awaitable: &'p PyAny) -> PyResult<&'p PyAny .call1((awaitable,)) } -fn create_future<'p>(event_loop: &'p PyAny) -> PyResult<&'p PyAny> { +fn create_future(event_loop: &PyAny) -> PyResult<&PyAny> { event_loop.call_method0("create_future") } From 849fae19475bb3c54e77ba2d55985ba6ae7372c6 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Fri, 2 Jul 2021 14:35:40 -0500 Subject: [PATCH 05/32] Changed return type of local_future_into_py back to &PyAny --- pytests/test_async_std_asyncio.rs | 2 +- pytests/tokio_asyncio/mod.rs | 2 +- src/async_std.rs | 6 +++--- src/generic.rs | 8 ++++---- src/tokio.rs | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index 193c567..800e81d 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -100,7 +100,7 @@ async fn test_local_coroutine() -> PyResult<()> { Ok(Python::with_gil(|py| py.None())) })?; - pyo3_asyncio::into_future(event_loop.as_ref(py), py_future.as_ref(py)) + pyo3_asyncio::into_future(event_loop.as_ref(py), py_future) })? .await?; diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index 7845d76..facec43 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -103,7 +103,7 @@ fn test_local_set_coroutine(event_loop: PyObject) -> PyResult<()> { Ok(Python::with_gil(|py| py.None())) })?; - pyo3_asyncio::into_future(event_loop.as_ref(py), py_future.as_ref(py)) + pyo3_asyncio::into_future(event_loop.as_ref(py), py_future) })? .await?; diff --git a/src/async_std.rs b/src/async_std.rs index 3ddb061..3a12306 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -195,13 +195,13 @@ where /// // Rc is non-send so it cannot be passed into pyo3_asyncio::async_std::into_coroutine /// let secs = Rc::new(secs); /// -/// pyo3_asyncio::async_std::local_future_into_py( +/// Ok(pyo3_asyncio::async_std::local_future_into_py( /// pyo3_asyncio::async_std::task_event_loop().unwrap().as_ref(py), /// async move { /// async_std::task::sleep(Duration::from_secs(*secs)).await; /// Python::with_gil(|py| Ok(py.None())) /// } -/// ) +/// )?.into()) /// } /// /// # #[cfg(all(feature = "async-std-runtime", feature = "attributes"))] @@ -221,7 +221,7 @@ where /// # #[cfg(not(all(feature = "async-std-runtime", feature = "attributes")))] /// # fn main() {} /// ``` -pub fn local_future_into_py(event_loop: &PyAny, fut: F) -> PyResult +pub fn local_future_into_py(event_loop: &PyAny, fut: F) -> PyResult<&PyAny> where F: Future> + 'static, { diff --git a/src/generic.rs b/src/generic.rs index 5b73c33..bb8a7fa 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -363,7 +363,7 @@ where /// /// /// Awaitable sleep function /// #[pyfunction] -/// fn sleep_for(py: Python, secs: u64) -> PyResult { +/// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { /// pyo3_asyncio::generic::local_future_into_py::( /// pyo3_asyncio::get_event_loop(py)?, /// async move { @@ -373,13 +373,13 @@ where /// ) /// } /// ``` -pub fn local_future_into_py(event_loop: &PyAny, fut: F) -> PyResult +pub fn local_future_into_py(event_loop: &PyAny, fut: F) -> PyResult<&PyAny> where R: SpawnLocalExt, F: Future> + 'static, { - let future_rx = PyObject::from(create_future(event_loop)?); - let future_tx1 = future_rx.clone(); + let future_rx = create_future(event_loop)?; + let future_tx1 = PyObject::from(future_rx); let future_tx2 = future_tx1.clone(); let event_loop = PyObject::from(event_loop); diff --git a/src/tokio.rs b/src/tokio.rs index ea96d01..b544094 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -284,10 +284,10 @@ where /// let secs = Rc::new(secs); /// let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); /// -/// pyo3_asyncio::tokio::local_future_into_py(event_loop.as_ref(py), async move { +/// Ok(pyo3_asyncio::tokio::local_future_into_py(event_loop.as_ref(py), async move { /// tokio::time::sleep(Duration::from_secs(*secs)).await; /// Python::with_gil(|py| Ok(py.None())) -/// }) +/// })?.into()) /// } /// /// # #[cfg(all(feature = "tokio-runtime", feature = "attributes"))] @@ -320,7 +320,7 @@ where /// # #[cfg(not(all(feature = "tokio-runtime", feature = "attributes")))] /// # fn main() {} /// ``` -pub fn local_future_into_py<'p, F>(event_loop: &'p PyAny, fut: F) -> PyResult +pub fn local_future_into_py<'p, F>(event_loop: &'p PyAny, fut: F) -> PyResult<&PyAny> where F: Future> + 'static, { From b0972d9ea447441acd32695ba261360a57aa02d8 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Fri, 2 Jul 2021 16:41:09 -0500 Subject: [PATCH 06/32] Added with_loop suffix to functions with explicit event_loop parameters --- README.md | 4 ++-- examples/async_std.rs | 2 +- examples/tokio.rs | 2 +- examples/tokio_current_thread.rs | 2 +- examples/tokio_multi_thread.rs | 2 +- pytests/common/mod.rs | 4 ++-- pytests/test_async_std_asyncio.rs | 16 +++++++++------- pytests/tokio_asyncio/mod.rs | 16 +++++++++------- src/async_std.rs | 10 ++++++---- src/generic.rs | 4 ++-- src/lib.rs | 4 ++-- src/tokio.rs | 10 ++++++---- 12 files changed, 42 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index b8310f1..e7ed791 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( pyo3_asyncio::async_std::task_event_loop().unwrap().as_ref(py), asyncio.call_method1("sleep", (1.into_py(py),))? ) @@ -80,7 +80,7 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), asyncio.call_method1("sleep", (1.into_py(py),))? ) diff --git a/examples/async_std.rs b/examples/async_std.rs index 75f17d5..d78dbd1 100644 --- a/examples/async_std.rs +++ b/examples/async_std.rs @@ -6,7 +6,7 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( pyo3_asyncio::async_std::task_event_loop() .unwrap() .as_ref(py), diff --git a/examples/tokio.rs b/examples/tokio.rs index a4b6981..e6e597b 100644 --- a/examples/tokio.rs +++ b/examples/tokio.rs @@ -6,7 +6,7 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), asyncio.call_method1("sleep", (1.into_py(py),))?, ) diff --git a/examples/tokio_current_thread.rs b/examples/tokio_current_thread.rs index 435e602..25a8f78 100644 --- a/examples/tokio_current_thread.rs +++ b/examples/tokio_current_thread.rs @@ -6,7 +6,7 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), asyncio.call_method1("sleep", (1.into_py(py),))?, ) diff --git a/examples/tokio_multi_thread.rs b/examples/tokio_multi_thread.rs index 2ff2d59..f7f2949 100644 --- a/examples/tokio_multi_thread.rs +++ b/examples/tokio_multi_thread.rs @@ -6,7 +6,7 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), asyncio.call_method1("sleep", (1.into_py(py),))?, ) diff --git a/pytests/common/mod.rs b/pytests/common/mod.rs index 2cc6791..dc388aa 100644 --- a/pytests/common/mod.rs +++ b/pytests/common/mod.rs @@ -17,7 +17,7 @@ pub(super) async fn test_into_future(event_loop: PyObject) -> PyResult<()> { let test_mod = PyModule::from_code(py, TEST_MOD, "test_rust_coroutine/test_mod.py", "test_mod")?; - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( event_loop.as_ref(py), test_mod.call_method1("py_sleep", (1.into_py(py),))?, ) @@ -47,7 +47,7 @@ pub(super) async fn test_other_awaitables(event_loop: PyObject) -> PyResult<()> ), )?; - pyo3_asyncio::into_future(event_loop.as_ref(py), task) + pyo3_asyncio::into_future_with_loop(event_loop.as_ref(py), task) })?; fut.await?; diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index 78ac799..1eb762d 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -29,7 +29,7 @@ async fn test_into_coroutine() -> PyResult<()> { "test_mod", )?; - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( pyo3_asyncio::async_std::task_event_loop() .unwrap() .as_ref(py), @@ -50,7 +50,7 @@ async fn test_async_sleep() -> PyResult<()> { task::sleep(Duration::from_secs(1)).await; Python::with_gil(|py| { - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( pyo3_asyncio::async_std::task_event_loop() .unwrap() .as_ref(py), @@ -86,7 +86,7 @@ fn test_init_twice(_event_loop: PyObject) -> PyResult<()> { async fn test_panic() -> PyResult<()> { let fut = Python::with_gil(|py| -> PyResult<_> { let event_loop = pyo3_asyncio::async_std::task_event_loop().unwrap(); - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( event_loop.as_ref(py), pyo3_asyncio::async_std::into_coroutine(event_loop.as_ref(py), async { panic!("this panic was intentional!") @@ -119,13 +119,15 @@ async fn test_local_coroutine() -> PyResult<()> { let event_loop = pyo3_asyncio::async_std::task_event_loop().unwrap(); - let py_future = - pyo3_asyncio::async_std::local_future_into_py(event_loop.as_ref(py), async move { + let py_future = pyo3_asyncio::async_std::local_future_into_py_with_loop( + event_loop.as_ref(py), + async move { async_std::task::sleep(Duration::from_secs(*non_send_secs)).await; Ok(Python::with_gil(|py| py.None())) - })?; + }, + )?; - pyo3_asyncio::into_future(event_loop.as_ref(py), py_future) + pyo3_asyncio::into_future_with_loop(event_loop.as_ref(py), py_future) })? .await?; diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index 4c9d88c..16bd7ba 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -30,7 +30,7 @@ async fn test_into_coroutine() -> PyResult<()> { "test_mod", )?; - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( event_loop.as_ref(py), test_mod.call_method1("sleep_for_1s", (sleeper_mod.getattr("sleep_for")?,))?, ) @@ -51,7 +51,7 @@ async fn test_async_sleep() -> PyResult<()> { tokio::time::sleep(Duration::from_secs(1)).await; Python::with_gil(|py| { - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( event_loop.as_ref(py), asyncio.as_ref(py).call_method1("sleep", (1.0,))?, ) @@ -97,13 +97,15 @@ fn test_local_set_coroutine(event_loop: PyObject) -> PyResult<()> { Python::with_gil(|py| { let non_send_secs = Rc::new(1); - let py_future = - pyo3_asyncio::tokio::local_future_into_py(event_loop.as_ref(py), async move { + let py_future = pyo3_asyncio::tokio::local_future_into_py_with_loop( + event_loop.as_ref(py), + async move { tokio::time::sleep(Duration::from_secs(*non_send_secs)).await; Ok(Python::with_gil(|py| py.None())) - })?; + }, + )?; - pyo3_asyncio::into_future(event_loop.as_ref(py), py_future) + pyo3_asyncio::into_future_with_loop(event_loop.as_ref(py), py_future) })? .await?; @@ -115,7 +117,7 @@ fn test_local_set_coroutine(event_loop: PyObject) -> PyResult<()> { async fn test_panic() -> PyResult<()> { let fut = Python::with_gil(|py| -> PyResult<_> { let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); - pyo3_asyncio::into_future( + pyo3_asyncio::into_future_with_loop( event_loop.as_ref(py), pyo3_asyncio::tokio::into_coroutine(event_loop.as_ref(py), async { panic!("this panic was intentional!") diff --git a/src/async_std.rs b/src/async_std.rs index c69d2c5..0924d12 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -87,6 +87,7 @@ impl SpawnLocalExt for AsyncStdRuntime { } } +/// Set the task local event loop for the given future pub async fn scope(event_loop: PyObject, fut: F) -> R where F: Future + Send + 'static, @@ -94,6 +95,7 @@ where AsyncStdRuntime::scope(event_loop, fut).await } +/// Set the task local event loop for the given !Send future pub async fn scope_local(event_loop: PyObject, fut: F) -> R where F: Future + 'static, @@ -198,7 +200,7 @@ where /// // Rc is non-send so it cannot be passed into pyo3_asyncio::async_std::into_coroutine /// let secs = Rc::new(secs); /// -/// Ok(pyo3_asyncio::async_std::local_future_into_py( +/// Ok(pyo3_asyncio::async_std::local_future_into_py_with_loop( /// pyo3_asyncio::async_std::task_event_loop().unwrap().as_ref(py), /// async move { /// async_std::task::sleep(Duration::from_secs(*secs)).await; @@ -212,7 +214,7 @@ where /// async fn main() -> PyResult<()> { /// Python::with_gil(|py| { /// let py_future = sleep_for(py, 1)?; -/// pyo3_asyncio::into_future( +/// pyo3_asyncio::into_future_with_loop( /// pyo3_asyncio::async_std::task_event_loop().unwrap().as_ref(py), /// py_future.as_ref(py) /// ) @@ -224,9 +226,9 @@ where /// # #[cfg(not(all(feature = "async-std-runtime", feature = "attributes")))] /// # fn main() {} /// ``` -pub fn local_future_into_py(event_loop: &PyAny, fut: F) -> PyResult<&PyAny> +pub fn local_future_into_py_with_loop(event_loop: &PyAny, fut: F) -> PyResult<&PyAny> where F: Future> + 'static, { - generic::local_future_into_py::(event_loop, fut) + generic::local_future_into_py_with_loop::(event_loop, fut) } diff --git a/src/generic.rs b/src/generic.rs index 35c56ab..0c45713 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -364,7 +364,7 @@ where /// /// Awaitable sleep function /// #[pyfunction] /// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { -/// pyo3_asyncio::generic::local_future_into_py::( +/// pyo3_asyncio::generic::local_future_into_py_with_loop::( /// pyo3_asyncio::get_event_loop(py)?, /// async move { /// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; @@ -373,7 +373,7 @@ where /// ) /// } /// ``` -pub fn local_future_into_py(event_loop: &PyAny, fut: F) -> PyResult<&PyAny> +pub fn local_future_into_py_with_loop(event_loop: &PyAny, fut: F) -> PyResult<&PyAny> where R: SpawnLocalExt, F: Future> + 'static, diff --git a/src/lib.rs b/src/lib.rs index 41bb9b1..df2b596 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -407,7 +407,7 @@ fn call_soon_threadsafe(event_loop: &PyAny, args: impl IntoPy>) -> P /// })?; /// /// Python::with_gil(|py| { -/// pyo3_asyncio::into_future( +/// pyo3_asyncio::into_future_with_loop( /// pyo3_asyncio::get_event_loop(py)?, /// test_mod /// .call_method1(py, "py_sleep", (seconds.into_py(py),))? @@ -418,7 +418,7 @@ fn call_soon_threadsafe(event_loop: &PyAny, args: impl IntoPy>) -> P /// Ok(()) /// } /// ``` -pub fn into_future( +pub fn into_future_with_loop( event_loop: &PyAny, awaitable: &PyAny, ) -> PyResult> + Send> { diff --git a/src/tokio.rs b/src/tokio.rs index b544094..3819110 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -93,6 +93,7 @@ impl SpawnLocalExt for TokioRuntime { } } +/// Set the task local event loop for the given future pub async fn scope(event_loop: PyObject, fut: F) -> R where F: Future + Send + 'static, @@ -100,6 +101,7 @@ where TokioRuntime::scope(event_loop, fut).await } +/// Set the task local event loop for the given !Send future pub async fn scope_local(event_loop: PyObject, fut: F) -> R where F: Future + 'static, @@ -284,7 +286,7 @@ where /// let secs = Rc::new(secs); /// let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); /// -/// Ok(pyo3_asyncio::tokio::local_future_into_py(event_loop.as_ref(py), async move { +/// Ok(pyo3_asyncio::tokio::local_future_into_py_with_loop(event_loop.as_ref(py), async move { /// tokio::time::sleep(Duration::from_secs(*secs)).await; /// Python::with_gil(|py| Ok(py.None())) /// })?.into()) @@ -305,7 +307,7 @@ where /// pyo3_asyncio::tokio::scope_local(event_loop, async { /// Python::with_gil(|py| { /// let py_future = sleep_for(py, 1)?; -/// pyo3_asyncio::into_future( +/// pyo3_asyncio::into_future_with_loop( /// pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), /// py_future.as_ref(py) /// ) @@ -320,9 +322,9 @@ where /// # #[cfg(not(all(feature = "tokio-runtime", feature = "attributes")))] /// # fn main() {} /// ``` -pub fn local_future_into_py<'p, F>(event_loop: &'p PyAny, fut: F) -> PyResult<&PyAny> +pub fn local_future_into_py_with_loop<'p, F>(event_loop: &'p PyAny, fut: F) -> PyResult<&PyAny> where F: Future> + 'static, { - generic::local_future_into_py::(event_loop, fut) + generic::local_future_into_py_with_loop::(event_loop, fut) } From 5806195c894285d6144268361371c0b3674666c2 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Fri, 2 Jul 2021 17:28:24 -0500 Subject: [PATCH 07/32] Added implicit event loop parameter variants for the into_future conversions --- README.md | 19 +--- examples/async_std.rs | 7 +- examples/tokio.rs | 5 +- examples/tokio_current_thread.rs | 5 +- examples/tokio_multi_thread.rs | 5 +- pyo3-asyncio-macros/src/lib.rs | 8 +- pytests/test_async_std_asyncio.rs | 44 ++++---- pytests/tokio_asyncio/mod.rs | 29 +++--- src/async_std.rs | 127 ++++++++++++++++++++--- src/generic.rs | 121 +++++++++++++++++++++- src/tokio.rs | 166 ++++++++++++++++++++++++++---- 11 files changed, 421 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index e7ed791..55ae2fd 100644 --- a/README.md +++ b/README.md @@ -44,12 +44,8 @@ use pyo3::prelude::*; async fn main() -> PyResult<()> { let fut = Python::with_gil(|py| { let asyncio = py.import("asyncio")?; - // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future_with_loop( - pyo3_asyncio::async_std::task_event_loop().unwrap().as_ref(py), - asyncio.call_method1("sleep", (1.into_py(py),))? - ) + pyo3_asyncio::async_std::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) })?; fut.await?; @@ -78,12 +74,8 @@ use pyo3::prelude::*; async fn main() -> PyResult<()> { let fut = Python::with_gil(|py| { let asyncio = py.import("asyncio")?; - // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future_with_loop( - pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), - asyncio.call_method1("sleep", (1.into_py(py),))? - ) + pyo3_asyncio::tokio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) })?; fut.await?; @@ -133,7 +125,7 @@ use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] fn rust_sleep(py: Python) -> PyResult { - pyo3_asyncio::async_std::into_coroutine(pyo3_asyncio::get_event_loop(py)?, async { + pyo3_asyncio::async_std::into_coroutine(pyo3_asyncio::async_std::current_event_loop(py)?, async { async_std::task::sleep(std::time::Duration::from_secs(1)).await; Ok(Python::with_gil(|py| py.None())) }) @@ -141,8 +133,6 @@ fn rust_sleep(py: Python) -> PyResult { #[pymodule] fn my_async_module(py: Python, m: &PyModule) -> PyResult<()> { - pyo3_asyncio::try_init(py)?; - m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; Ok(()) @@ -159,7 +149,7 @@ use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] fn rust_sleep(py: Python) -> PyResult { - pyo3_asyncio::tokio::into_coroutine(pyo3_asyncio::get_event_loop(py)?, async { + pyo3_asyncio::tokio::into_coroutine(pyo3_asyncio::tokio::current_event_loop(py)?, async { tokio::time::sleep(std::time::Duration::from_secs(1)).await; Ok(Python::with_gil(|py| py.None())) }) @@ -167,7 +157,6 @@ fn rust_sleep(py: Python) -> PyResult { #[pymodule] fn my_async_module(py: Python, m: &PyModule) -> PyResult<()> { - pyo3_asyncio::try_init(py)?; // Tokio needs explicit initialization before any pyo3-asyncio conversions. // The module import is a prime place to do this. pyo3_asyncio::tokio::init_multi_thread_once(); diff --git a/examples/async_std.rs b/examples/async_std.rs index d78dbd1..ba7c8e7 100644 --- a/examples/async_std.rs +++ b/examples/async_std.rs @@ -6,12 +6,7 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future_with_loop( - pyo3_asyncio::async_std::task_event_loop() - .unwrap() - .as_ref(py), - asyncio.call_method1("sleep", (1.into_py(py),))?, - ) + pyo3_asyncio::async_std::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) })?; println!("sleeping for 1s"); diff --git a/examples/tokio.rs b/examples/tokio.rs index e6e597b..794c43a 100644 --- a/examples/tokio.rs +++ b/examples/tokio.rs @@ -6,10 +6,7 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future_with_loop( - pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), - asyncio.call_method1("sleep", (1.into_py(py),))?, - ) + pyo3_asyncio::tokio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) })?; println!("sleeping for 1s"); diff --git a/examples/tokio_current_thread.rs b/examples/tokio_current_thread.rs index 25a8f78..5b8819a 100644 --- a/examples/tokio_current_thread.rs +++ b/examples/tokio_current_thread.rs @@ -6,10 +6,7 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future_with_loop( - pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), - asyncio.call_method1("sleep", (1.into_py(py),))?, - ) + pyo3_asyncio::tokio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) })?; println!("sleeping for 1s"); diff --git a/examples/tokio_multi_thread.rs b/examples/tokio_multi_thread.rs index f7f2949..0deb688 100644 --- a/examples/tokio_multi_thread.rs +++ b/examples/tokio_multi_thread.rs @@ -6,10 +6,7 @@ async fn main() -> PyResult<()> { let asyncio = py.import("asyncio")?; // convert asyncio.sleep into a Rust Future - pyo3_asyncio::into_future_with_loop( - pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), - asyncio.call_method1("sleep", (1.into_py(py),))?, - ) + pyo3_asyncio::tokio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) })?; println!("sleeping for 1s"); diff --git a/pyo3-asyncio-macros/src/lib.rs b/pyo3-asyncio-macros/src/lib.rs index fd4498a..e3e2679 100644 --- a/pyo3-asyncio-macros/src/lib.rs +++ b/pyo3-asyncio-macros/src/lib.rs @@ -146,7 +146,9 @@ pub fn async_std_test(_attr: TokenStream, item: TokenStream) -> TokenStream { #body } - let event_loop = pyo3_asyncio::async_std::task_event_loop().unwrap(); + let event_loop = Python::with_gil(|py| { + pyo3_asyncio::async_std::task_event_loop(py).unwrap().into() + }); Box::pin(pyo3_asyncio::async_std::re_exports::spawn_blocking(move || { #name(event_loop) @@ -224,7 +226,9 @@ pub fn tokio_test(_attr: TokenStream, item: TokenStream) -> TokenStream { #body } - let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); + let event_loop = Python::with_gil(|py| { + pyo3_asyncio::tokio::task_event_loop(py).unwrap().into() + }); Box::pin(async move { match pyo3_asyncio::tokio::get_runtime().spawn_blocking(move || #name(event_loop)).await { diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index 1eb762d..c962755 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -29,10 +29,7 @@ async fn test_into_coroutine() -> PyResult<()> { "test_mod", )?; - pyo3_asyncio::into_future_with_loop( - pyo3_asyncio::async_std::task_event_loop() - .unwrap() - .as_ref(py), + pyo3_asyncio::async_std::into_future( test_mod.call_method1("sleep_for_1s", (sleeper_mod.getattr("sleep_for")?,))?, ) })?; @@ -50,12 +47,7 @@ async fn test_async_sleep() -> PyResult<()> { task::sleep(Duration::from_secs(1)).await; Python::with_gil(|py| { - pyo3_asyncio::into_future_with_loop( - pyo3_asyncio::async_std::task_event_loop() - .unwrap() - .as_ref(py), - asyncio.as_ref(py).call_method1("sleep", (1.0,))?, - ) + pyo3_asyncio::async_std::into_future(asyncio.as_ref(py).call_method1("sleep", (1.0,))?) })? .await?; @@ -69,12 +61,18 @@ fn test_blocking_sleep(_event_loop: PyObject) -> PyResult<()> { #[pyo3_asyncio::async_std::test] async fn test_into_future() -> PyResult<()> { - common::test_into_future(pyo3_asyncio::async_std::task_event_loop().unwrap()).await + common::test_into_future(Python::with_gil(|py| { + pyo3_asyncio::async_std::task_event_loop(py).unwrap().into() + })) + .await } #[pyo3_asyncio::async_std::test] async fn test_other_awaitables() -> PyResult<()> { - common::test_other_awaitables(pyo3_asyncio::async_std::task_event_loop().unwrap()).await + common::test_other_awaitables(Python::with_gil(|py| { + pyo3_asyncio::async_std::task_event_loop(py).unwrap().into() + })) + .await } #[pyo3_asyncio::async_std::test] @@ -85,10 +83,9 @@ fn test_init_twice(_event_loop: PyObject) -> PyResult<()> { #[pyo3_asyncio::async_std::test] async fn test_panic() -> PyResult<()> { let fut = Python::with_gil(|py| -> PyResult<_> { - let event_loop = pyo3_asyncio::async_std::task_event_loop().unwrap(); - pyo3_asyncio::into_future_with_loop( - event_loop.as_ref(py), - pyo3_asyncio::async_std::into_coroutine(event_loop.as_ref(py), async { + let event_loop = pyo3_asyncio::async_std::task_event_loop(py).unwrap(); + pyo3_asyncio::async_std::into_future( + pyo3_asyncio::async_std::into_coroutine(event_loop, async { panic!("this panic was intentional!") })? .as_ref(py), @@ -117,17 +114,12 @@ async fn test_local_coroutine() -> PyResult<()> { Python::with_gil(|py| { let non_send_secs = Rc::new(1); - let event_loop = pyo3_asyncio::async_std::task_event_loop().unwrap(); - - let py_future = pyo3_asyncio::async_std::local_future_into_py_with_loop( - event_loop.as_ref(py), - async move { - async_std::task::sleep(Duration::from_secs(*non_send_secs)).await; - Ok(Python::with_gil(|py| py.None())) - }, - )?; + let py_future = pyo3_asyncio::async_std::local_future_into_py(py, async move { + async_std::task::sleep(Duration::from_secs(*non_send_secs)).await; + Ok(Python::with_gil(|py| py.None())) + })?; - pyo3_asyncio::into_future_with_loop(event_loop.as_ref(py), py_future) + pyo3_asyncio::async_std::into_future(py_future) })? .await?; diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index 16bd7ba..ba7c0c5 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -16,8 +16,6 @@ fn sleep_for(py: Python, secs: &PyAny) -> PyResult { #[pyo3_asyncio::tokio::test] async fn test_into_coroutine() -> PyResult<()> { - let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); - let fut = Python::with_gil(|py| { let sleeper_mod = PyModule::new(py, "rust_sleeper")?; @@ -30,8 +28,7 @@ async fn test_into_coroutine() -> PyResult<()> { "test_mod", )?; - pyo3_asyncio::into_future_with_loop( - event_loop.as_ref(py), + pyo3_asyncio::tokio::into_future( test_mod.call_method1("sleep_for_1s", (sleeper_mod.getattr("sleep_for")?,))?, ) })?; @@ -43,18 +40,13 @@ async fn test_into_coroutine() -> PyResult<()> { #[pyo3_asyncio::tokio::test] async fn test_async_sleep() -> PyResult<()> { - let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); - let asyncio = Python::with_gil(|py| py.import("asyncio").map(|asyncio| PyObject::from(asyncio)))?; tokio::time::sleep(Duration::from_secs(1)).await; Python::with_gil(|py| { - pyo3_asyncio::into_future_with_loop( - event_loop.as_ref(py), - asyncio.as_ref(py).call_method1("sleep", (1.0,))?, - ) + pyo3_asyncio::tokio::into_future(asyncio.as_ref(py).call_method1("sleep", (1.0,))?) })? .await?; @@ -68,12 +60,18 @@ fn test_blocking_sleep(_event_loop: PyObject) -> PyResult<()> { #[pyo3_asyncio::tokio::test] async fn test_into_future() -> PyResult<()> { - common::test_into_future(pyo3_asyncio::tokio::task_event_loop().unwrap()).await + common::test_into_future(Python::with_gil(|py| { + pyo3_asyncio::tokio::task_event_loop(py).unwrap().into() + })) + .await } #[pyo3_asyncio::tokio::test] async fn test_other_awaitables() -> PyResult<()> { - common::test_other_awaitables(pyo3_asyncio::tokio::task_event_loop().unwrap()).await + common::test_other_awaitables(Python::with_gil(|py| { + pyo3_asyncio::tokio::task_event_loop(py).unwrap().into() + })) + .await } #[pyo3_asyncio::tokio::test] @@ -116,10 +114,9 @@ fn test_local_set_coroutine(event_loop: PyObject) -> PyResult<()> { #[pyo3_asyncio::tokio::test] async fn test_panic() -> PyResult<()> { let fut = Python::with_gil(|py| -> PyResult<_> { - let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); - pyo3_asyncio::into_future_with_loop( - event_loop.as_ref(py), - pyo3_asyncio::tokio::into_coroutine(event_loop.as_ref(py), async { + let event_loop = pyo3_asyncio::tokio::task_event_loop(py).unwrap(); + pyo3_asyncio::tokio::into_future( + pyo3_asyncio::tokio::into_coroutine(event_loop, async { panic!("this panic was intentional!") })? .as_ref(py), diff --git a/src/async_std.rs b/src/async_std.rs index 0924d12..3d35b56 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -3,9 +3,12 @@ use std::{any::Any, future::Future, panic::AssertUnwindSafe, pin::Pin}; use async_std::task; use futures::prelude::*; use once_cell::unsync::OnceCell; -use pyo3::prelude::*; +use pyo3::{prelude::*, PyNativeType}; -use crate::generic::{self, JoinError, Runtime, SpawnLocalExt}; +use crate::{ + generic::{self, JoinError, Runtime, SpawnLocalExt}, + into_future_with_loop, +}; /// attributes /// re-exports for macros @@ -50,8 +53,8 @@ impl Runtime for AsyncStdRuntime { EVENT_LOOP.with(|c| c.set(event_loop).unwrap()); Box::pin(fut) } - fn get_task_event_loop() -> Option { - EVENT_LOOP.with(|c| c.get().map(|event_loop| event_loop.clone())) + fn get_task_event_loop(py: Python) -> Option<&PyAny> { + EVENT_LOOP.with(|c| c.get().map(|event_loop| event_loop.clone().into_ref(py))) } fn spawn(fut: F) -> Self::JoinHandle @@ -103,9 +106,14 @@ where AsyncStdRuntime::scope_local(event_loop, fut).await } +/// Get the current event loop from either Python or Rust async task local context +pub fn current_event_loop(py: Python) -> PyResult<&PyAny> { + generic::current_event_loop::(py) +} + /// Get the task local event loop for the current async_std task -pub fn task_event_loop() -> Option { - AsyncStdRuntime::get_task_event_loop() +pub fn task_event_loop(py: Python) -> Option<&PyAny> { + AsyncStdRuntime::get_task_event_loop(py) } /// Run the event loop until the given Future completes @@ -196,12 +204,11 @@ where /// /// /// Awaitable non-send sleep function /// #[pyfunction] -/// fn sleep_for(py: Python, secs: u64) -> PyResult { +/// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { /// // Rc is non-send so it cannot be passed into pyo3_asyncio::async_std::into_coroutine /// let secs = Rc::new(secs); -/// /// Ok(pyo3_asyncio::async_std::local_future_into_py_with_loop( -/// pyo3_asyncio::async_std::task_event_loop().unwrap().as_ref(py), +/// pyo3_asyncio::async_std::current_event_loop(py)?, /// async move { /// async_std::task::sleep(Duration::from_secs(*secs)).await; /// Python::with_gil(|py| Ok(py.None())) @@ -214,10 +221,7 @@ where /// async fn main() -> PyResult<()> { /// Python::with_gil(|py| { /// let py_future = sleep_for(py, 1)?; -/// pyo3_asyncio::into_future_with_loop( -/// pyo3_asyncio::async_std::task_event_loop().unwrap().as_ref(py), -/// py_future.as_ref(py) -/// ) +/// pyo3_asyncio::async_std::into_future(py_future) /// })? /// .await?; /// @@ -232,3 +236,100 @@ where { generic::local_future_into_py_with_loop::(event_loop, fut) } + +/// Convert a `!Send` Rust Future into a Python awaitable +/// +/// # Arguments +/// * `py` - The current PyO3 GIL guard +/// * `fut` - The Rust future to be converted +/// +/// # Examples +/// +/// ``` +/// use std::{rc::Rc, time::Duration}; +/// +/// use pyo3::prelude::*; +/// +/// /// Awaitable non-send sleep function +/// #[pyfunction] +/// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { +/// // Rc is non-send so it cannot be passed into pyo3_asyncio::async_std::into_coroutine +/// let secs = Rc::new(secs); +/// pyo3_asyncio::async_std::local_future_into_py(py, async move { +/// async_std::task::sleep(Duration::from_secs(*secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// }) +/// } +/// +/// # #[cfg(all(feature = "async-std-runtime", feature = "attributes"))] +/// #[pyo3_asyncio::async_std::main] +/// async fn main() -> PyResult<()> { +/// Python::with_gil(|py| { +/// let py_future = sleep_for(py, 1)?; +/// pyo3_asyncio::async_std::into_future(py_future) +/// })? +/// .await?; +/// +/// Ok(()) +/// } +/// # #[cfg(not(all(feature = "async-std-runtime", feature = "attributes")))] +/// # fn main() {} +/// ``` +pub fn local_future_into_py(py: Python, fut: F) -> PyResult<&PyAny> +where + F: Future> + 'static, +{ + generic::local_future_into_py::(py, fut) +} + +/// Convert a Python `awaitable` into a Rust Future +/// +/// This function converts the `awaitable` into a Python Task using `run_coroutine_threadsafe`. A +/// completion handler sends the result of this Task through a +/// `futures::channel::oneshot::Sender>` and the future returned by this function +/// simply awaits the result through the `futures::channel::oneshot::Receiver>`. +/// +/// # Arguments +/// * `awaitable` - The Python `awaitable` to be converted +/// +/// # Examples +/// +/// ``` +/// use std::time::Duration; +/// +/// use pyo3::prelude::*; +/// +/// const PYTHON_CODE: &'static str = r#" +/// import asyncio +/// +/// async def py_sleep(duration): +/// await asyncio.sleep(duration) +/// "#; +/// +/// async fn py_sleep(seconds: f32) -> PyResult<()> { +/// let test_mod = Python::with_gil(|py| -> PyResult { +/// Ok( +/// PyModule::from_code( +/// py, +/// PYTHON_CODE, +/// "test_into_future/test_mod.py", +/// "test_mod" +/// )? +/// .into() +/// ) +/// })?; +/// +/// Python::with_gil(|py| { +/// pyo3_asyncio::async_std::into_future( +/// test_mod +/// .call_method1(py, "py_sleep", (seconds.into_py(py),))? +/// .as_ref(py), +/// ) +/// })? +/// .await?; +/// Ok(()) +/// } +/// ``` +pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { + into_future_with_loop(current_event_loop(awaitable.py())?, awaitable) +} diff --git a/src/generic.rs b/src/generic.rs index 0c45713..dc97c5c 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -22,7 +22,7 @@ pub trait Runtime { where F: Future + Send + 'static; /// Get the task local event loop for the current task - fn get_task_event_loop() -> Option; + fn get_task_event_loop(py: Python) -> Option<&PyAny>; /// Spawn a future onto this runtime's event loop fn spawn(fut: F) -> Self::JoinHandle @@ -43,6 +43,18 @@ pub trait SpawnLocalExt: Runtime { F: Future + 'static; } +/// Get the current event loop from either Python or Rust async task local context +pub fn current_event_loop(py: Python) -> PyResult<&PyAny> +where + R: Runtime, +{ + if let Some(event_loop) = R::get_task_event_loop(py) { + Ok(event_loop) + } else { + get_event_loop(py) + } +} + /// Run the event loop until the given Future completes /// /// After this function returns, the event loop can be resumed with either [`run_until_complete`] or @@ -89,7 +101,7 @@ pub trait SpawnLocalExt: Runtime { /// # { /// # unreachable!() /// # } -/// # fn get_task_event_loop() -> Option { +/// # fn get_task_event_loop(py: Python) -> Option<&PyAny> { /// # unreachable!() /// # } /// # @@ -201,7 +213,7 @@ fn set_result(event_loop: &PyAny, future: &PyAny, result: PyResult) -> /// # { /// # unreachable!() /// # } -/// # fn get_task_event_loop() -> Option { +/// # fn get_task_event_loop(py: Python) -> Option<&PyAny> { /// # unreachable!() /// # } /// # @@ -329,7 +341,7 @@ where /// # { /// # unreachable!() /// # } -/// # fn get_task_event_loop() -> Option { +/// # fn get_task_event_loop(py: Python) -> Option<&PyAny> { /// # unreachable!() /// # } /// # @@ -420,3 +432,104 @@ where Ok(future_rx) } + +/// Convert a `!Send` Rust Future into a Python awaitable with a generic runtime +/// +/// # Arguments +/// * `py` - The current PyO3 GIL guard +/// * `fut` - The Rust future to be converted +/// +/// # Examples +/// +/// ```no_run +/// # use std::{task::{Context, Poll}, pin::Pin, future::Future}; +/// # +/// # use pyo3_asyncio::generic::{JoinError, SpawnLocalExt, Runtime}; +/// # +/// # struct MyCustomJoinError; +/// # +/// # impl JoinError for MyCustomJoinError { +/// # fn is_panic(&self) -> bool { +/// # unreachable!() +/// # } +/// # } +/// # +/// # struct MyCustomJoinHandle; +/// # +/// # impl Future for MyCustomJoinHandle { +/// # type Output = Result<(), MyCustomJoinError>; +/// # +/// # fn poll(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { +/// # unreachable!() +/// # } +/// # } +/// # +/// # struct MyCustomRuntime; +/// # +/// # impl MyCustomRuntime { +/// # async fn sleep(_: Duration) { +/// # unreachable!() +/// # } +/// # } +/// # +/// # impl Runtime for MyCustomRuntime { +/// # type JoinError = MyCustomJoinError; +/// # type JoinHandle = MyCustomJoinHandle; +/// # +/// # fn scope(_event_loop: PyObject, fut: F) -> Pin + Send>> +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # fn get_task_event_loop(py: Python) -> Option<&PyAny> { +/// # unreachable!() +/// # } +/// # +/// # fn spawn(fut: F) -> Self::JoinHandle +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # } +/// # +/// # impl SpawnLocalExt for MyCustomRuntime { +/// # fn scope_local(_event_loop: PyObject, fut: F) -> Pin>> +/// # where +/// # F: Future + 'static +/// # { +/// # unreachable!() +/// # } +/// # +/// # fn spawn_local(fut: F) -> Self::JoinHandle +/// # where +/// # F: Future + 'static +/// # { +/// # unreachable!() +/// # } +/// # } +/// # +/// use std::time::Duration; +/// +/// use pyo3::prelude::*; +/// +/// /// Awaitable sleep function +/// #[pyfunction] +/// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { +/// pyo3_asyncio::generic::local_future_into_py_with_loop::( +/// pyo3_asyncio::get_event_loop(py)?, +/// async move { +/// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// } +/// ) +/// } +/// ``` +pub fn local_future_into_py(py: Python, fut: F) -> PyResult<&PyAny> +where + R: SpawnLocalExt, + F: Future> + 'static, +{ + local_future_into_py_with_loop::(current_event_loop::(py)?, fut) +} diff --git a/src/tokio.rs b/src/tokio.rs index 3819110..22d4648 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -6,9 +6,12 @@ use ::tokio::{ }; use futures::future::pending; use once_cell::{sync::OnceCell, unsync::OnceCell as UnsyncOnceCell}; -use pyo3::prelude::*; +use pyo3::{prelude::*, PyNativeType}; -use crate::generic::{self, Runtime as GenericRuntime, SpawnLocalExt}; +use crate::{ + generic::{self, Runtime as GenericRuntime, SpawnLocalExt}, + into_future_with_loop, +}; /// attributes /// re-exports for macros @@ -60,8 +63,8 @@ impl GenericRuntime for TokioRuntime { Box::pin(EVENT_LOOP.scope(cell, fut)) } - fn get_task_event_loop() -> Option { - EVENT_LOOP.with(|c| c.get().map(|event_loop| event_loop.clone())) + fn get_task_event_loop(py: Python) -> Option<&PyAny> { + EVENT_LOOP.with(|c| c.get().map(|event_loop| event_loop.clone().into_ref(py))) } fn spawn(fut: F) -> Self::JoinHandle @@ -109,9 +112,14 @@ where TokioRuntime::scope_local(event_loop, fut).await } +/// Get the current event loop from either Python or Rust async task local context +pub fn current_event_loop(py: Python) -> PyResult<&PyAny> { + generic::current_event_loop::(py) +} + /// Get the task local event loop for the current tokio task -pub fn task_event_loop() -> Option { - TokioRuntime::get_task_event_loop() +pub fn task_event_loop(py: Python) -> Option<&PyAny> { + TokioRuntime::get_task_event_loop(py) } /// Initialize the Tokio Runtime with a custom build @@ -253,10 +261,13 @@ where /// fn sleep_for(py: Python, secs: &PyAny) -> PyResult { /// let secs = secs.extract()?; /// -/// pyo3_asyncio::tokio::into_coroutine(pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), async move { -/// tokio::time::sleep(Duration::from_secs(secs)).await; -/// Python::with_gil(|py| Ok(py.None())) -/// }) +/// pyo3_asyncio::tokio::into_coroutine( +/// pyo3_asyncio::tokio::current_event_loop(py)?, +/// async move { +/// tokio::time::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// } +/// ) /// } /// ``` pub fn into_coroutine(event_loop: &PyAny, fut: F) -> PyResult @@ -281,21 +292,25 @@ where /// /// /// Awaitable non-send sleep function /// #[pyfunction] -/// fn sleep_for(py: Python, secs: u64) -> PyResult { +/// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { /// // Rc is non-send so it cannot be passed into pyo3_asyncio::tokio::into_coroutine /// let secs = Rc::new(secs); -/// let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); /// -/// Ok(pyo3_asyncio::tokio::local_future_into_py_with_loop(event_loop.as_ref(py), async move { -/// tokio::time::sleep(Duration::from_secs(*secs)).await; -/// Python::with_gil(|py| Ok(py.None())) -/// })?.into()) +/// pyo3_asyncio::tokio::local_future_into_py_with_loop( +/// pyo3_asyncio::tokio::current_event_loop(py)?, +/// async move { +/// tokio::time::sleep(Duration::from_secs(*secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// } +/// ) /// } /// /// # #[cfg(all(feature = "tokio-runtime", feature = "attributes"))] /// #[pyo3_asyncio::tokio::main] /// async fn main() -> PyResult<()> { -/// let event_loop = pyo3_asyncio::tokio::task_event_loop().unwrap(); +/// let event_loop = Python::with_gil(|py| { +/// PyObject::from(pyo3_asyncio::tokio::task_event_loop(py).unwrap()) +/// }); /// /// // the main coroutine is running in a Send context, so we cannot use LocalSet here. Instead /// // we use spawn_blocking in order to use LocalSet::block_on @@ -307,10 +322,7 @@ where /// pyo3_asyncio::tokio::scope_local(event_loop, async { /// Python::with_gil(|py| { /// let py_future = sleep_for(py, 1)?; -/// pyo3_asyncio::into_future_with_loop( -/// pyo3_asyncio::tokio::task_event_loop().unwrap().as_ref(py), -/// py_future.as_ref(py) -/// ) +/// pyo3_asyncio::tokio::into_future(py_future) /// })? /// .await?; /// @@ -328,3 +340,115 @@ where { generic::local_future_into_py_with_loop::(event_loop, fut) } + +/// Convert a `!Send` Rust Future into a Python awaitable +/// +/// # Arguments +/// * `py` - The current PyO3 GIL guard +/// * `fut` - The Rust future to be converted +/// +/// # Examples +/// +/// ``` +/// use std::{rc::Rc, time::Duration}; +/// +/// use pyo3::prelude::*; +/// +/// /// Awaitable non-send sleep function +/// #[pyfunction] +/// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { +/// // Rc is non-send so it cannot be passed into pyo3_asyncio::tokio::into_coroutine +/// let secs = Rc::new(secs); +/// pyo3_asyncio::tokio::local_future_into_py(py, async move { +/// tokio::time::sleep(Duration::from_secs(*secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// }) +/// } +/// +/// # #[cfg(all(feature = "tokio-runtime", feature = "attributes"))] +/// #[pyo3_asyncio::tokio::main] +/// async fn main() -> PyResult<()> { +/// let event_loop = Python::with_gil(|py| { +/// PyObject::from(pyo3_asyncio::tokio::task_event_loop(py).unwrap()) +/// }); +/// +/// // the main coroutine is running in a Send context, so we cannot use LocalSet here. Instead +/// // we use spawn_blocking in order to use LocalSet::block_on +/// tokio::task::spawn_blocking(move || { +/// // LocalSet allows us to work with !Send futures within tokio. Without it, any calls to +/// // pyo3_asyncio::tokio::local_future_into_py will panic. +/// tokio::task::LocalSet::new().block_on( +/// pyo3_asyncio::tokio::get_runtime(), +/// pyo3_asyncio::tokio::scope_local(event_loop, async { +/// Python::with_gil(|py| { +/// let py_future = sleep_for(py, 1)?; +/// pyo3_asyncio::tokio::into_future(py_future) +/// })? +/// .await?; +/// +/// Ok(()) +/// }) +/// ) +/// }).await.unwrap() +/// } +/// # #[cfg(not(all(feature = "tokio-runtime", feature = "attributes")))] +/// # fn main() {} +/// ``` +pub fn local_future_into_py(py: Python, fut: F) -> PyResult<&PyAny> +where + F: Future> + 'static, +{ + generic::local_future_into_py::(py, fut) +} + +/// Convert a Python `awaitable` into a Rust Future +/// +/// This function converts the `awaitable` into a Python Task using `run_coroutine_threadsafe`. A +/// completion handler sends the result of this Task through a +/// `futures::channel::oneshot::Sender>` and the future returned by this function +/// simply awaits the result through the `futures::channel::oneshot::Receiver>`. +/// +/// # Arguments +/// * `awaitable` - The Python `awaitable` to be converted +/// +/// # Examples +/// +/// ``` +/// use std::time::Duration; +/// +/// use pyo3::prelude::*; +/// +/// const PYTHON_CODE: &'static str = r#" +/// import asyncio +/// +/// async def py_sleep(duration): +/// await asyncio.sleep(duration) +/// "#; +/// +/// async fn py_sleep(seconds: f32) -> PyResult<()> { +/// let test_mod = Python::with_gil(|py| -> PyResult { +/// Ok( +/// PyModule::from_code( +/// py, +/// PYTHON_CODE, +/// "test_into_future/test_mod.py", +/// "test_mod" +/// )? +/// .into() +/// ) +/// })?; +/// +/// Python::with_gil(|py| { +/// pyo3_asyncio::tokio::into_future( +/// test_mod +/// .call_method1(py, "py_sleep", (seconds.into_py(py),))? +/// .as_ref(py), +/// ) +/// })? +/// .await?; +/// Ok(()) +/// } +/// ``` +pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { + into_future_with_loop(current_event_loop(awaitable.py())?, awaitable) +} From 9c4be342b6e61b2f638ca794fe5ebc846cf114b3 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Fri, 2 Jul 2021 17:55:21 -0500 Subject: [PATCH 08/32] Changed get_task_event_loop implementations to return None if called outside a task --- README.md | 4 +- pytests/test_async_std_asyncio.rs | 5 +- pytests/tokio_asyncio/mod.rs | 5 +- src/async_std.rs | 50 ++++++-- src/generic.rs | 183 ++++++++++++++++++++++++++++-- src/tokio.rs | 21 ++-- 6 files changed, 230 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 55ae2fd..4606e82 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] fn rust_sleep(py: Python) -> PyResult { - pyo3_asyncio::async_std::into_coroutine(pyo3_asyncio::async_std::current_event_loop(py)?, async { + pyo3_asyncio::async_std::into_coroutine(py, async { async_std::task::sleep(std::time::Duration::from_secs(1)).await; Ok(Python::with_gil(|py| py.None())) }) @@ -149,7 +149,7 @@ use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] fn rust_sleep(py: Python) -> PyResult { - pyo3_asyncio::tokio::into_coroutine(pyo3_asyncio::tokio::current_event_loop(py)?, async { + pyo3_asyncio::tokio::into_coroutine(py, async { tokio::time::sleep(std::time::Duration::from_secs(1)).await; Ok(Python::with_gil(|py| py.None())) }) diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index c962755..8ba7d84 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -9,7 +9,7 @@ use pyo3::{prelude::*, wrap_pyfunction}; fn sleep_for(py: Python, secs: &PyAny) -> PyResult { let secs = secs.extract()?; - pyo3_asyncio::async_std::into_coroutine(pyo3_asyncio::get_event_loop(py)?, async move { + pyo3_asyncio::async_std::into_coroutine(py, async move { task::sleep(Duration::from_secs(secs)).await; Python::with_gil(|py| Ok(py.None())) }) @@ -83,9 +83,8 @@ fn test_init_twice(_event_loop: PyObject) -> PyResult<()> { #[pyo3_asyncio::async_std::test] async fn test_panic() -> PyResult<()> { let fut = Python::with_gil(|py| -> PyResult<_> { - let event_loop = pyo3_asyncio::async_std::task_event_loop(py).unwrap(); pyo3_asyncio::async_std::into_future( - pyo3_asyncio::async_std::into_coroutine(event_loop, async { + pyo3_asyncio::async_std::into_coroutine(py, async { panic!("this panic was intentional!") })? .as_ref(py), diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index ba7c0c5..0ae4a93 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -8,7 +8,7 @@ use crate::common; fn sleep_for(py: Python, secs: &PyAny) -> PyResult { let secs = secs.extract()?; - pyo3_asyncio::tokio::into_coroutine(pyo3_asyncio::get_event_loop(py)?, async move { + pyo3_asyncio::tokio::into_coroutine(py, async move { tokio::time::sleep(Duration::from_secs(secs)).await; Python::with_gil(|py| Ok(py.None())) }) @@ -114,9 +114,8 @@ fn test_local_set_coroutine(event_loop: PyObject) -> PyResult<()> { #[pyo3_asyncio::tokio::test] async fn test_panic() -> PyResult<()> { let fut = Python::with_gil(|py| -> PyResult<_> { - let event_loop = pyo3_asyncio::tokio::task_event_loop(py).unwrap(); pyo3_asyncio::tokio::into_future( - pyo3_asyncio::tokio::into_coroutine(event_loop, async { + pyo3_asyncio::tokio::into_coroutine(py, async { panic!("this panic was intentional!") })? .as_ref(py), diff --git a/src/async_std.rs b/src/async_std.rs index 3d35b56..08b610d 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -54,7 +54,10 @@ impl Runtime for AsyncStdRuntime { Box::pin(fut) } fn get_task_event_loop(py: Python) -> Option<&PyAny> { - EVENT_LOOP.with(|c| c.get().map(|event_loop| event_loop.clone().into_ref(py))) + match EVENT_LOOP.try_with(|c| c.get().map(|event_loop| event_loop.clone().into_ref(py))) { + Ok(event_loop) => event_loop, + Err(_) => None, + } } fn spawn(fut: F) -> Self::JoinHandle @@ -173,20 +176,47 @@ where /// fn sleep_for(py: Python, secs: &PyAny) -> PyResult { /// let secs = secs.extract()?; /// -/// pyo3_asyncio::async_std::into_coroutine( -/// pyo3_asyncio::get_event_loop(py)?, -/// async move { -/// async_std::task::sleep(Duration::from_secs(secs)).await; -/// Python::with_gil(|py| Ok(py.None())) -/// } -/// ) +/// pyo3_asyncio::async_std::into_coroutine(py, async move { +/// async_std::task::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// }) +/// } +/// ``` +pub fn into_coroutine(py: Python, fut: F) -> PyResult +where + F: Future> + Send + 'static, +{ + generic::into_coroutine::(py, fut) +} + +/// Convert a Rust Future into a Python awaitable +/// +/// # Arguments +/// * `py` - The current PyO3 GIL guard +/// * `fut` - The Rust future to be converted +/// +/// # Examples +/// +/// ``` +/// use std::time::Duration; +/// +/// use pyo3::prelude::*; +/// +/// /// Awaitable sleep function +/// #[pyfunction] +/// fn sleep_for<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { +/// let secs = secs.extract()?; +/// pyo3_asyncio::async_std::future_into_py(py, async move { +/// async_std::task::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// }) /// } /// ``` -pub fn into_coroutine(event_loop: &PyAny, fut: F) -> PyResult +pub fn future_into_py(py: Python, fut: F) -> PyResult<&PyAny> where F: Future> + Send + 'static, { - generic::into_coroutine::(event_loop, fut) + generic::future_into_py::(py, fut) } /// Convert a `!Send` Rust Future into a Python awaitable diff --git a/src/generic.rs b/src/generic.rs index dc97c5c..f7f4632 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -139,7 +139,7 @@ where { let event_loop = get_event_loop(py)?; - let coro = into_coroutine::(event_loop, async move { + let coro = future_into_py_with_loop::(event_loop, async move { fut.await?; Ok(Python::with_gil(|py| py.None())) })?; @@ -231,11 +231,10 @@ fn set_result(event_loop: &PyAny, future: &PyAny, result: PyResult) -> /// /// /// Awaitable sleep function /// #[pyfunction] -/// fn sleep_for(py: Python, secs: &PyAny) -> PyResult { +/// fn sleep_for<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { /// let secs = secs.extract()?; -/// -/// pyo3_asyncio::generic::into_coroutine::( -/// pyo3_asyncio::get_event_loop(py)?, +/// pyo3_asyncio::generic::future_into_py_with_loop::( +/// pyo3_asyncio::generic::current_event_loop::(py)?, /// async move { /// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; /// Python::with_gil(|py| Ok(py.None())) @@ -243,14 +242,14 @@ fn set_result(event_loop: &PyAny, future: &PyAny, result: PyResult) -> /// ) /// } /// ``` -pub fn into_coroutine(event_loop: &PyAny, fut: F) -> PyResult +pub fn future_into_py_with_loop(event_loop: &PyAny, fut: F) -> PyResult<&PyAny> where R: Runtime, F: Future> + Send + 'static, { - let future_rx: PyObject = create_future(event_loop)?.into(); - let future_tx1 = future_rx.clone(); - let future_tx2 = future_rx.clone(); + let future_rx = create_future(event_loop)?; + let future_tx1 = PyObject::from(future_rx); + let future_tx2 = future_tx1.clone(); let event_loop = PyObject::from(event_loop); @@ -292,6 +291,172 @@ where Ok(future_rx) } +/// Convert a Rust Future into a Python awaitable with a generic runtime +/// +/// # Arguments +/// * `py` - The current PyO3 GIL guard +/// * `fut` - The Rust future to be converted +/// +/// # Examples +/// +/// ```no_run +/// # use std::{task::{Context, Poll}, pin::Pin, future::Future}; +/// # +/// # use pyo3_asyncio::generic::{JoinError, Runtime}; +/// # +/// # struct MyCustomJoinError; +/// # +/// # impl JoinError for MyCustomJoinError { +/// # fn is_panic(&self) -> bool { +/// # unreachable!() +/// # } +/// # } +/// # +/// # struct MyCustomJoinHandle; +/// # +/// # impl Future for MyCustomJoinHandle { +/// # type Output = Result<(), MyCustomJoinError>; +/// # +/// # fn poll(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { +/// # unreachable!() +/// # } +/// # } +/// # +/// # struct MyCustomRuntime; +/// # +/// # impl MyCustomRuntime { +/// # async fn sleep(_: Duration) { +/// # unreachable!() +/// # } +/// # } +/// # +/// # impl Runtime for MyCustomRuntime { +/// # type JoinError = MyCustomJoinError; +/// # type JoinHandle = MyCustomJoinHandle; +/// # +/// # fn scope(_event_loop: PyObject, fut: F) -> Pin + Send>> +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # fn get_task_event_loop(py: Python) -> Option<&PyAny> { +/// # unreachable!() +/// # } +/// # +/// # fn spawn(fut: F) -> Self::JoinHandle +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # } +/// # +/// use std::time::Duration; +/// +/// use pyo3::prelude::*; +/// +/// /// Awaitable sleep function +/// #[pyfunction] +/// fn sleep_for<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { +/// let secs = secs.extract()?; +/// pyo3_asyncio::generic::future_into_py::(py, async move { +/// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// }) +/// } +/// ``` +pub fn future_into_py(py: Python, fut: F) -> PyResult<&PyAny> +where + R: Runtime, + F: Future> + Send + 'static, +{ + future_into_py_with_loop::(current_event_loop::(py)?, fut) +} + +/// Convert a Rust Future into a Python awaitable with a generic runtime +/// +/// # Arguments +/// * `py` - The current PyO3 GIL guard +/// * `fut` - The Rust future to be converted +/// +/// # Examples +/// +/// ```no_run +/// # use std::{task::{Context, Poll}, pin::Pin, future::Future}; +/// # +/// # use pyo3_asyncio::generic::{JoinError, Runtime}; +/// # +/// # struct MyCustomJoinError; +/// # +/// # impl JoinError for MyCustomJoinError { +/// # fn is_panic(&self) -> bool { +/// # unreachable!() +/// # } +/// # } +/// # +/// # struct MyCustomJoinHandle; +/// # +/// # impl Future for MyCustomJoinHandle { +/// # type Output = Result<(), MyCustomJoinError>; +/// # +/// # fn poll(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { +/// # unreachable!() +/// # } +/// # } +/// # +/// # struct MyCustomRuntime; +/// # +/// # impl MyCustomRuntime { +/// # async fn sleep(_: Duration) { +/// # unreachable!() +/// # } +/// # } +/// # +/// # impl Runtime for MyCustomRuntime { +/// # type JoinError = MyCustomJoinError; +/// # type JoinHandle = MyCustomJoinHandle; +/// # +/// # fn scope(_event_loop: PyObject, fut: F) -> Pin + Send>> +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # fn get_task_event_loop(py: Python) -> Option<&PyAny> { +/// # unreachable!() +/// # } +/// # +/// # fn spawn(fut: F) -> Self::JoinHandle +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # } +/// # +/// use std::time::Duration; +/// +/// use pyo3::prelude::*; +/// +/// /// Awaitable sleep function +/// #[pyfunction] +/// fn sleep_for(py: Python, secs: &PyAny) -> PyResult { +/// let secs = secs.extract()?; +/// pyo3_asyncio::generic::into_coroutine::(py, async move { +/// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// }) +/// } +/// ``` +pub fn into_coroutine(py: Python, fut: F) -> PyResult +where + R: Runtime, + F: Future> + Send + 'static, +{ + Ok(future_into_py::(py, fut)?.into()) +} + /// Convert a `!Send` Rust Future into a Python awaitable with a generic runtime /// /// # Arguments diff --git a/src/tokio.rs b/src/tokio.rs index 22d4648..01289b5 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -64,7 +64,10 @@ impl GenericRuntime for TokioRuntime { } fn get_task_event_loop(py: Python) -> Option<&PyAny> { - EVENT_LOOP.with(|c| c.get().map(|event_loop| event_loop.clone().into_ref(py))) + match EVENT_LOOP.try_with(|c| c.get().map(|event_loop| event_loop.clone().into_ref(py))) { + Ok(event_loop) => event_loop, + Err(_) => None, + } } fn spawn(fut: F) -> Self::JoinHandle @@ -260,21 +263,17 @@ where /// #[pyfunction] /// fn sleep_for(py: Python, secs: &PyAny) -> PyResult { /// let secs = secs.extract()?; -/// -/// pyo3_asyncio::tokio::into_coroutine( -/// pyo3_asyncio::tokio::current_event_loop(py)?, -/// async move { -/// tokio::time::sleep(Duration::from_secs(secs)).await; -/// Python::with_gil(|py| Ok(py.None())) -/// } -/// ) +/// pyo3_asyncio::tokio::into_coroutine(py, async move { +/// tokio::time::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// }) /// } /// ``` -pub fn into_coroutine(event_loop: &PyAny, fut: F) -> PyResult +pub fn into_coroutine(py: Python, fut: F) -> PyResult where F: Future> + Send + 'static, { - generic::into_coroutine::(event_loop, fut) + generic::into_coroutine::(py, fut) } /// Convert a `!Send` Rust Future into a Python awaitable From 8c49ffdb0ea3eda46c58c8b33591dd39b58a91a9 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Fri, 2 Jul 2021 19:05:30 -0500 Subject: [PATCH 09/32] Added new try_init/try_close get_cached_event_loop behaviour --- pyo3-asyncio-macros/src/lib.rs | 14 ++-- pyo3-asyncio-macros/src/tokio.rs | 14 ++-- pytests/common/mod.rs | 8 --- pytests/test_async_std_asyncio.rs | 5 -- pytests/test_async_std_run_forever.rs | 58 ++++++++-------- pytests/tokio_asyncio/mod.rs | 5 -- pytests/tokio_run_forever/mod.rs | 58 ++++++++-------- src/async_std.rs | 33 +++++++++ src/generic.rs | 98 +++++++++++++++++++++++++++ src/lib.rs | 51 +++++++++----- src/tokio.rs | 33 +++++++++ 11 files changed, 261 insertions(+), 116 deletions(-) diff --git a/pyo3-asyncio-macros/src/lib.rs b/pyo3-asyncio-macros/src/lib.rs index e3e2679..7a83eb8 100644 --- a/pyo3-asyncio-macros/src/lib.rs +++ b/pyo3-asyncio-macros/src/lib.rs @@ -50,15 +50,11 @@ pub fn async_std_main(_attr: TokenStream, item: TokenStream) -> TokenStream { } pyo3::Python::with_gil(|py| { - pyo3_asyncio::with_runtime(py, || { - pyo3_asyncio::async_std::run_until_complete(py, main())?; - - Ok(()) - }) - .map_err(|e| { - e.print_and_set_sys_last_vars(py); - }) - .unwrap(); + pyo3_asyncio::async_std::run(py, main()) + .map_err(|e| { + e.print_and_set_sys_last_vars(py); + }) + .unwrap(); }); } }; diff --git a/pyo3-asyncio-macros/src/tokio.rs b/pyo3-asyncio-macros/src/tokio.rs index 98d711e..5eaf1f0 100644 --- a/pyo3-asyncio-macros/src/tokio.rs +++ b/pyo3-asyncio-macros/src/tokio.rs @@ -257,15 +257,11 @@ fn parse_knobs( #rt_init pyo3::Python::with_gil(|py| { - pyo3_asyncio::with_runtime(py, || { - pyo3_asyncio::tokio::run_until_complete(py, main())?; - - Ok(()) - }) - .map_err(|e| { - e.print_and_set_sys_last_vars(py); - }) - .unwrap(); + pyo3_asyncio::tokio::run(py, main()) + .map_err(|e| { + e.print_and_set_sys_last_vars(py); + }) + .unwrap(); }); } }; diff --git a/pytests/common/mod.rs b/pytests/common/mod.rs index dc388aa..d010dcb 100644 --- a/pytests/common/mod.rs +++ b/pytests/common/mod.rs @@ -54,11 +54,3 @@ pub(super) async fn test_other_awaitables(event_loop: PyObject) -> PyResult<()> Ok(()) } - -pub(super) fn test_init_twice() -> PyResult<()> { - // try_init has already been called in test main - ensure a second call doesn't mess the other - // tests up - Python::with_gil(|py| pyo3_asyncio::try_init(py))?; - - Ok(()) -} diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index 8ba7d84..c9cb874 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -75,11 +75,6 @@ async fn test_other_awaitables() -> PyResult<()> { .await } -#[pyo3_asyncio::async_std::test] -fn test_init_twice(_event_loop: PyObject) -> PyResult<()> { - common::test_init_twice() -} - #[pyo3_asyncio::async_std::test] async fn test_panic() -> PyResult<()> { let fut = Python::with_gil(|py| -> PyResult<_> { diff --git a/pytests/test_async_std_run_forever.rs b/pytests/test_async_std_run_forever.rs index ce7206d..03d11ec 100644 --- a/pytests/test_async_std_run_forever.rs +++ b/pytests/test_async_std_run_forever.rs @@ -2,44 +2,40 @@ use std::time::Duration; use pyo3::prelude::*; -fn dump_err(py: Python<'_>) -> impl FnOnce(PyErr) + '_ { - move |e| { - // We can't display Python exceptions via std::fmt::Display, - // so print the error here manually. - e.print_and_set_sys_last_vars(py); - } +fn dump_err(py: Python, e: PyErr) { + // We can't display Python exceptions via std::fmt::Display, + // so print the error here manually. + e.print_and_set_sys_last_vars(py); } fn main() { Python::with_gil(|py| { - pyo3_asyncio::with_runtime(py, || { - let event_loop = PyObject::from(pyo3_asyncio::get_event_loop(py).unwrap()); + let event_loop = PyObject::from(pyo3_asyncio::get_event_loop(py).unwrap()); - async_std::task::spawn(async move { - async_std::task::sleep(Duration::from_secs(1)).await; + async_std::task::spawn(async move { + async_std::task::sleep(Duration::from_secs(1)).await; - Python::with_gil(|py| { - event_loop - .as_ref(py) - .call_method1( - "call_soon_threadsafe", - (event_loop - .as_ref(py) - .getattr("stop") - .map_err(dump_err(py)) - .unwrap(),), - ) - .map_err(dump_err(py)) - .unwrap(); - }) - }); + Python::with_gil(|py| { + event_loop + .as_ref(py) + .call_method1( + "call_soon_threadsafe", + (event_loop + .as_ref(py) + .getattr("stop") + .map_err(|e| dump_err(py, e)) + .unwrap(),), + ) + .map_err(|e| dump_err(py, e)) + .unwrap(); + }) + }); - pyo3_asyncio::run_forever(py)?; + pyo3_asyncio::run_forever(py)?; - println!("test test_run_forever ... ok"); - Ok(()) - }) - .map_err(dump_err(py)) - .unwrap(); + println!("test test_run_forever ... ok"); + Ok(()) }) + .map_err(|e| Python::with_gil(|py| dump_err(py, e))) + .unwrap() } diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index 0ae4a93..dc85d82 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -74,11 +74,6 @@ async fn test_other_awaitables() -> PyResult<()> { .await } -#[pyo3_asyncio::tokio::test] -fn test_init_twice(_event_loop: PyObject) -> PyResult<()> { - common::test_init_twice() -} - #[pyo3_asyncio::tokio::test] fn test_init_tokio_twice(_event_loop: PyObject) -> PyResult<()> { // tokio has already been initialized in test main. call these functions to diff --git a/pytests/tokio_run_forever/mod.rs b/pytests/tokio_run_forever/mod.rs index 60f79fe..b798f3b 100644 --- a/pytests/tokio_run_forever/mod.rs +++ b/pytests/tokio_run_forever/mod.rs @@ -2,44 +2,40 @@ use std::time::Duration; use pyo3::prelude::*; -fn dump_err(py: Python<'_>) -> impl FnOnce(PyErr) + '_ { - move |e| { - // We can't display Python exceptions via std::fmt::Display, - // so print the error here manually. - e.print_and_set_sys_last_vars(py); - } +fn dump_err(py: Python<'_>, e: PyErr) { + // We can't display Python exceptions via std::fmt::Display, + // so print the error here manually. + e.print_and_set_sys_last_vars(py); } pub(super) fn test_main() { Python::with_gil(|py| { - pyo3_asyncio::with_runtime(py, || { - let event_loop = PyObject::from(pyo3_asyncio::get_event_loop(py).unwrap()); + let event_loop = PyObject::from(pyo3_asyncio::get_event_loop(py).unwrap()); - pyo3_asyncio::tokio::get_runtime().spawn(async move { - tokio::time::sleep(Duration::from_secs(1)).await; + pyo3_asyncio::tokio::get_runtime().spawn(async move { + tokio::time::sleep(Duration::from_secs(1)).await; - Python::with_gil(|py| { - event_loop - .as_ref(py) - .call_method1( - "call_soon_threadsafe", - (event_loop - .as_ref(py) - .getattr("stop") - .map_err(dump_err(py)) - .unwrap(),), - ) - .map_err(dump_err(py)) - .unwrap(); - }) - }); + Python::with_gil(|py| { + event_loop + .as_ref(py) + .call_method1( + "call_soon_threadsafe", + (event_loop + .as_ref(py) + .getattr("stop") + .map_err(|e| dump_err(py, e)) + .unwrap(),), + ) + .map_err(|e| dump_err(py, e)) + .unwrap(); + }) + }); - pyo3_asyncio::run_forever(py)?; + pyo3_asyncio::run_forever(py)?; - println!("test test_run_forever ... ok"); - Ok(()) - }) - .map_err(dump_err(py)) - .unwrap(); + println!("test test_run_forever ... ok"); + Ok(()) }) + .map_err(|e| Python::with_gil(|py| dump_err(py, e))) + .unwrap(); } diff --git a/src/async_std.rs b/src/async_std.rs index 08b610d..152c31e 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -158,6 +158,39 @@ where generic::run_until_complete::(py, fut) } +/// Run the event loop until the given Future completes +/// +/// # Arguments +/// * `py` - The current PyO3 GIL guard +/// * `fut` - The future to drive to completion +/// +/// # Examples +/// +/// ```no_run +/// # use std::time::Duration; +/// # +/// # use pyo3::prelude::*; +/// # +/// fn main() { +/// Python::with_gil(|py| { +/// pyo3_asyncio::async_std::run(py, async move { +/// async_std::task::sleep(Duration::from_secs(1)).await; +/// Ok(()) +/// }) +/// .map_err(|e| { +/// e.print_and_set_sys_last_vars(py); +/// }) +/// .unwrap(); +/// }) +/// } +/// ``` +pub fn run(py: Python, fut: F) -> PyResult<()> +where + F: Future> + Send + 'static, +{ + generic::run::(py, fut) +} + /// Convert a Rust Future into a Python awaitable /// /// # Arguments diff --git a/src/generic.rs b/src/generic.rs index f7f4632..ae16e74 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -149,6 +149,104 @@ where Ok(()) } +/// Run the event loop until the given Future completes +/// +/// # Arguments +/// * `py` - The current PyO3 GIL guard +/// * `fut` - The future to drive to completion +/// +/// # Examples +/// +/// ```no_run +/// # use std::{task::{Context, Poll}, pin::Pin, future::Future}; +/// # +/// # use pyo3_asyncio::generic::{JoinError, Runtime}; +/// # +/// # struct MyCustomJoinError; +/// # +/// # impl JoinError for MyCustomJoinError { +/// # fn is_panic(&self) -> bool { +/// # unreachable!() +/// # } +/// # } +/// # +/// # struct MyCustomJoinHandle; +/// # +/// # impl Future for MyCustomJoinHandle { +/// # type Output = Result<(), MyCustomJoinError>; +/// # +/// # fn poll(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { +/// # unreachable!() +/// # } +/// # } +/// # +/// # struct MyCustomRuntime; +/// # +/// # impl Runtime for MyCustomRuntime { +/// # type JoinError = MyCustomJoinError; +/// # type JoinHandle = MyCustomJoinHandle; +/// # +/// # fn scope(_event_loop: PyObject, fut: F) -> Pin + Send>> +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # fn get_task_event_loop(py: Python) -> Option<&PyAny> { +/// # unreachable!() +/// # } +/// # +/// # fn spawn(fut: F) -> Self::JoinHandle +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # } +/// # +/// # use std::time::Duration; +/// # async fn custom_sleep(_duration: Duration) { } +/// # +/// # use pyo3::prelude::*; +/// # +/// fn main() { +/// Python::with_gil(|py| { +/// pyo3_asyncio::generic::run::(py, async move { +/// custom_sleep(Duration::from_secs(1)).await; +/// Ok(()) +/// }) +/// .map_err(|e| { +/// e.print_and_set_sys_last_vars(py); +/// }) +/// .unwrap(); +/// }) +/// } +/// ``` +pub fn run(py: Python, fut: F) -> PyResult<()> +where + R: Runtime, + F: Future> + Send + 'static, +{ + let event_loop = get_event_loop(py)?; + + let result = run_until_complete::(py, fut); + + event_loop.call_method1( + "run_until_complete", + (event_loop.call_method0("shutdown_asyncgens")?,), + )?; + // how to do this prior to 3.9? + // event_loop.call_method1( + // "run_until_complete", + // (event_loop.call_method0("shutdown_default_executor")?,), + // )?; + event_loop.call_method0("close")?; + + result?; + + Ok(()) +} + fn set_result(event_loop: &PyAny, future: &PyAny, result: PyResult) -> PyResult<()> { match result { Ok(val) => { diff --git a/src/lib.rs b/src/lib.rs index df2b596..be183e9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -145,10 +145,11 @@ pub mod doc_test { doctest!("../README.md", readme_md); } -const EXPECT_INIT: &str = "PyO3 Asyncio has not been initialized"; - static ASYNCIO: OnceCell = OnceCell::new(); static ENSURE_FUTURE: OnceCell = OnceCell::new(); + +const EXPECT_INIT: &str = "PyO3 Asyncio has not been initialized"; +static CACHED_EVENT_LOOP: OnceCell = OnceCell::new(); static EXECUTOR: OnceCell = OnceCell::new(); fn ensure_future<'p>(py: Python<'p>, awaitable: &'p PyAny) -> PyResult<&'p PyAny> { @@ -210,17 +211,15 @@ where /// - Calling `try_init` a second time returns `Ok(())` and does nothing. /// > In future versions this may return an `Err`. pub fn try_init(py: Python) -> PyResult<()> { - ASYNCIO.get_or_try_init(|| -> PyResult { - let asyncio = py.import("asyncio")?; - let event_loop = asyncio.call_method0("get_event_loop")?; + CACHED_EVENT_LOOP.get_or_try_init(|| -> PyResult { + let event_loop = get_event_loop(py)?; let executor = py .import("concurrent.futures.thread")? - .getattr("ThreadPoolExecutor")? - .call0()?; + .call_method0("ThreadPoolExecutor")?; event_loop.call_method1("set_default_executor", (executor,))?; - EXECUTOR.get_or_init(|| executor.into()); - Ok(asyncio.into()) + EXECUTOR.set(executor.into()).unwrap(); + Ok(event_loop.into()) })?; Ok(()) @@ -237,6 +236,11 @@ pub fn get_event_loop(py: Python) -> PyResult<&PyAny> { asyncio(py)?.call_method0("get_event_loop") } +/// Get a reference to the Python event loop cached by `try_init` (0.13 behaviour) +pub fn get_cached_event_loop(py: Python) -> &PyAny { + CACHED_EVENT_LOOP.get().expect(EXPECT_INIT).as_ref(py) +} + /// Run the event loop forever /// /// This can be called instead of `run_until_complete` to run the event loop @@ -301,15 +305,26 @@ pub fn run_forever(py: Python) -> PyResult<()> { /// Shutdown the event loops and perform any necessary cleanup pub fn try_close(py: Python) -> PyResult<()> { - // Shutdown the executor and wait until all threads are cleaned up - EXECUTOR - .get() - .expect(EXPECT_INIT) - .call_method0(py, "shutdown")?; - - let event_loop = get_event_loop(py)?; - event_loop.call_method0("stop")?; - event_loop.call_method0("close")?; + if let Some(exec) = EXECUTOR.get() { + // Shutdown the executor and wait until all threads are cleaned up + exec.call_method0(py, "shutdown")?; + } + + if let Some(event_loop) = CACHED_EVENT_LOOP.get() { + let event_loop = event_loop.as_ref(py); + + event_loop.call_method1( + "run_until_complete", + (event_loop.call_method0("shutdown_asyncgens")?,), + )?; + // how to do this prior to 3.9? + // event_loop.call_method1( + // "run_until_complete", + // (event_loop.call_method0("shutdown_default_executor")?,), + // )?; + event_loop.call_method0("close")?; + } + Ok(()) } diff --git a/src/tokio.rs b/src/tokio.rs index 01289b5..8f4d0b8 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -246,6 +246,39 @@ where generic::run_until_complete::(py, fut) } +/// Run the event loop until the given Future completes +/// +/// # Arguments +/// * `py` - The current PyO3 GIL guard +/// * `fut` - The future to drive to completion +/// +/// # Examples +/// +/// ```no_run +/// # use std::time::Duration; +/// # +/// # use pyo3::prelude::*; +/// # +/// fn main() { +/// Python::with_gil(|py| { +/// pyo3_asyncio::tokio::run(py, async move { +/// tokio::time::sleep(Duration::from_secs(1)).await; +/// Ok(()) +/// }) +/// .map_err(|e| { +/// e.print_and_set_sys_last_vars(py); +/// }) +/// .unwrap(); +/// }) +/// } +/// ``` +pub fn run(py: Python, fut: F) -> PyResult<()> +where + F: Future> + Send + 'static, +{ + generic::run::(py, fut) +} + /// Convert a Rust Future into a Python awaitable /// /// # Arguments From 536ac93e982f08399f27ab863b06c47ca7b607ee Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Fri, 2 Jul 2021 19:30:50 -0500 Subject: [PATCH 10/32] Changed into_coroutine back to using cached event loop, 0.14 equivalent is future_into_py which uses non-cached event loop --- pytests/test_async_std_asyncio.rs | 18 ++++++------ pytests/test_tokio_current_thread_asyncio.rs | 12 ++++++-- pytests/test_tokio_multi_thread_asyncio.rs | 12 ++++++-- pytests/tokio_asyncio/mod.rs | 12 ++++---- src/generic.rs | 7 +++-- src/tokio.rs | 30 ++++++++++++++++++++ 6 files changed, 68 insertions(+), 23 deletions(-) diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index c9cb874..1fbd26b 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -78,12 +78,9 @@ async fn test_other_awaitables() -> PyResult<()> { #[pyo3_asyncio::async_std::test] async fn test_panic() -> PyResult<()> { let fut = Python::with_gil(|py| -> PyResult<_> { - pyo3_asyncio::async_std::into_future( - pyo3_asyncio::async_std::into_coroutine(py, async { - panic!("this panic was intentional!") - })? - .as_ref(py), - ) + pyo3_asyncio::async_std::into_future(pyo3_asyncio::async_std::future_into_py(py, async { + panic!("this panic was intentional!") + })?) })?; match fut.await { @@ -98,9 +95,12 @@ async fn test_panic() -> PyResult<()> { } } -#[pyo3_asyncio::async_std::main] -async fn main() -> pyo3::PyResult<()> { - pyo3_asyncio::testing::main().await +fn main() -> pyo3::PyResult<()> { + Python::with_gil(|py| { + // into_coroutine requires the 0.13 API + pyo3_asyncio::try_init(py)?; + pyo3_asyncio::async_std::run(py, pyo3_asyncio::testing::main()) + }) } #[pyo3_asyncio::async_std::test] diff --git a/pytests/test_tokio_current_thread_asyncio.rs b/pytests/test_tokio_current_thread_asyncio.rs index 75f1a94..40e8c39 100644 --- a/pytests/test_tokio_current_thread_asyncio.rs +++ b/pytests/test_tokio_current_thread_asyncio.rs @@ -1,7 +1,13 @@ mod common; mod tokio_asyncio; -#[pyo3_asyncio::tokio::main(flavor = "current_thread")] -async fn main() -> pyo3::PyResult<()> { - pyo3_asyncio::testing::main().await +use pyo3::prelude::*; + +fn main() -> pyo3::PyResult<()> { + Python::with_gil(|py| { + // into_coroutine requires the 0.13 API + pyo3_asyncio::try_init(py)?; + pyo3_asyncio::tokio::init_current_thread(); + pyo3_asyncio::tokio::run(py, pyo3_asyncio::testing::main()) + }) } diff --git a/pytests/test_tokio_multi_thread_asyncio.rs b/pytests/test_tokio_multi_thread_asyncio.rs index 11e582e..f880628 100644 --- a/pytests/test_tokio_multi_thread_asyncio.rs +++ b/pytests/test_tokio_multi_thread_asyncio.rs @@ -1,7 +1,13 @@ mod common; mod tokio_asyncio; -#[pyo3_asyncio::tokio::main] -async fn main() -> pyo3::PyResult<()> { - pyo3_asyncio::testing::main().await +use pyo3::prelude::*; + +fn main() -> pyo3::PyResult<()> { + Python::with_gil(|py| { + // into_coroutine requires the 0.13 API + pyo3_asyncio::try_init(py)?; + pyo3_asyncio::tokio::init_multi_thread(); + pyo3_asyncio::tokio::run(py, pyo3_asyncio::testing::main()) + }) } diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index dc85d82..d372271 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -17,6 +17,9 @@ fn sleep_for(py: Python, secs: &PyAny) -> PyResult { #[pyo3_asyncio::tokio::test] async fn test_into_coroutine() -> PyResult<()> { let fut = Python::with_gil(|py| { + // into_coroutine requires the 0.13 API + pyo3_asyncio::try_init(py)?; + let sleeper_mod = PyModule::new(py, "rust_sleeper")?; sleeper_mod.add_wrapped(wrap_pyfunction!(sleep_for))?; @@ -109,12 +112,9 @@ fn test_local_set_coroutine(event_loop: PyObject) -> PyResult<()> { #[pyo3_asyncio::tokio::test] async fn test_panic() -> PyResult<()> { let fut = Python::with_gil(|py| -> PyResult<_> { - pyo3_asyncio::tokio::into_future( - pyo3_asyncio::tokio::into_coroutine(py, async { - panic!("this panic was intentional!") - })? - .as_ref(py), - ) + pyo3_asyncio::tokio::into_future(pyo3_asyncio::tokio::future_into_py(py, async { + panic!("this panic was intentional!") + })?) })?; match fut.await { diff --git a/src/generic.rs b/src/generic.rs index ae16e74..c1b72dd 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -2,7 +2,10 @@ use std::{future::Future, pin::Pin}; use pyo3::prelude::*; -use crate::{call_soon_threadsafe, create_future, dump_err, err::RustPanic, get_event_loop}; +use crate::{ + call_soon_threadsafe, create_future, dump_err, err::RustPanic, get_cached_event_loop, + get_event_loop, +}; /// Generic utilities for a JoinError pub trait JoinError { @@ -552,7 +555,7 @@ where R: Runtime, F: Future> + Send + 'static, { - Ok(future_into_py::(py, fut)?.into()) + Ok(future_into_py_with_loop::(get_cached_event_loop(py), fut)?.into()) } /// Convert a `!Send` Rust Future into a Python awaitable with a generic runtime diff --git a/src/tokio.rs b/src/tokio.rs index 8f4d0b8..edb33db 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -309,6 +309,36 @@ where generic::into_coroutine::(py, fut) } +/// Convert a Rust Future into a Python awaitable +/// +/// # Arguments +/// * `py` - The current PyO3 GIL guard +/// * `fut` - The Rust future to be converted +/// +/// # Examples +/// +/// ``` +/// use std::time::Duration; +/// +/// use pyo3::prelude::*; +/// +/// /// Awaitable sleep function +/// #[pyfunction] +/// fn sleep_for<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { +/// let secs = secs.extract()?; +/// pyo3_asyncio::tokio::future_into_py(py, async move { +/// tokio::time::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// }) +/// } +/// ``` +pub fn future_into_py(py: Python, fut: F) -> PyResult<&PyAny> +where + F: Future> + Send + 'static, +{ + generic::future_into_py::(py, fut) +} + /// Convert a `!Send` Rust Future into a Python awaitable /// /// # Arguments From 130676542121d1a54408b422de525b5867aa70b1 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Fri, 2 Jul 2021 19:37:39 -0500 Subject: [PATCH 11/32] Added 0.13 into_future back in, makes use of get_cached_event_loop --- pytests/common/mod.rs | 13 ++++++++ pytests/test_async_std_asyncio.rs | 5 +++ pytests/tokio_asyncio/mod.rs | 5 +++ src/lib.rs | 55 ++++++++++++++++++++++++++++++- 4 files changed, 77 insertions(+), 1 deletion(-) diff --git a/pytests/common/mod.rs b/pytests/common/mod.rs index d010dcb..667d33f 100644 --- a/pytests/common/mod.rs +++ b/pytests/common/mod.rs @@ -28,6 +28,19 @@ pub(super) async fn test_into_future(event_loop: PyObject) -> PyResult<()> { Ok(()) } +pub(super) async fn test_into_future_0_13() -> PyResult<()> { + let fut = Python::with_gil(|py| { + let test_mod = + PyModule::from_code(py, TEST_MOD, "test_rust_coroutine/test_mod.py", "test_mod")?; + + pyo3_asyncio::into_future(test_mod.call_method1("py_sleep", (1.into_py(py),))?) + })?; + + fut.await?; + + Ok(()) +} + pub(super) fn test_blocking_sleep() -> PyResult<()> { thread::sleep(Duration::from_secs(1)); Ok(()) diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index 1fbd26b..fc3110c 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -67,6 +67,11 @@ async fn test_into_future() -> PyResult<()> { .await } +#[pyo3_asyncio::async_std::test] +async fn test_into_future_0_13() -> PyResult<()> { + common::test_into_future_0_13().await +} + #[pyo3_asyncio::async_std::test] async fn test_other_awaitables() -> PyResult<()> { common::test_other_awaitables(Python::with_gil(|py| { diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index d372271..a0b9d65 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -69,6 +69,11 @@ async fn test_into_future() -> PyResult<()> { .await } +#[pyo3_asyncio::tokio::test] +async fn test_into_future_0_13() -> PyResult<()> { + common::test_into_future_0_13().await +} + #[pyo3_asyncio::tokio::test] async fn test_other_awaitables() -> PyResult<()> { common::test_other_awaitables(Python::with_gil(|py| { diff --git a/src/lib.rs b/src/lib.rs index be183e9..e0a30ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -113,7 +113,7 @@ use std::future::Future; use futures::channel::oneshot; use once_cell::sync::OnceCell; -use pyo3::{exceptions::PyKeyboardInterrupt, prelude::*, types::PyTuple}; +use pyo3::{exceptions::PyKeyboardInterrupt, prelude::*, types::PyTuple, PyNativeType}; /// Re-exported for #[test] attributes #[cfg(all(feature = "attributes", feature = "testing"))] @@ -459,6 +459,59 @@ pub fn into_future_with_loop( }) } +/// Convert a Python `awaitable` into a Rust Future +/// +/// This function converts the `awaitable` into a Python Task using `run_coroutine_threadsafe`. A +/// completion handler sends the result of this Task through a +/// `futures::channel::oneshot::Sender>` and the future returned by this function +/// simply awaits the result through the `futures::channel::oneshot::Receiver>`. +/// +/// # Arguments +/// * `awaitable` - The Python `awaitable` to be converted +/// +/// # Examples +/// +/// ``` +/// use std::time::Duration; +/// +/// use pyo3::prelude::*; +/// +/// const PYTHON_CODE: &'static str = r#" +/// import asyncio +/// +/// async def py_sleep(duration): +/// await asyncio.sleep(duration) +/// "#; +/// +/// async fn py_sleep(seconds: f32) -> PyResult<()> { +/// let test_mod = Python::with_gil(|py| -> PyResult { +/// Ok( +/// PyModule::from_code( +/// py, +/// PYTHON_CODE, +/// "test_into_future/test_mod.py", +/// "test_mod" +/// )? +/// .into() +/// ) +/// })?; +/// +/// Python::with_gil(|py| { +/// // Only works with cached event loop +/// pyo3_asyncio::into_future( +/// test_mod +/// .call_method1(py, "py_sleep", (seconds.into_py(py),))? +/// .as_ref(py), +/// ) +/// })? +/// .await?; +/// Ok(()) +/// } +/// ``` +pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { + into_future_with_loop(get_cached_event_loop(awaitable.py()), awaitable) +} + fn dump_err(py: Python<'_>) -> impl FnOnce(PyErr) + '_ { move |e| { // We can't display Python exceptions via std::fmt::Display, From 2d315cf2bd5d59d4fd4da80ebce372fb2e8100da Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Wed, 7 Jul 2021 11:30:42 -0500 Subject: [PATCH 12/32] Made tokio initialization lazy, current thread scheduler init is a little more complicated now --- README.md | 5 +- pyo3-asyncio-macros/src/tokio.rs | 21 ++-- .../test_tokio_current_thread_run_forever.rs | 8 +- .../test_tokio_multi_thread_run_forever.rs | 2 - pytests/tokio_asyncio/mod.rs | 10 -- src/tokio.rs | 96 ++++--------------- 6 files changed, 38 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index 4102f2f..a2e5cb2 100644 --- a/README.md +++ b/README.md @@ -162,10 +162,7 @@ fn rust_sleep(py: Python) -> PyResult { #[pymodule] fn my_async_module(py: Python, m: &PyModule) -> PyResult<()> { pyo3_asyncio::try_init(py)?; - // Tokio needs explicit initialization before any pyo3-asyncio conversions. - // The module import is a prime place to do this. - pyo3_asyncio::tokio::init_multi_thread_once(); - + m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; Ok(()) diff --git a/pyo3-asyncio-macros/src/tokio.rs b/pyo3-asyncio-macros/src/tokio.rs index 98d711e..b30627e 100644 --- a/pyo3-asyncio-macros/src/tokio.rs +++ b/pyo3-asyncio-macros/src/tokio.rs @@ -219,7 +219,7 @@ fn parse_knobs( let config = config.build()?; - let mut rt = match config.flavor { + let builder = match config.flavor { RuntimeFlavor::CurrentThread => quote! { pyo3_asyncio::tokio::re_exports::runtime::Builder::new_current_thread() }, @@ -227,8 +227,15 @@ fn parse_knobs( pyo3_asyncio::tokio::re_exports::runtime::Builder::new_multi_thread() }, }; + + let mut builder_init = quote! { + builder.enable_all(); + }; if let Some(v) = config.worker_threads { - rt = quote! { #rt.worker_threads(#v) }; + builder_init = quote! { + builder.worker_threads(#v); + #builder_init; + }; } let rt_init = match config.flavor { @@ -247,12 +254,10 @@ fn parse_knobs( #body } - pyo3_asyncio::tokio::init( - #rt - .enable_all() - .build() - .unwrap() - ); + let mut builder = #builder; + #builder_init; + + pyo3_asyncio::tokio::init(builder); #rt_init diff --git a/pytests/test_tokio_current_thread_run_forever.rs b/pytests/test_tokio_current_thread_run_forever.rs index f2a72ad..439d506 100644 --- a/pytests/test_tokio_current_thread_run_forever.rs +++ b/pytests/test_tokio_current_thread_run_forever.rs @@ -1,7 +1,13 @@ mod tokio_run_forever; fn main() { - pyo3_asyncio::tokio::init_current_thread(); + let mut builder = tokio::runtime::Builder::new_current_thread(); + builder.enable_all(); + + pyo3_asyncio::tokio::init(builder); + std::thread::spawn(move || { + pyo3_asyncio::tokio::get_runtime().block_on(std::future::pending::<()>()); + }); tokio_run_forever::test_main(); } diff --git a/pytests/test_tokio_multi_thread_run_forever.rs b/pytests/test_tokio_multi_thread_run_forever.rs index 59d1ce3..243c1c6 100644 --- a/pytests/test_tokio_multi_thread_run_forever.rs +++ b/pytests/test_tokio_multi_thread_run_forever.rs @@ -1,7 +1,5 @@ mod tokio_run_forever; fn main() { - pyo3_asyncio::tokio::init_multi_thread(); - tokio_run_forever::test_main(); } diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index 8953d23..64da061 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -73,16 +73,6 @@ fn test_init_twice() -> PyResult<()> { common::test_init_twice() } -#[pyo3_asyncio::tokio::test] -fn test_init_tokio_twice() -> PyResult<()> { - // tokio has already been initialized in test main. call these functions to - // make sure they don't cause problems with the other tests. - pyo3_asyncio::tokio::init_multi_thread_once(); - pyo3_asyncio::tokio::init_current_thread_once(); - - Ok(()) -} - #[pyo3_asyncio::tokio::test] fn test_local_set_coroutine() -> PyResult<()> { tokio::task::LocalSet::new().block_on(pyo3_asyncio::tokio::get_runtime(), async { diff --git a/src/tokio.rs b/src/tokio.rs index c629918..d3e13bb 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -1,11 +1,10 @@ -use std::{future::Future, thread}; +use std::{future::Future, sync::Mutex}; use ::tokio::{ runtime::{Builder, Runtime}, task, }; -use futures::future::pending; -use once_cell::sync::OnceCell; +use once_cell::sync::{Lazy, OnceCell}; use pyo3::prelude::*; use crate::generic; @@ -30,10 +29,9 @@ pub use pyo3_asyncio_macros::tokio_main as main; #[cfg(all(feature = "attributes", feature = "testing"))] pub use pyo3_asyncio_macros::tokio_test as test; +static TOKIO_BUILDER: Lazy> = Lazy::new(|| Mutex::new(multi_thread())); static TOKIO_RUNTIME: OnceCell = OnceCell::new(); -const EXPECT_TOKIO_INIT: &str = "Tokio runtime must be initialized"; - impl generic::JoinError for task::JoinError { fn is_panic(&self) -> bool { task::JoinError::is_panic(self) @@ -65,79 +63,26 @@ impl generic::SpawnLocalExt for TokioRuntime { } } -/// Initialize the Tokio Runtime with a custom build -pub fn init(runtime: Runtime) { - TOKIO_RUNTIME - .set(runtime) - .expect("Tokio Runtime has already been initialized"); -} - -fn current_thread() -> Runtime { - Builder::new_current_thread() - .enable_all() - .build() - .expect("Couldn't build the current-thread Tokio runtime") -} - -fn start_current_thread() { - thread::spawn(move || { - TOKIO_RUNTIME.get().unwrap().block_on(pending::<()>()); - }); -} - -/// Initialize the Tokio Runtime with current-thread scheduler -/// -/// # Panics -/// This function will panic if called a second time. See [`init_current_thread_once`] if you want -/// to avoid this panic. -pub fn init_current_thread() { - init(current_thread()); - start_current_thread(); +/// Initialize the Tokio runtime with a custom build +pub fn init(builder: Builder) { + *TOKIO_BUILDER.lock().unwrap() = builder } /// Get a reference to the current tokio runtime pub fn get_runtime<'a>() -> &'a Runtime { - TOKIO_RUNTIME.get().expect(EXPECT_TOKIO_INIT) -} - -fn multi_thread() -> Runtime { - Builder::new_multi_thread() - .enable_all() - .build() - .expect("Couldn't build the multi-thread Tokio runtime") -} - -/// Initialize the Tokio Runtime with the multi-thread scheduler -/// -/// # Panics -/// This function will panic if called a second time. See [`init_multi_thread_once`] if you want to -/// avoid this panic. -pub fn init_multi_thread() { - init(multi_thread()); -} - -/// Ensure that the Tokio Runtime is initialized -/// -/// If the runtime has not been initialized already, the multi-thread scheduler -/// is used. Calling this function a second time is a no-op. -pub fn init_multi_thread_once() { - TOKIO_RUNTIME.get_or_init(|| multi_thread()); -} - -/// Ensure that the Tokio Runtime is initialized -/// -/// If the runtime has not been initialized already, the current-thread -/// scheduler is used. Calling this function a second time is a no-op. -pub fn init_current_thread_once() { - let mut initialized = false; TOKIO_RUNTIME.get_or_init(|| { - initialized = true; - current_thread() - }); + TOKIO_BUILDER + .lock() + .unwrap() + .build() + .expect("Unable to build Tokio runtime") + }) +} - if initialized { - start_current_thread(); - } +fn multi_thread() -> Builder { + let mut builder = Builder::new_multi_thread(); + builder.enable_all(); + builder } /// Run the event loop until the given Future completes @@ -157,16 +102,9 @@ pub fn init_current_thread_once() { /// # use std::time::Duration; /// # /// # use pyo3::prelude::*; -/// # use tokio::runtime::{Builder, Runtime}; -/// # -/// # let runtime = Builder::new_current_thread() -/// # .enable_all() -/// # .build() -/// # .expect("Couldn't build the runtime"); /// # /// # Python::with_gil(|py| { /// # pyo3_asyncio::with_runtime(py, || { -/// # pyo3_asyncio::tokio::init_current_thread(); /// pyo3_asyncio::tokio::run_until_complete(py, async move { /// tokio::time::sleep(Duration::from_secs(1)).await; /// Ok(()) From d16c75b0fde3da0e81110de1e7d1d91387c1fca3 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Wed, 7 Jul 2021 16:53:32 -0500 Subject: [PATCH 13/32] Added deprecation attributes to the 0.13 API calls, made event_loop parameter on test attributes optional to support previous usage --- pyo3-asyncio-macros/src/lib.rs | 72 ++++++++++++++++---- pytests/common/mod.rs | 1 + pytests/test_async_std_asyncio.rs | 51 ++++++++++++-- pytests/test_tokio_current_thread_asyncio.rs | 1 + pytests/test_tokio_multi_thread_asyncio.rs | 1 + pytests/tokio_asyncio/mod.rs | 53 +++++++++++--- src/async_std.rs | 5 ++ src/generic.rs | 18 +++-- src/lib.rs | 25 +++++-- src/testing.rs | 12 ++-- src/tokio.rs | 5 ++ 11 files changed, 199 insertions(+), 45 deletions(-) diff --git a/pyo3-asyncio-macros/src/lib.rs b/pyo3-asyncio-macros/src/lib.rs index 7a83eb8..15b0ce7 100644 --- a/pyo3-asyncio-macros/src/lib.rs +++ b/pyo3-asyncio-macros/src/lib.rs @@ -124,6 +124,13 @@ pub fn tokio_main(args: TokenStream, item: TokenStream) -> TokenStream { /// thread::sleep(Duration::from_secs(1)); /// Ok(()) /// } +/// +/// // blocking test functions can optionally accept an event_loop parameter +/// #[pyo3_asyncio::async_std::test] +/// fn test_blocking_sleep_with_event_loop(event_loop: PyObject) -> PyResult<()> { +/// thread::sleep(Duration::from_secs(1)); +/// Ok(()) +/// } /// ``` #[cfg(not(test))] // NOTE: exporting main breaks tests, we should file an issue. #[proc_macro_attribute] @@ -136,20 +143,32 @@ pub fn async_std_test(_attr: TokenStream, item: TokenStream) -> TokenStream { let vis = &input.vis; let fn_impl = if input.sig.asyncness.is_none() { - quote! { - #vis fn #name() -> std::pin::Pin> + Send>> { - #sig { - #body - } - + // Optionally pass an event_loop parameter to blocking tasks + let task = if sig.inputs.is_empty() { + quote! { + Box::pin(pyo3_asyncio::async_std::re_exports::spawn_blocking(move || { + #name() + })) + } + } else { + quote! { let event_loop = Python::with_gil(|py| { pyo3_asyncio::async_std::task_event_loop(py).unwrap().into() }); - Box::pin(pyo3_asyncio::async_std::re_exports::spawn_blocking(move || { #name(event_loop) })) } + }; + + quote! { + #vis fn #name() -> std::pin::Pin> + Send>> { + #sig { + #body + } + + #task + } } } else { quote! { @@ -204,6 +223,13 @@ pub fn async_std_test(_attr: TokenStream, item: TokenStream) -> TokenStream { /// thread::sleep(Duration::from_secs(1)); /// Ok(()) /// } +/// +/// // blocking test functions can optionally accept an event_loop parameter +/// #[pyo3_asyncio::tokio::test] +/// fn test_blocking_sleep_with_event_loop(event_loop: PyObject) -> PyResult<()> { +/// thread::sleep(Duration::from_secs(1)); +/// Ok(()) +/// } /// ``` #[cfg(not(test))] // NOTE: exporting main breaks tests, we should file an issue. #[proc_macro_attribute] @@ -216,16 +242,24 @@ pub fn tokio_test(_attr: TokenStream, item: TokenStream) -> TokenStream { let vis = &input.vis; let fn_impl = if input.sig.asyncness.is_none() { - quote! { - #vis fn #name() -> std::pin::Pin> + Send>> { - #sig { - #body - } - + // Optionally pass an event_loop parameter to blocking tasks + let task = if sig.inputs.is_empty() { + quote! { + Box::pin(async move { + match pyo3_asyncio::tokio::get_runtime().spawn_blocking(move || #name()).await { + Ok(result) => result, + Err(e) => { + assert!(e.is_panic()); + Err(pyo3::exceptions::PyException::new_err("rust future panicked")) + } + } + }) + } + } else { + quote! { let event_loop = Python::with_gil(|py| { pyo3_asyncio::tokio::task_event_loop(py).unwrap().into() }); - Box::pin(async move { match pyo3_asyncio::tokio::get_runtime().spawn_blocking(move || #name(event_loop)).await { Ok(result) => result, @@ -236,6 +270,16 @@ pub fn tokio_test(_attr: TokenStream, item: TokenStream) -> TokenStream { } }) } + }; + + quote! { + #vis fn #name() -> std::pin::Pin> + Send>> { + #sig { + #body + } + + #task + } } } else { quote! { diff --git a/pytests/common/mod.rs b/pytests/common/mod.rs index 667d33f..a6f4e16 100644 --- a/pytests/common/mod.rs +++ b/pytests/common/mod.rs @@ -28,6 +28,7 @@ pub(super) async fn test_into_future(event_loop: PyObject) -> PyResult<()> { Ok(()) } +#[allow(deprecated)] pub(super) async fn test_into_future_0_13() -> PyResult<()> { let fut = Python::with_gil(|py| { let test_mod = diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index fc3110c..9d7019b 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -6,7 +6,8 @@ use async_std::task; use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] -fn sleep_for(py: Python, secs: &PyAny) -> PyResult { +#[allow(deprecated)] +fn sleep_into_coroutine(py: Python, secs: &PyAny) -> PyResult { let secs = secs.extract()?; pyo3_asyncio::async_std::into_coroutine(py, async move { @@ -15,22 +16,57 @@ fn sleep_for(py: Python, secs: &PyAny) -> PyResult { }) } +#[pyfunction] +fn sleep<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { + let secs = secs.extract()?; + + pyo3_asyncio::async_std::future_into_py(py, async move { + task::sleep(Duration::from_secs(secs)).await; + Python::with_gil(|py| Ok(py.None())) + }) +} + #[pyo3_asyncio::async_std::test] async fn test_into_coroutine() -> PyResult<()> { let fut = Python::with_gil(|py| { let sleeper_mod = PyModule::new(py, "rust_sleeper")?; - sleeper_mod.add_wrapped(wrap_pyfunction!(sleep_for))?; + sleeper_mod.add_wrapped(wrap_pyfunction!(sleep_into_coroutine))?; + + let test_mod = PyModule::from_code( + py, + common::TEST_MOD, + "test_into_coroutine_mod.py", + "test_into_coroutine_mod", + )?; + + pyo3_asyncio::async_std::into_future(test_mod.call_method1( + "sleep_for_1s", + (sleeper_mod.getattr("sleep_into_coroutine")?,), + )?) + })?; + + fut.await?; + + Ok(()) +} + +#[pyo3_asyncio::async_std::test] +async fn test_future_into_py() -> PyResult<()> { + let fut = Python::with_gil(|py| { + let sleeper_mod = PyModule::new(py, "rust_sleeper")?; + + sleeper_mod.add_wrapped(wrap_pyfunction!(sleep))?; let test_mod = PyModule::from_code( py, common::TEST_MOD, - "test_rust_coroutine/test_mod.py", - "test_mod", + "test_future_into_py_mod.py", + "test_future_into_py_mod", )?; pyo3_asyncio::async_std::into_future( - test_mod.call_method1("sleep_for_1s", (sleeper_mod.getattr("sleep_for")?,))?, + test_mod.call_method1("sleep_for_1s", (sleeper_mod.getattr("sleep")?,))?, ) })?; @@ -55,7 +91,7 @@ async fn test_async_sleep() -> PyResult<()> { } #[pyo3_asyncio::async_std::test] -fn test_blocking_sleep(_event_loop: PyObject) -> PyResult<()> { +fn test_blocking_sleep() -> PyResult<()> { common::test_blocking_sleep() } @@ -100,6 +136,7 @@ async fn test_panic() -> PyResult<()> { } } +#[allow(deprecated)] fn main() -> pyo3::PyResult<()> { Python::with_gil(|py| { // into_coroutine requires the 0.13 API @@ -109,7 +146,7 @@ fn main() -> pyo3::PyResult<()> { } #[pyo3_asyncio::async_std::test] -async fn test_local_coroutine() -> PyResult<()> { +async fn test_local_future_into_py() -> PyResult<()> { Python::with_gil(|py| { let non_send_secs = Rc::new(1); diff --git a/pytests/test_tokio_current_thread_asyncio.rs b/pytests/test_tokio_current_thread_asyncio.rs index 40e8c39..866eab2 100644 --- a/pytests/test_tokio_current_thread_asyncio.rs +++ b/pytests/test_tokio_current_thread_asyncio.rs @@ -3,6 +3,7 @@ mod tokio_asyncio; use pyo3::prelude::*; +#[allow(deprecated)] fn main() -> pyo3::PyResult<()> { Python::with_gil(|py| { // into_coroutine requires the 0.13 API diff --git a/pytests/test_tokio_multi_thread_asyncio.rs b/pytests/test_tokio_multi_thread_asyncio.rs index f880628..ae68b0e 100644 --- a/pytests/test_tokio_multi_thread_asyncio.rs +++ b/pytests/test_tokio_multi_thread_asyncio.rs @@ -3,6 +3,7 @@ mod tokio_asyncio; use pyo3::prelude::*; +#[allow(deprecated)] fn main() -> pyo3::PyResult<()> { Python::with_gil(|py| { // into_coroutine requires the 0.13 API diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index a0b9d65..f12c3eb 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -5,7 +5,8 @@ use pyo3::{prelude::*, wrap_pyfunction}; use crate::common; #[pyfunction] -fn sleep_for(py: Python, secs: &PyAny) -> PyResult { +#[allow(deprecated)] +fn sleep_into_coroutine(py: Python, secs: &PyAny) -> PyResult { let secs = secs.extract()?; pyo3_asyncio::tokio::into_coroutine(py, async move { @@ -14,25 +15,57 @@ fn sleep_for(py: Python, secs: &PyAny) -> PyResult { }) } +#[pyfunction] +fn sleep<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { + let secs = secs.extract()?; + + pyo3_asyncio::tokio::future_into_py(py, async move { + tokio::time::sleep(Duration::from_secs(secs)).await; + Python::with_gil(|py| Ok(py.None())) + }) +} + #[pyo3_asyncio::tokio::test] async fn test_into_coroutine() -> PyResult<()> { let fut = Python::with_gil(|py| { - // into_coroutine requires the 0.13 API - pyo3_asyncio::try_init(py)?; + let sleeper_mod = PyModule::new(py, "rust_sleeper")?; + + sleeper_mod.add_wrapped(wrap_pyfunction!(sleep_into_coroutine))?; + + let test_mod = PyModule::from_code( + py, + common::TEST_MOD, + "test_into_coroutine_mod.py", + "test_into_coroutine_mod", + )?; + + pyo3_asyncio::tokio::into_future(test_mod.call_method1( + "sleep_for_1s", + (sleeper_mod.getattr("sleep_into_coroutine")?,), + )?) + })?; + + fut.await?; + + Ok(()) +} +#[pyo3_asyncio::tokio::test] +async fn test_future_into_py() -> PyResult<()> { + let fut = Python::with_gil(|py| { let sleeper_mod = PyModule::new(py, "rust_sleeper")?; - sleeper_mod.add_wrapped(wrap_pyfunction!(sleep_for))?; + sleeper_mod.add_wrapped(wrap_pyfunction!(sleep))?; let test_mod = PyModule::from_code( py, common::TEST_MOD, - "test_rust_coroutine/test_mod.py", - "test_mod", + "test_future_into_py_mod.py", + "test_future_into_py_mod", )?; pyo3_asyncio::tokio::into_future( - test_mod.call_method1("sleep_for_1s", (sleeper_mod.getattr("sleep_for")?,))?, + test_mod.call_method1("sleep_for_1s", (sleeper_mod.getattr("sleep")?,))?, ) })?; @@ -57,7 +90,7 @@ async fn test_async_sleep() -> PyResult<()> { } #[pyo3_asyncio::tokio::test] -fn test_blocking_sleep(_event_loop: PyObject) -> PyResult<()> { +fn test_blocking_sleep() -> PyResult<()> { common::test_blocking_sleep() } @@ -83,7 +116,7 @@ async fn test_other_awaitables() -> PyResult<()> { } #[pyo3_asyncio::tokio::test] -fn test_init_tokio_twice(_event_loop: PyObject) -> PyResult<()> { +fn test_init_tokio_twice() -> PyResult<()> { // tokio has already been initialized in test main. call these functions to // make sure they don't cause problems with the other tests. pyo3_asyncio::tokio::init_multi_thread_once(); @@ -93,7 +126,7 @@ fn test_init_tokio_twice(_event_loop: PyObject) -> PyResult<()> { } #[pyo3_asyncio::tokio::test] -fn test_local_set_coroutine(event_loop: PyObject) -> PyResult<()> { +fn test_local_future_into_py(event_loop: PyObject) -> PyResult<()> { tokio::task::LocalSet::new().block_on(pyo3_asyncio::tokio::get_runtime(), async { Python::with_gil(|py| { let non_send_secs = Rc::new(1); diff --git a/src/async_std.rs b/src/async_std.rs index 152c31e..4006e05 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -215,6 +215,11 @@ where /// }) /// } /// ``` +#[deprecated( + since = "0.14.0", + note = "Use the pyo3_asyncio::async_std::future_into_py instead" +)] +#[allow(deprecated)] pub fn into_coroutine(py: Python, fut: F) -> PyResult where F: Future> + Send + 'static, diff --git a/src/generic.rs b/src/generic.rs index c1b72dd..4eb484c 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -2,6 +2,7 @@ use std::{future::Future, pin::Pin}; use pyo3::prelude::*; +#[allow(deprecated)] use crate::{ call_soon_threadsafe, create_future, dump_err, err::RustPanic, get_cached_event_loop, get_event_loop, @@ -238,11 +239,15 @@ where "run_until_complete", (event_loop.call_method0("shutdown_asyncgens")?,), )?; + // how to do this prior to 3.9? - // event_loop.call_method1( - // "run_until_complete", - // (event_loop.call_method0("shutdown_default_executor")?,), - // )?; + if event_loop.hasattr("shutdown_default_executor")? { + event_loop.call_method1( + "run_until_complete", + (event_loop.call_method0("shutdown_default_executor")?,), + )?; + } + event_loop.call_method0("close")?; result?; @@ -550,6 +555,11 @@ where /// }) /// } /// ``` +#[deprecated( + since = "0.14.0", + note = "Use the pyo3_asyncio::generic::future_into_py instead" +)] +#[allow(deprecated)] pub fn into_coroutine(py: Python, fut: F) -> PyResult where R: Runtime, diff --git a/src/lib.rs b/src/lib.rs index e0a30ea..f7632a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -192,6 +192,11 @@ fn create_future(event_loop: &PyAny) -> PyResult<&PyAny> { /// }) /// } /// ``` +#[deprecated( + since = "0.14.0", + note = "Use the pyo3_asyncio::async_std::run or pyo3_asyncio::tokio::run instead" +)] +#[allow(deprecated)] pub fn with_runtime(py: Python, f: F) -> PyResult where F: FnOnce() -> PyResult, @@ -210,6 +215,7 @@ where /// - Must be called before any other pyo3-asyncio functions. /// - Calling `try_init` a second time returns `Ok(())` and does nothing. /// > In future versions this may return an `Err`. +#[deprecated(since = "0.14.0")] pub fn try_init(py: Python) -> PyResult<()> { CACHED_EVENT_LOOP.get_or_try_init(|| -> PyResult { let event_loop = get_event_loop(py)?; @@ -237,6 +243,7 @@ pub fn get_event_loop(py: Python) -> PyResult<&PyAny> { } /// Get a reference to the Python event loop cached by `try_init` (0.13 behaviour) +#[deprecated(since = "0.14.0")] pub fn get_cached_event_loop(py: Python) -> &PyAny { CACHED_EVENT_LOOP.get().expect(EXPECT_INIT).as_ref(py) } @@ -304,6 +311,7 @@ pub fn run_forever(py: Python) -> PyResult<()> { } /// Shutdown the event loops and perform any necessary cleanup +#[deprecated(since = "0.14.0")] pub fn try_close(py: Python) -> PyResult<()> { if let Some(exec) = EXECUTOR.get() { // Shutdown the executor and wait until all threads are cleaned up @@ -317,11 +325,15 @@ pub fn try_close(py: Python) -> PyResult<()> { "run_until_complete", (event_loop.call_method0("shutdown_asyncgens")?,), )?; + // how to do this prior to 3.9? - // event_loop.call_method1( - // "run_until_complete", - // (event_loop.call_method0("shutdown_default_executor")?,), - // )?; + if event_loop.hasattr("shutdown_default_executor")? { + event_loop.call_method1( + "run_until_complete", + (event_loop.call_method0("shutdown_default_executor")?,), + )?; + } + event_loop.call_method0("close")?; } @@ -508,6 +520,11 @@ pub fn into_future_with_loop( /// Ok(()) /// } /// ``` +#[deprecated( + since = "0.14.0", + note = "Use pyo3_asyncio::async_std::into_future or pyo3_asyncio::tokio::into_future" +)] +#[allow(deprecated)] pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { into_future_with_loop(get_cached_event_loop(awaitable.py()), awaitable) } diff --git a/src/testing.rs b/src/testing.rs index 07fbf8e..0b43ef3 100644 --- a/src/testing.rs +++ b/src/testing.rs @@ -95,7 +95,7 @@ //! } //! //! #[pyo3_asyncio::async_std::test] -//! fn test_blocking_sleep(_event_loop: PyObject) -> PyResult<()> { +//! fn test_blocking_sleep() -> PyResult<()> { //! thread::sleep(Duration::from_secs(1)); //! Ok(()) //! } @@ -125,7 +125,7 @@ //! } //! //! #[pyo3_asyncio::tokio::test] -//! fn test_blocking_sleep(_event_loop: PyObject) -> PyResult<()> { +//! fn test_blocking_sleep() -> PyResult<()> { //! thread::sleep(Duration::from_secs(1)); //! Ok(()) //! } @@ -163,7 +163,7 @@ //! } //! # #[cfg(feature = "async-std-runtime")] //! #[pyo3_asyncio::async_std::test] -//! fn test_async_std_sync_test_compiles(_event_loop: PyObject) -> PyResult<()> { +//! fn test_async_std_sync_test_compiles() -> PyResult<()> { //! Ok(()) //! } //! @@ -174,7 +174,7 @@ //! } //! # #[cfg(feature = "tokio-runtime")] //! #[pyo3_asyncio::tokio::test] -//! fn test_tokio_sync_test_compiles(_event_loop: PyObject) -> PyResult<()> { +//! fn test_tokio_sync_test_compiles() -> PyResult<()> { //! Ok(()) //! } //! } @@ -341,7 +341,7 @@ mod tests { } #[cfg(feature = "async-std-runtime")] #[pyo3_asyncio::async_std::test] - fn test_async_std_sync_test_compiles(_event_loop: PyObject) -> PyResult<()> { + fn test_async_std_sync_test_compiles() -> PyResult<()> { Ok(()) } @@ -352,7 +352,7 @@ mod tests { } #[cfg(feature = "tokio-runtime")] #[pyo3_asyncio::tokio::test] - fn test_tokio_sync_test_compiles(_event_loop: PyObject) -> PyResult<()> { + fn test_tokio_sync_test_compiles() -> PyResult<()> { Ok(()) } } diff --git a/src/tokio.rs b/src/tokio.rs index edb33db..33f12c0 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -302,6 +302,11 @@ where /// }) /// } /// ``` +#[deprecated( + since = "0.14.0", + note = "Use the pyo3_asyncio::tokio::future_into_py instead" +)] +#[allow(deprecated)] pub fn into_coroutine(py: Python, fut: F) -> PyResult where F: Future> + Send + 'static, From 9ab2dce48992d67f74e7a85bd3c17006902e3ef1 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Wed, 7 Jul 2021 17:16:32 -0500 Subject: [PATCH 14/32] Changed some names so that the 0.13 API is not broken and the naming is more consistent --- pytests/test_async_std_run_forever.rs | 2 +- pytests/tokio_run_forever/mod.rs | 2 +- src/async_std.rs | 8 +- src/generic.rs | 43 ++++------- src/lib.rs | 105 ++++++++++++++------------ src/tokio.rs | 8 +- 6 files changed, 80 insertions(+), 88 deletions(-) diff --git a/pytests/test_async_std_run_forever.rs b/pytests/test_async_std_run_forever.rs index 03d11ec..567814f 100644 --- a/pytests/test_async_std_run_forever.rs +++ b/pytests/test_async_std_run_forever.rs @@ -10,7 +10,7 @@ fn dump_err(py: Python, e: PyErr) { fn main() { Python::with_gil(|py| { - let event_loop = PyObject::from(pyo3_asyncio::get_event_loop(py).unwrap()); + let event_loop = PyObject::from(pyo3_asyncio::get_running_loop(py).unwrap()); async_std::task::spawn(async move { async_std::task::sleep(Duration::from_secs(1)).await; diff --git a/pytests/tokio_run_forever/mod.rs b/pytests/tokio_run_forever/mod.rs index b798f3b..ce3365b 100644 --- a/pytests/tokio_run_forever/mod.rs +++ b/pytests/tokio_run_forever/mod.rs @@ -10,7 +10,7 @@ fn dump_err(py: Python<'_>, e: PyErr) { pub(super) fn test_main() { Python::with_gil(|py| { - let event_loop = PyObject::from(pyo3_asyncio::get_event_loop(py).unwrap()); + let event_loop = PyObject::from(pyo3_asyncio::get_running_loop(py).unwrap()); pyo3_asyncio::tokio::get_runtime().spawn(async move { tokio::time::sleep(Duration::from_secs(1)).await; diff --git a/src/async_std.rs b/src/async_std.rs index 4006e05..c84ec4f 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -110,8 +110,8 @@ where } /// Get the current event loop from either Python or Rust async task local context -pub fn current_event_loop(py: Python) -> PyResult<&PyAny> { - generic::current_event_loop::(py) +pub fn get_current_loop(py: Python) -> PyResult<&PyAny> { + generic::get_current_loop::(py) } /// Get the task local event loop for the current async_std task @@ -276,7 +276,7 @@ where /// // Rc is non-send so it cannot be passed into pyo3_asyncio::async_std::into_coroutine /// let secs = Rc::new(secs); /// Ok(pyo3_asyncio::async_std::local_future_into_py_with_loop( -/// pyo3_asyncio::async_std::current_event_loop(py)?, +/// pyo3_asyncio::async_std::get_current_loop(py)?, /// async move { /// async_std::task::sleep(Duration::from_secs(*secs)).await; /// Python::with_gil(|py| Ok(py.None())) @@ -399,5 +399,5 @@ where /// } /// ``` pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { - into_future_with_loop(current_event_loop(awaitable.py())?, awaitable) + into_future_with_loop(get_current_loop(awaitable.py())?, awaitable) } diff --git a/src/generic.rs b/src/generic.rs index 4eb484c..456d60f 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -4,8 +4,8 @@ use pyo3::prelude::*; #[allow(deprecated)] use crate::{ - call_soon_threadsafe, create_future, dump_err, err::RustPanic, get_cached_event_loop, - get_event_loop, + call_soon_threadsafe, close, create_future, dump_err, err::RustPanic, get_event_loop, + get_running_loop, }; /// Generic utilities for a JoinError @@ -48,14 +48,14 @@ pub trait SpawnLocalExt: Runtime { } /// Get the current event loop from either Python or Rust async task local context -pub fn current_event_loop(py: Python) -> PyResult<&PyAny> +pub fn get_current_loop(py: Python) -> PyResult<&PyAny> where R: Runtime, { if let Some(event_loop) = R::get_task_event_loop(py) { Ok(event_loop) } else { - get_event_loop(py) + get_running_loop(py) } } @@ -141,7 +141,7 @@ where R: Runtime, F: Future> + Send + 'static, { - let event_loop = get_event_loop(py)?; + let event_loop = get_running_loop(py)?; let coro = future_into_py_with_loop::(event_loop, async move { fut.await?; @@ -231,28 +231,13 @@ where R: Runtime, F: Future> + Send + 'static, { - let event_loop = get_event_loop(py)?; + let event_loop = get_running_loop(py)?; let result = run_until_complete::(py, fut); - event_loop.call_method1( - "run_until_complete", - (event_loop.call_method0("shutdown_asyncgens")?,), - )?; - - // how to do this prior to 3.9? - if event_loop.hasattr("shutdown_default_executor")? { - event_loop.call_method1( - "run_until_complete", - (event_loop.call_method0("shutdown_default_executor")?,), - )?; - } - - event_loop.call_method0("close")?; + close(event_loop)?; - result?; - - Ok(()) + result } fn set_result(event_loop: &PyAny, future: &PyAny, result: PyResult) -> PyResult<()> { @@ -340,7 +325,7 @@ fn set_result(event_loop: &PyAny, future: &PyAny, result: PyResult) -> /// fn sleep_for<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { /// let secs = secs.extract()?; /// pyo3_asyncio::generic::future_into_py_with_loop::( -/// pyo3_asyncio::generic::current_event_loop::(py)?, +/// pyo3_asyncio::generic::get_current_loop::(py)?, /// async move { /// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; /// Python::with_gil(|py| Ok(py.None())) @@ -477,7 +462,7 @@ where R: Runtime, F: Future> + Send + 'static, { - future_into_py_with_loop::(current_event_loop::(py)?, fut) + future_into_py_with_loop::(get_current_loop::(py)?, fut) } /// Convert a Rust Future into a Python awaitable with a generic runtime @@ -565,7 +550,7 @@ where R: Runtime, F: Future> + Send + 'static, { - Ok(future_into_py_with_loop::(get_cached_event_loop(py), fut)?.into()) + Ok(future_into_py_with_loop::(get_event_loop(py), fut)?.into()) } /// Convert a `!Send` Rust Future into a Python awaitable with a generic runtime @@ -653,7 +638,7 @@ where /// #[pyfunction] /// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { /// pyo3_asyncio::generic::local_future_into_py_with_loop::( -/// pyo3_asyncio::get_event_loop(py)?, +/// pyo3_asyncio::get_running_loop(py)?, /// async move { /// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; /// Python::with_gil(|py| Ok(py.None())) @@ -794,7 +779,7 @@ where /// #[pyfunction] /// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { /// pyo3_asyncio::generic::local_future_into_py_with_loop::( -/// pyo3_asyncio::get_event_loop(py)?, +/// pyo3_asyncio::get_running_loop(py)?, /// async move { /// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; /// Python::with_gil(|py| Ok(py.None())) @@ -807,5 +792,5 @@ where R: SpawnLocalExt, F: Future> + 'static, { - local_future_into_py_with_loop::(current_event_loop::(py)?, fut) + local_future_into_py_with_loop::(get_current_loop::(py)?, fut) } diff --git a/src/lib.rs b/src/lib.rs index f7632a6..3b71ef4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -210,6 +210,25 @@ where Ok(result) } +fn close(event_loop: &PyAny) -> PyResult<()> { + event_loop.call_method1( + "run_until_complete", + (event_loop.call_method0("shutdown_asyncgens")?,), + )?; + + // how to do this prior to 3.9? + if event_loop.hasattr("shutdown_default_executor")? { + event_loop.call_method1( + "run_until_complete", + (event_loop.call_method0("shutdown_default_executor")?,), + )?; + } + + event_loop.call_method0("close")?; + + Ok(()) +} + /// Attempt to initialize the Python and Rust event loops /// /// - Must be called before any other pyo3-asyncio functions. @@ -218,7 +237,7 @@ where #[deprecated(since = "0.14.0")] pub fn try_init(py: Python) -> PyResult<()> { CACHED_EVENT_LOOP.get_or_try_init(|| -> PyResult { - let event_loop = get_event_loop(py)?; + let event_loop = get_running_loop(py)?; let executor = py .import("concurrent.futures.thread")? .call_method0("ThreadPoolExecutor")?; @@ -238,13 +257,15 @@ fn asyncio(py: Python) -> PyResult<&PyAny> { } /// Get a reference to the Python Event Loop from Rust -pub fn get_event_loop(py: Python) -> PyResult<&PyAny> { +pub fn get_running_loop(py: Python) -> PyResult<&PyAny> { + // Ideally should call get_running_loop, but calls get_event_loop for compatibility between + // versions. asyncio(py)?.call_method0("get_event_loop") } /// Get a reference to the Python event loop cached by `try_init` (0.13 behaviour) #[deprecated(since = "0.14.0")] -pub fn get_cached_event_loop(py: Python) -> &PyAny { +pub fn get_event_loop(py: Python) -> &PyAny { CACHED_EVENT_LOOP.get().expect(EXPECT_INIT).as_ref(py) } @@ -263,43 +284,40 @@ pub fn get_cached_event_loop(py: Python) -> &PyAny { /// /// ``` /// # #[cfg(feature = "async-std-runtime")] -/// fn main() { +/// fn main() -> pyo3::PyResult<()> { /// use std::time::Duration; /// use pyo3::prelude::*; /// /// Python::with_gil(|py| { -/// pyo3_asyncio::with_runtime(py, || -> PyResult<()> { -/// let event_loop = PyObject::from(pyo3_asyncio::get_event_loop(py)?); -/// // Wait 1 second, then stop the event loop -/// async_std::task::spawn(async move { -/// async_std::task::sleep(Duration::from_secs(1)).await; -/// Python::with_gil(|py| { -/// event_loop -/// .as_ref(py) -/// .call_method1( -/// "call_soon_threadsafe", -/// (event_loop -/// .as_ref(py) -/// .getattr("stop") -/// .map_err(|e| e.print_and_set_sys_last_vars(py)) -/// .unwrap(),), -/// ) +/// let event_loop = PyObject::from(pyo3_asyncio::get_running_loop(py)?); +/// // Wait 1 second, then stop the event loop +/// async_std::task::spawn(async move { +/// async_std::task::sleep(Duration::from_secs(1)).await; +/// Python::with_gil(|py| { +/// event_loop +/// .as_ref(py) +/// .call_method1( +/// "call_soon_threadsafe", +/// (event_loop +/// .as_ref(py) +/// .getattr("stop") /// .map_err(|e| e.print_and_set_sys_last_vars(py)) -/// .unwrap(); -/// }) -/// }); +/// .unwrap(),), +/// ) +/// .unwrap(); +/// }) +/// }); /// -/// pyo3_asyncio::run_forever(py)?; -/// Ok(()) -/// }) -/// .map_err(|e| e.print_and_set_sys_last_vars(py)) -/// .unwrap(); +/// pyo3_asyncio::run_forever(py)?; +/// pyo3_asyncio::get_running_loop(py)?.call_method0("close")?; +/// +/// Ok(()) /// }) /// } /// # #[cfg(not(feature = "async-std-runtime"))] -/// fn main() {} +/// # fn main() {} pub fn run_forever(py: Python) -> PyResult<()> { - if let Err(e) = get_event_loop(py)?.call_method0("run_forever") { + let result = if let Err(e) = get_running_loop(py)?.call_method0("run_forever") { if e.is_instance::(py) { Ok(()) } else { @@ -307,7 +325,11 @@ pub fn run_forever(py: Python) -> PyResult<()> { } } else { Ok(()) - } + }; + + close(get_running_loop(py)?)?; + + result } /// Shutdown the event loops and perform any necessary cleanup @@ -319,22 +341,7 @@ pub fn try_close(py: Python) -> PyResult<()> { } if let Some(event_loop) = CACHED_EVENT_LOOP.get() { - let event_loop = event_loop.as_ref(py); - - event_loop.call_method1( - "run_until_complete", - (event_loop.call_method0("shutdown_asyncgens")?,), - )?; - - // how to do this prior to 3.9? - if event_loop.hasattr("shutdown_default_executor")? { - event_loop.call_method1( - "run_until_complete", - (event_loop.call_method0("shutdown_default_executor")?,), - )?; - } - - event_loop.call_method0("close")?; + close(event_loop.as_ref(py))?; } Ok(()) @@ -435,7 +442,7 @@ fn call_soon_threadsafe(event_loop: &PyAny, args: impl IntoPy>) -> P /// /// Python::with_gil(|py| { /// pyo3_asyncio::into_future_with_loop( -/// pyo3_asyncio::get_event_loop(py)?, +/// pyo3_asyncio::get_running_loop(py)?, /// test_mod /// .call_method1(py, "py_sleep", (seconds.into_py(py),))? /// .as_ref(py), @@ -526,7 +533,7 @@ pub fn into_future_with_loop( )] #[allow(deprecated)] pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { - into_future_with_loop(get_cached_event_loop(awaitable.py()), awaitable) + into_future_with_loop(get_event_loop(awaitable.py()), awaitable) } fn dump_err(py: Python<'_>) -> impl FnOnce(PyErr) + '_ { diff --git a/src/tokio.rs b/src/tokio.rs index 33f12c0..103cb37 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -116,8 +116,8 @@ where } /// Get the current event loop from either Python or Rust async task local context -pub fn current_event_loop(py: Python) -> PyResult<&PyAny> { - generic::current_event_loop::(py) +pub fn get_current_loop(py: Python) -> PyResult<&PyAny> { + generic::get_current_loop::(py) } /// Get the task local event loop for the current tokio task @@ -364,7 +364,7 @@ where /// let secs = Rc::new(secs); /// /// pyo3_asyncio::tokio::local_future_into_py_with_loop( -/// pyo3_asyncio::tokio::current_event_loop(py)?, +/// pyo3_asyncio::tokio::get_current_loop(py)?, /// async move { /// tokio::time::sleep(Duration::from_secs(*secs)).await; /// Python::with_gil(|py| Ok(py.None())) @@ -517,5 +517,5 @@ where /// } /// ``` pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { - into_future_with_loop(current_event_loop(awaitable.py())?, awaitable) + into_future_with_loop(get_current_loop(awaitable.py())?, awaitable) } From f47b8f8bb8567c2397bd0bf9b2108f616089974b Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Wed, 7 Jul 2021 17:59:45 -0500 Subject: [PATCH 15/32] Deprecated run_forever, made run_forever tests work without deprecated calls --- pytests/test_async_std_asyncio.rs | 18 ++++---- pytests/test_async_std_run_forever.rs | 13 ++++-- pytests/tokio_run_forever/mod.rs | 13 ++++-- src/generic.rs | 8 ++-- src/lib.rs | 61 ++++++++++++++------------- 5 files changed, 63 insertions(+), 50 deletions(-) diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index 9d7019b..e6f2434 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -136,15 +136,6 @@ async fn test_panic() -> PyResult<()> { } } -#[allow(deprecated)] -fn main() -> pyo3::PyResult<()> { - Python::with_gil(|py| { - // into_coroutine requires the 0.13 API - pyo3_asyncio::try_init(py)?; - pyo3_asyncio::async_std::run(py, pyo3_asyncio::testing::main()) - }) -} - #[pyo3_asyncio::async_std::test] async fn test_local_future_into_py() -> PyResult<()> { Python::with_gil(|py| { @@ -161,3 +152,12 @@ async fn test_local_future_into_py() -> PyResult<()> { Ok(()) } + +#[allow(deprecated)] +fn main() -> pyo3::PyResult<()> { + Python::with_gil(|py| { + // into_coroutine requires the 0.13 API + pyo3_asyncio::try_init(py)?; + pyo3_asyncio::async_std::run(py, pyo3_asyncio::testing::main()) + }) +} diff --git a/pytests/test_async_std_run_forever.rs b/pytests/test_async_std_run_forever.rs index 567814f..335c182 100644 --- a/pytests/test_async_std_run_forever.rs +++ b/pytests/test_async_std_run_forever.rs @@ -10,17 +10,22 @@ fn dump_err(py: Python, e: PyErr) { fn main() { Python::with_gil(|py| { - let event_loop = PyObject::from(pyo3_asyncio::get_running_loop(py).unwrap()); + let asyncio = py.import("asyncio")?; + + let event_loop = asyncio.call_method0("new_event_loop")?; + asyncio.call_method1("set_event_loop", (event_loop,))?; + + let event_loop_hdl = PyObject::from(event_loop); async_std::task::spawn(async move { async_std::task::sleep(Duration::from_secs(1)).await; Python::with_gil(|py| { - event_loop + event_loop_hdl .as_ref(py) .call_method1( "call_soon_threadsafe", - (event_loop + (event_loop_hdl .as_ref(py) .getattr("stop") .map_err(|e| dump_err(py, e)) @@ -31,7 +36,7 @@ fn main() { }) }); - pyo3_asyncio::run_forever(py)?; + event_loop.call_method0("run_forever")?; println!("test test_run_forever ... ok"); Ok(()) diff --git a/pytests/tokio_run_forever/mod.rs b/pytests/tokio_run_forever/mod.rs index ce3365b..dab0a08 100644 --- a/pytests/tokio_run_forever/mod.rs +++ b/pytests/tokio_run_forever/mod.rs @@ -10,17 +10,22 @@ fn dump_err(py: Python<'_>, e: PyErr) { pub(super) fn test_main() { Python::with_gil(|py| { - let event_loop = PyObject::from(pyo3_asyncio::get_running_loop(py).unwrap()); + let asyncio = py.import("asyncio")?; + + let event_loop = asyncio.call_method0("new_event_loop")?; + asyncio.call_method1("set_event_loop", (event_loop,))?; + + let event_loop_hdl = PyObject::from(event_loop); pyo3_asyncio::tokio::get_runtime().spawn(async move { tokio::time::sleep(Duration::from_secs(1)).await; Python::with_gil(|py| { - event_loop + event_loop_hdl .as_ref(py) .call_method1( "call_soon_threadsafe", - (event_loop + (event_loop_hdl .as_ref(py) .getattr("stop") .map_err(|e| dump_err(py, e)) @@ -31,7 +36,7 @@ pub(super) fn test_main() { }) }); - pyo3_asyncio::run_forever(py)?; + event_loop.call_method0("run_forever")?; println!("test test_run_forever ... ok"); Ok(()) diff --git a/src/generic.rs b/src/generic.rs index 456d60f..3a7f4f8 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -4,8 +4,8 @@ use pyo3::prelude::*; #[allow(deprecated)] use crate::{ - call_soon_threadsafe, close, create_future, dump_err, err::RustPanic, get_event_loop, - get_running_loop, + asyncio_get_event_loop, call_soon_threadsafe, close, create_future, dump_err, err::RustPanic, + get_event_loop, get_running_loop, }; /// Generic utilities for a JoinError @@ -141,7 +141,7 @@ where R: Runtime, F: Future> + Send + 'static, { - let event_loop = get_running_loop(py)?; + let event_loop = asyncio_get_event_loop(py)?; let coro = future_into_py_with_loop::(event_loop, async move { fut.await?; @@ -231,7 +231,7 @@ where R: Runtime, F: Future> + Send + 'static, { - let event_loop = get_running_loop(py)?; + let event_loop = asyncio_get_event_loop(py)?; let result = run_until_complete::(py, fut); diff --git a/src/lib.rs b/src/lib.rs index 3b71ef4..8d39222 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -237,7 +237,7 @@ fn close(event_loop: &PyAny) -> PyResult<()> { #[deprecated(since = "0.14.0")] pub fn try_init(py: Python) -> PyResult<()> { CACHED_EVENT_LOOP.get_or_try_init(|| -> PyResult { - let event_loop = get_running_loop(py)?; + let event_loop = asyncio_get_event_loop(py)?; let executor = py .import("concurrent.futures.thread")? .call_method0("ThreadPoolExecutor")?; @@ -256,11 +256,15 @@ fn asyncio(py: Python) -> PyResult<&PyAny> { .map(|asyncio| asyncio.as_ref(py)) } +fn asyncio_get_event_loop(py: Python) -> PyResult<&PyAny> { + asyncio(py)?.call_method0("get_event_loop") +} + /// Get a reference to the Python Event Loop from Rust pub fn get_running_loop(py: Python) -> PyResult<&PyAny> { // Ideally should call get_running_loop, but calls get_event_loop for compatibility between // versions. - asyncio(py)?.call_method0("get_event_loop") + asyncio(py)?.call_method0("get_running_loop") } /// Get a reference to the Python event loop cached by `try_init` (0.13 behaviour) @@ -289,35 +293,38 @@ pub fn get_event_loop(py: Python) -> &PyAny { /// use pyo3::prelude::*; /// /// Python::with_gil(|py| { -/// let event_loop = PyObject::from(pyo3_asyncio::get_running_loop(py)?); -/// // Wait 1 second, then stop the event loop -/// async_std::task::spawn(async move { -/// async_std::task::sleep(Duration::from_secs(1)).await; -/// Python::with_gil(|py| { -/// event_loop -/// .as_ref(py) -/// .call_method1( -/// "call_soon_threadsafe", -/// (event_loop -/// .as_ref(py) -/// .getattr("stop") -/// .map_err(|e| e.print_and_set_sys_last_vars(py)) -/// .unwrap(),), -/// ) -/// .unwrap(); -/// }) -/// }); +/// pyo3_asyncio::with_runtime(py, || { +/// let event_loop_hdl = PyObject::from(pyo3_asyncio::get_event_loop(py)); +/// // Wait 1 second, then stop the event loop +/// async_std::task::spawn(async move { +/// async_std::task::sleep(Duration::from_secs(1)).await; +/// Python::with_gil(|py| { +/// event_loop_hdl +/// .as_ref(py) +/// .call_method1( +/// "call_soon_threadsafe", +/// (event_loop_hdl +/// .as_ref(py) +/// .getattr("stop") +/// .map_err(|e| e.print_and_set_sys_last_vars(py)) +/// .unwrap(),), +/// ) +/// .unwrap(); +/// }) +/// }); /// -/// pyo3_asyncio::run_forever(py)?; -/// pyo3_asyncio::get_running_loop(py)?.call_method0("close")?; +/// pyo3_asyncio::run_forever(py)?; /// -/// Ok(()) +/// Ok(()) +/// }) /// }) /// } /// # #[cfg(not(feature = "async-std-runtime"))] /// # fn main() {} +#[deprecated(since = "0.14.0")] +#[allow(deprecated)] pub fn run_forever(py: Python) -> PyResult<()> { - let result = if let Err(e) = get_running_loop(py)?.call_method0("run_forever") { + if let Err(e) = get_event_loop(py).call_method0("run_forever") { if e.is_instance::(py) { Ok(()) } else { @@ -325,11 +332,7 @@ pub fn run_forever(py: Python) -> PyResult<()> { } } else { Ok(()) - }; - - close(get_running_loop(py)?)?; - - result + } } /// Shutdown the event loops and perform any necessary cleanup From 8de1197626e089cd183c1fcf5c636869b216e4d3 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Wed, 7 Jul 2021 18:02:24 -0500 Subject: [PATCH 16/32] Got tests working with get_running_loop to ensure that the semantics are correct, changing back to get_event_loop for compatibility with Python <3.7 --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 8d39222..dd20e3d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -264,7 +264,7 @@ fn asyncio_get_event_loop(py: Python) -> PyResult<&PyAny> { pub fn get_running_loop(py: Python) -> PyResult<&PyAny> { // Ideally should call get_running_loop, but calls get_event_loop for compatibility between // versions. - asyncio(py)?.call_method0("get_running_loop") + asyncio(py)?.call_method0("get_event_loop") } /// Get a reference to the Python event loop cached by `try_init` (0.13 behaviour) From 4fffe010eef35d431301578483842d1a1024a0a4 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Wed, 14 Jul 2021 17:15:31 -0500 Subject: [PATCH 17/32] Minor naming change, fixed some docs / generic impls, made async-std scope remember old value (actually acts as a scope now) --- pyo3-asyncio-macros/src/lib.rs | 4 +- pytests/test_async_std_asyncio.rs | 8 +- pytests/tokio_asyncio/mod.rs | 4 +- src/async_std.rs | 86 +++++++++++++----- src/generic.rs | 143 ++++++++++++++++++++++++++---- src/lib.rs | 1 + src/tokio.rs | 65 ++++++++++---- 7 files changed, 249 insertions(+), 62 deletions(-) diff --git a/pyo3-asyncio-macros/src/lib.rs b/pyo3-asyncio-macros/src/lib.rs index 15b0ce7..7903734 100644 --- a/pyo3-asyncio-macros/src/lib.rs +++ b/pyo3-asyncio-macros/src/lib.rs @@ -153,7 +153,7 @@ pub fn async_std_test(_attr: TokenStream, item: TokenStream) -> TokenStream { } else { quote! { let event_loop = Python::with_gil(|py| { - pyo3_asyncio::async_std::task_event_loop(py).unwrap().into() + pyo3_asyncio::async_std::get_current_loop(py).unwrap().into() }); Box::pin(pyo3_asyncio::async_std::re_exports::spawn_blocking(move || { #name(event_loop) @@ -258,7 +258,7 @@ pub fn tokio_test(_attr: TokenStream, item: TokenStream) -> TokenStream { } else { quote! { let event_loop = Python::with_gil(|py| { - pyo3_asyncio::tokio::task_event_loop(py).unwrap().into() + pyo3_asyncio::tokio::get_current_loop(py).unwrap().into() }); Box::pin(async move { match pyo3_asyncio::tokio::get_runtime().spawn_blocking(move || #name(event_loop)).await { diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index e6f2434..0148533 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -98,7 +98,9 @@ fn test_blocking_sleep() -> PyResult<()> { #[pyo3_asyncio::async_std::test] async fn test_into_future() -> PyResult<()> { common::test_into_future(Python::with_gil(|py| { - pyo3_asyncio::async_std::task_event_loop(py).unwrap().into() + pyo3_asyncio::async_std::get_current_loop(py) + .unwrap() + .into() })) .await } @@ -111,7 +113,9 @@ async fn test_into_future_0_13() -> PyResult<()> { #[pyo3_asyncio::async_std::test] async fn test_other_awaitables() -> PyResult<()> { common::test_other_awaitables(Python::with_gil(|py| { - pyo3_asyncio::async_std::task_event_loop(py).unwrap().into() + pyo3_asyncio::async_std::get_current_loop(py) + .unwrap() + .into() })) .await } diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index f12c3eb..93d92eb 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -97,7 +97,7 @@ fn test_blocking_sleep() -> PyResult<()> { #[pyo3_asyncio::tokio::test] async fn test_into_future() -> PyResult<()> { common::test_into_future(Python::with_gil(|py| { - pyo3_asyncio::tokio::task_event_loop(py).unwrap().into() + pyo3_asyncio::tokio::get_current_loop(py).unwrap().into() })) .await } @@ -110,7 +110,7 @@ async fn test_into_future_0_13() -> PyResult<()> { #[pyo3_asyncio::tokio::test] async fn test_other_awaitables() -> PyResult<()> { common::test_other_awaitables(Python::with_gil(|py| { - pyo3_asyncio::tokio::task_event_loop(py).unwrap().into() + pyo3_asyncio::tokio::get_current_loop(py).unwrap().into() })) .await } diff --git a/src/async_std.rs b/src/async_std.rs index c84ec4f..3718204 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -1,14 +1,10 @@ -use std::{any::Any, future::Future, panic::AssertUnwindSafe, pin::Pin}; +use std::{any::Any, cell::RefCell, future::Future, panic::AssertUnwindSafe, pin::Pin}; use async_std::task; use futures::prelude::*; -use once_cell::unsync::OnceCell; -use pyo3::{prelude::*, PyNativeType}; +use pyo3::prelude::*; -use crate::{ - generic::{self, JoinError, Runtime, SpawnLocalExt}, - into_future_with_loop, -}; +use crate::generic::{self, JoinError, Runtime, SpawnLocalExt}; /// attributes /// re-exports for macros @@ -37,7 +33,7 @@ impl JoinError for AsyncStdJoinErr { } async_std::task_local! { - static EVENT_LOOP: OnceCell = OnceCell::new() + static EVENT_LOOP: RefCell> = RefCell::new(None); } struct AsyncStdRuntime; @@ -50,11 +46,19 @@ impl Runtime for AsyncStdRuntime { where F: Future + Send + 'static, { - EVENT_LOOP.with(|c| c.set(event_loop).unwrap()); - Box::pin(fut) + let old = EVENT_LOOP.with(|c| c.replace(Some(event_loop))); + Box::pin(async move { + let result = fut.await; + EVENT_LOOP.with(|c| c.replace(old)); + result + }) } fn get_task_event_loop(py: Python) -> Option<&PyAny> { - match EVENT_LOOP.try_with(|c| c.get().map(|event_loop| event_loop.clone().into_ref(py))) { + match EVENT_LOOP.try_with(|c| { + c.borrow() + .as_ref() + .map(|event_loop| event_loop.clone().into_ref(py)) + }) { Ok(event_loop) => event_loop, Err(_) => None, } @@ -78,8 +82,12 @@ impl SpawnLocalExt for AsyncStdRuntime { where F: Future + 'static, { - EVENT_LOOP.with(|c| c.set(event_loop).unwrap()); - Box::pin(fut) + let old = EVENT_LOOP.with(|c| c.replace(Some(event_loop))); + Box::pin(async move { + let result = fut.await; + EVENT_LOOP.with(|c| c.replace(old)); + result + }) } fn spawn_local(fut: F) -> Self::JoinHandle @@ -110,21 +118,20 @@ where } /// Get the current event loop from either Python or Rust async task local context +/// +/// This function first checks if the runtime has a task-local reference to the Python event loop. +/// If not, it calls [`get_running_loop`](`crate::get_running_loop`) to get the event loop +/// associated with the current OS thread. pub fn get_current_loop(py: Python) -> PyResult<&PyAny> { generic::get_current_loop::(py) } -/// Get the task local event loop for the current async_std task -pub fn task_event_loop(py: Python) -> Option<&PyAny> { - AsyncStdRuntime::get_task_event_loop(py) -} - /// Run the event loop until the given Future completes /// /// The event loop runs until the given future is complete. /// /// After this function returns, the event loop can be resumed with either [`run_until_complete`] or -/// [`crate::run_forever`] +/// [`run_forever`](`crate::run_forever`) /// /// # Arguments /// * `py` - The current PyO3 GIL guard @@ -227,6 +234,39 @@ where generic::into_coroutine::(py, fut) } +/// Convert a Rust Future into a Python awaitable +/// +/// # Arguments +/// * `event_loop` - The Python event loop that the awaitable should be attached to +/// * `fut` - The Rust future to be converted +/// +/// # Examples +/// +/// ``` +/// use std::time::Duration; +/// +/// use pyo3::prelude::*; +/// +/// /// Awaitable sleep function +/// #[pyfunction] +/// fn sleep_for<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { +/// let secs = secs.extract()?; +/// pyo3_asyncio::async_std::future_into_py_with_loop( +/// pyo3_asyncio::async_std::get_current_loop(py)?, +/// async move { +/// async_std::task::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// } +/// ) +/// } +/// ``` +pub fn future_into_py_with_loop(event_loop: &PyAny, fut: F) -> PyResult<&PyAny> +where + F: Future> + Send + 'static, +{ + generic::future_into_py_with_loop::(event_loop, fut) +} + /// Convert a Rust Future into a Python awaitable /// /// # Arguments @@ -260,7 +300,7 @@ where /// Convert a `!Send` Rust Future into a Python awaitable /// /// # Arguments -/// * `py` - The current PyO3 GIL guard +/// * `event_loop` - The Python event loop that the awaitable should be attached to /// * `fut` - The Rust future to be converted /// /// # Examples @@ -273,7 +313,7 @@ where /// /// Awaitable non-send sleep function /// #[pyfunction] /// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { -/// // Rc is non-send so it cannot be passed into pyo3_asyncio::async_std::into_coroutine +/// // Rc is non-send so it cannot be passed into pyo3_asyncio::async_std::future_into_py /// let secs = Rc::new(secs); /// Ok(pyo3_asyncio::async_std::local_future_into_py_with_loop( /// pyo3_asyncio::async_std::get_current_loop(py)?, @@ -321,7 +361,7 @@ where /// /// Awaitable non-send sleep function /// #[pyfunction] /// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { -/// // Rc is non-send so it cannot be passed into pyo3_asyncio::async_std::into_coroutine +/// // Rc is non-send so it cannot be passed into pyo3_asyncio::async_std::future_into_py /// let secs = Rc::new(secs); /// pyo3_asyncio::async_std::local_future_into_py(py, async move { /// async_std::task::sleep(Duration::from_secs(*secs)).await; @@ -399,5 +439,5 @@ where /// } /// ``` pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { - into_future_with_loop(get_current_loop(awaitable.py())?, awaitable) + generic::into_future::(awaitable) } diff --git a/src/generic.rs b/src/generic.rs index 3a7f4f8..77d01cc 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -1,11 +1,11 @@ use std::{future::Future, pin::Pin}; -use pyo3::prelude::*; +use pyo3::{prelude::*, PyNativeType}; #[allow(deprecated)] use crate::{ asyncio_get_event_loop, call_soon_threadsafe, close, create_future, dump_err, err::RustPanic, - get_event_loop, get_running_loop, + get_event_loop, get_running_loop, into_future_with_loop, }; /// Generic utilities for a JoinError @@ -48,6 +48,10 @@ pub trait SpawnLocalExt: Runtime { } /// Get the current event loop from either Python or Rust async task local context +/// +/// This function first checks if the runtime has a task-local reference to the Python event loop. +/// If not, it calls [`get_running_loop`](crate::get_running_loop`) to get the event loop associated +/// with the current OS thread. pub fn get_current_loop(py: Python) -> PyResult<&PyAny> where R: Runtime, @@ -62,7 +66,7 @@ where /// Run the event loop until the given Future completes /// /// After this function returns, the event loop can be resumed with either [`run_until_complete`] or -/// [`crate::run_forever`] +/// [`run_forever`](`crate::run_forever`) /// /// # Arguments /// * `py` - The current PyO3 GIL guard @@ -255,10 +259,116 @@ fn set_result(event_loop: &PyAny, future: &PyAny, result: PyResult) -> Ok(()) } +/// Convert a Python `awaitable` into a Rust Future +/// +/// This function simply forwards the future and the `event_loop` returned by [`get_current_loop`] +/// to [`into_future_with_loop`](`crate::into_future_with_loop`). See +/// [`into_future_with_loop`](`crate::into_future_with_loop`) for more details. +/// +/// # Arguments +/// * `awaitable` - The Python `awaitable` to be converted +/// +/// # Examples +/// +/// ```no_run +/// # use std::{pin::Pin, future::Future, task::{Context, Poll}, time::Duration}; +/// # +/// # use pyo3::prelude::*; +/// # +/// # use pyo3_asyncio::generic::{JoinError, Runtime}; +/// # +/// # struct MyCustomJoinError; +/// # +/// # impl JoinError for MyCustomJoinError { +/// # fn is_panic(&self) -> bool { +/// # unreachable!() +/// # } +/// # } +/// # +/// # struct MyCustomJoinHandle; +/// # +/// # impl Future for MyCustomJoinHandle { +/// # type Output = Result<(), MyCustomJoinError>; +/// # +/// # fn poll(self: Pin<&mut Self>, _cx: &mut Context) -> Poll { +/// # unreachable!() +/// # } +/// # } +/// # +/// # struct MyCustomRuntime; +/// # +/// # impl MyCustomRuntime { +/// # async fn sleep(_: Duration) { +/// # unreachable!() +/// # } +/// # } +/// # +/// # impl Runtime for MyCustomRuntime { +/// # type JoinError = MyCustomJoinError; +/// # type JoinHandle = MyCustomJoinHandle; +/// # +/// # fn scope(_event_loop: PyObject, fut: F) -> Pin + Send>> +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # fn get_task_event_loop(py: Python) -> Option<&PyAny> { +/// # unreachable!() +/// # } +/// # +/// # fn spawn(fut: F) -> Self::JoinHandle +/// # where +/// # F: Future + Send + 'static +/// # { +/// # unreachable!() +/// # } +/// # } +/// # +/// const PYTHON_CODE: &'static str = r#" +/// import asyncio +/// +/// async def py_sleep(duration): +/// await asyncio.sleep(duration) +/// "#; +/// +/// async fn py_sleep(seconds: f32) -> PyResult<()> { +/// let test_mod = Python::with_gil(|py| -> PyResult { +/// Ok( +/// PyModule::from_code( +/// py, +/// PYTHON_CODE, +/// "test_into_future/test_mod.py", +/// "test_mod" +/// )? +/// .into() +/// ) +/// })?; +/// +/// Python::with_gil(|py| { +/// pyo3_asyncio::generic::into_future::( +/// test_mod +/// .call_method1(py, "py_sleep", (seconds.into_py(py),))? +/// .as_ref(py), +/// ) +/// })? +/// .await?; +/// Ok(()) +/// } +/// ``` +pub fn into_future( + awaitable: &PyAny, +) -> PyResult> + Send> +where + R: Runtime, +{ + into_future_with_loop(get_current_loop::(awaitable.py())?, awaitable) +} + /// Convert a Rust Future into a Python awaitable with a generic runtime /// /// # Arguments -/// * `py` - The current PyO3 GIL guard +/// * `event_loop` - The Python event loop that the awaitable should be attached to /// * `fut` - The Rust future to be converted /// /// # Examples @@ -556,7 +666,7 @@ where /// Convert a `!Send` Rust Future into a Python awaitable with a generic runtime /// /// # Arguments -/// * `py` - The current PyO3 GIL guard +/// * `event_loop` - The Python event loop that the awaitable should be attached to /// * `fut` - The Rust future to be converted /// /// # Examples @@ -630,17 +740,20 @@ where /// # } /// # } /// # -/// use std::time::Duration; +/// use std::{rc::Rc, time::Duration}; /// /// use pyo3::prelude::*; /// /// /// Awaitable sleep function /// #[pyfunction] /// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { +/// // Rc is !Send so it cannot be passed into pyo3_asyncio::generic::future_into_py +/// let secs = Rc::new(secs); +/// /// pyo3_asyncio::generic::local_future_into_py_with_loop::( /// pyo3_asyncio::get_running_loop(py)?, /// async move { -/// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; +/// MyCustomRuntime::sleep(Duration::from_secs(*secs)).await; /// Python::with_gil(|py| Ok(py.None())) /// } /// ) @@ -771,20 +884,20 @@ where /// # } /// # } /// # -/// use std::time::Duration; +/// use std::{rc::Rc, time::Duration}; /// /// use pyo3::prelude::*; /// /// /// Awaitable sleep function /// #[pyfunction] /// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { -/// pyo3_asyncio::generic::local_future_into_py_with_loop::( -/// pyo3_asyncio::get_running_loop(py)?, -/// async move { -/// MyCustomRuntime::sleep(Duration::from_secs(secs)).await; -/// Python::with_gil(|py| Ok(py.None())) -/// } -/// ) +/// // Rc is !Send so it cannot be passed into pyo3_asyncio::generic::future_into_py +/// let secs = Rc::new(secs); +/// +/// pyo3_asyncio::generic::local_future_into_py::(py, async move { +/// MyCustomRuntime::sleep(Duration::from_secs(*secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// }) /// } /// ``` pub fn local_future_into_py(py: Python, fut: F) -> PyResult<&PyAny> diff --git a/src/lib.rs b/src/lib.rs index dd20e3d..9492f1a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -414,6 +414,7 @@ fn call_soon_threadsafe(event_loop: &PyAny, args: impl IntoPy>) -> P /// simply awaits the result through the `futures::channel::oneshot::Receiver>`. /// /// # Arguments +/// * `event_loop` - The Python event loop that the awaitable should be attached to /// * `awaitable` - The Python `awaitable` to be converted /// /// # Examples diff --git a/src/tokio.rs b/src/tokio.rs index 103cb37..8540ed8 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -6,12 +6,9 @@ use ::tokio::{ }; use futures::future::pending; use once_cell::{sync::OnceCell, unsync::OnceCell as UnsyncOnceCell}; -use pyo3::{prelude::*, PyNativeType}; +use pyo3::prelude::*; -use crate::{ - generic::{self, Runtime as GenericRuntime, SpawnLocalExt}, - into_future_with_loop, -}; +use crate::generic::{self, Runtime as GenericRuntime, SpawnLocalExt}; /// attributes /// re-exports for macros @@ -116,15 +113,14 @@ where } /// Get the current event loop from either Python or Rust async task local context +/// +/// This function first checks if the runtime has a task-local reference to the Python event loop. +/// If not, it calls [`get_running_loop`](`crate::get_running_loop`) to get the event loop +/// associated with the current OS thread. pub fn get_current_loop(py: Python) -> PyResult<&PyAny> { generic::get_current_loop::(py) } -/// Get the task local event loop for the current tokio task -pub fn task_event_loop(py: Python) -> Option<&PyAny> { - TokioRuntime::get_task_event_loop(py) -} - /// Initialize the Tokio Runtime with a custom build pub fn init(runtime: Runtime) { TOKIO_RUNTIME @@ -314,6 +310,39 @@ where generic::into_coroutine::(py, fut) } +/// Convert a Rust Future into a Python awaitable +/// +/// # Arguments +/// * `event_loop` - The Python event loop that the awaitable should be attached to +/// * `fut` - The Rust future to be converted +/// +/// # Examples +/// +/// ``` +/// use std::time::Duration; +/// +/// use pyo3::prelude::*; +/// +/// /// Awaitable sleep function +/// #[pyfunction] +/// fn sleep_for<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { +/// let secs = secs.extract()?; +/// pyo3_asyncio::tokio::future_into_py_with_loop( +/// pyo3_asyncio::tokio::get_current_loop(py)?, +/// async move { +/// tokio::time::sleep(Duration::from_secs(secs)).await; +/// Python::with_gil(|py| Ok(py.None())) +/// } +/// ) +/// } +/// ``` +pub fn future_into_py_with_loop(event_loop: &PyAny, fut: F) -> PyResult<&PyAny> +where + F: Future> + Send + 'static, +{ + generic::future_into_py_with_loop::(event_loop, fut) +} + /// Convert a Rust Future into a Python awaitable /// /// # Arguments @@ -347,7 +376,7 @@ where /// Convert a `!Send` Rust Future into a Python awaitable /// /// # Arguments -/// * `py` - The current PyO3 GIL guard +/// * `event_loop` - The Python event loop that the awaitable should be attached to /// * `fut` - The Rust future to be converted /// /// # Examples @@ -360,7 +389,7 @@ where /// /// Awaitable non-send sleep function /// #[pyfunction] /// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { -/// // Rc is non-send so it cannot be passed into pyo3_asyncio::tokio::into_coroutine +/// // Rc is non-send so it cannot be passed into pyo3_asyncio::tokio::future_into_py /// let secs = Rc::new(secs); /// /// pyo3_asyncio::tokio::local_future_into_py_with_loop( @@ -375,9 +404,9 @@ where /// # #[cfg(all(feature = "tokio-runtime", feature = "attributes"))] /// #[pyo3_asyncio::tokio::main] /// async fn main() -> PyResult<()> { -/// let event_loop = Python::with_gil(|py| { -/// PyObject::from(pyo3_asyncio::tokio::task_event_loop(py).unwrap()) -/// }); +/// let event_loop = Python::with_gil(|py| -> PyResult { +/// Ok(pyo3_asyncio::tokio::get_current_loop(py)?.into()) +/// })?; /// /// // the main coroutine is running in a Send context, so we cannot use LocalSet here. Instead /// // we use spawn_blocking in order to use LocalSet::block_on @@ -424,7 +453,7 @@ where /// /// Awaitable non-send sleep function /// #[pyfunction] /// fn sleep_for(py: Python, secs: u64) -> PyResult<&PyAny> { -/// // Rc is non-send so it cannot be passed into pyo3_asyncio::tokio::into_coroutine +/// // Rc is non-send so it cannot be passed into pyo3_asyncio::tokio::future_into_py /// let secs = Rc::new(secs); /// pyo3_asyncio::tokio::local_future_into_py(py, async move { /// tokio::time::sleep(Duration::from_secs(*secs)).await; @@ -436,7 +465,7 @@ where /// #[pyo3_asyncio::tokio::main] /// async fn main() -> PyResult<()> { /// let event_loop = Python::with_gil(|py| { -/// PyObject::from(pyo3_asyncio::tokio::task_event_loop(py).unwrap()) +/// PyObject::from(pyo3_asyncio::tokio::get_current_loop(py).unwrap()) /// }); /// /// // the main coroutine is running in a Send context, so we cannot use LocalSet here. Instead @@ -517,5 +546,5 @@ where /// } /// ``` pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { - into_future_with_loop(get_current_loop(awaitable.py())?, awaitable) + generic::into_future::(awaitable) } From 8b1b493f08b6335674f1bb2c09fd8775977781bb Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Wed, 14 Jul 2021 17:30:07 -0500 Subject: [PATCH 18/32] Changed get_running_loop to use asyncio.get_running_loop when available, falling back to asyncio.get_event_loop when necessary --- src/lib.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 9492f1a..6f38a20 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,6 +147,7 @@ pub mod doc_test { static ASYNCIO: OnceCell = OnceCell::new(); static ENSURE_FUTURE: OnceCell = OnceCell::new(); +static GET_RUNNING_LOOP: OnceCell = OnceCell::new(); const EXPECT_INIT: &str = "PyO3 Asyncio has not been initialized"; static CACHED_EVENT_LOOP: OnceCell = OnceCell::new(); @@ -261,10 +262,28 @@ fn asyncio_get_event_loop(py: Python) -> PyResult<&PyAny> { } /// Get a reference to the Python Event Loop from Rust +/// +/// Equivalent to `asyncio.get_running_loop()` in Python 3.7+ +/// > For Python 3.6, this function falls back to `asyncio.get_event_loop()` which has slightly +/// different behaviour. See the [`asyncio.get_event_loop`](https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_event_loop) +/// docs to better understand the differences. pub fn get_running_loop(py: Python) -> PyResult<&PyAny> { // Ideally should call get_running_loop, but calls get_event_loop for compatibility between // versions. - asyncio(py)?.call_method0("get_event_loop") + GET_RUNNING_LOOP + .get_or_try_init(|| -> PyResult { + let asyncio = asyncio(py)?; + + if asyncio.hasattr("get_running_loop")? { + // correct behaviour with Python 3.7+ + Ok(asyncio.getattr("get_running_loop")?.into()) + } else { + // Python 3.6 compatibility mode + Ok(asyncio.getattr("get_event_loop")?.into()) + } + })? + .as_ref(py) + .call0() } /// Get a reference to the Python event loop cached by `try_init` (0.13 behaviour) From c61ef224c2ed2ed392dfbd46876b40f19de277bc Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Thu, 15 Jul 2021 16:14:24 -0500 Subject: [PATCH 19/32] Fixed crates.io badge --- README.md | 2 +- src/lib.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4606e82..63b0703 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Actions Status](https://github.com/awestlake87/pyo3-asyncio/workflows/CI/badge.svg)](https://github.com/awestlake87/pyo3-asyncio/actions) [![codecov](https://codecov.io/gh/awestlake87/pyo3-asyncio/branch/master/graph/badge.svg)](https://codecov.io/gh/awestlake87/pyo3-asyncio) -[![crates.io](http://meritbadge.herokuapp.com/pyo3-asyncio)](https://crates.io/crates/pyo3-asyncio) +[![crates.io](https://img.shields.io/crates/v/pyo3-asyncio)](https://crates.io/crates/pyo3-asyncio) [![minimum rustc 1.46](https://img.shields.io/badge/rustc-1.46+-blue.svg)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) [Rust](http://www.rust-lang.org/) bindings for [Python](https://www.python.org/)'s [Asyncio Library](https://docs.python.org/3/library/asyncio.html). This crate facilitates interactions between Rust Futures and Python Coroutines and manages the lifecycle of their corresponding event loops. diff --git a/src/lib.rs b/src/lib.rs index 6f38a20..4548d51 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -263,13 +263,13 @@ fn asyncio_get_event_loop(py: Python) -> PyResult<&PyAny> { /// Get a reference to the Python Event Loop from Rust /// -/// Equivalent to `asyncio.get_running_loop()` in Python 3.7+ +/// Equivalent to `asyncio.get_running_loop()` in Python 3.7+. /// > For Python 3.6, this function falls back to `asyncio.get_event_loop()` which has slightly /// different behaviour. See the [`asyncio.get_event_loop`](https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.get_event_loop) /// docs to better understand the differences. pub fn get_running_loop(py: Python) -> PyResult<&PyAny> { - // Ideally should call get_running_loop, but calls get_event_loop for compatibility between - // versions. + // Ideally should call get_running_loop, but calls get_event_loop for compatibility when + // get_running_loop is not available. GET_RUNNING_LOOP .get_or_try_init(|| -> PyResult { let asyncio = asyncio(py)?; From d800b9088d4d129f04cce54d731b7ba16e9187f7 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Tue, 3 Aug 2021 10:02:29 -0500 Subject: [PATCH 20/32] Bumped pyo3 version to 0.14, fixed misc problems due to the auto-initialize feature being disabled in 0.14 --- Cargo.toml | 6 +++--- pyo3-asyncio-macros/Cargo.toml | 2 +- pyo3-asyncio-macros/src/lib.rs | 2 ++ pyo3-asyncio-macros/src/tokio.rs | 2 ++ pytests/test_async_std_asyncio.rs | 2 ++ pytests/test_async_std_run_forever.rs | 2 ++ pytests/test_tokio_current_thread_asyncio.rs | 2 ++ pytests/test_tokio_current_thread_run_forever.rs | 1 + pytests/test_tokio_multi_thread_asyncio.rs | 2 ++ pytests/test_tokio_multi_thread_run_forever.rs | 2 ++ src/async_std.rs | 5 +++++ src/lib.rs | 6 ++++++ src/testing.rs | 2 +- src/tokio.rs | 1 + 14 files changed, 32 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 58a0226..c517d38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pyo3-asyncio" description = "PyO3 utilities for Python's Asyncio library" -version = "0.13.4" +version = "0.14.0" authors = ["Andrew J Westlake "] readme = "README.md" keywords = ["pyo3", "python", "ffi", "async", "asyncio"] @@ -89,8 +89,8 @@ futures = "0.3" inventory = "0.1" lazy_static = "1.4" once_cell = "1.5" -pyo3 = "0.13" -pyo3-asyncio-macros = { path = "pyo3-asyncio-macros", version = "=0.13.4", optional = true } +pyo3 = "0.14" +pyo3-asyncio-macros = { path = "pyo3-asyncio-macros", version = "=0.14.0", optional = true } [dependencies.async-std] version = "1.9" diff --git a/pyo3-asyncio-macros/Cargo.toml b/pyo3-asyncio-macros/Cargo.toml index b7765c1..08377c6 100644 --- a/pyo3-asyncio-macros/Cargo.toml +++ b/pyo3-asyncio-macros/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pyo3-asyncio-macros" description = "Proc Macro Attributes for PyO3 Asyncio" -version = "0.13.4" +version = "0.14.0" authors = ["Andrew J Westlake "] readme = "../README.md" keywords = ["pyo3", "python", "ffi", "async", "asyncio"] diff --git a/pyo3-asyncio-macros/src/lib.rs b/pyo3-asyncio-macros/src/lib.rs index 7903734..92da632 100644 --- a/pyo3-asyncio-macros/src/lib.rs +++ b/pyo3-asyncio-macros/src/lib.rs @@ -49,6 +49,8 @@ pub fn async_std_main(_attr: TokenStream, item: TokenStream) -> TokenStream { #body } + pyo3::prepare_freethreaded_python(); + pyo3::Python::with_gil(|py| { pyo3_asyncio::async_std::run(py, main()) .map_err(|e| { diff --git a/pyo3-asyncio-macros/src/tokio.rs b/pyo3-asyncio-macros/src/tokio.rs index 5eaf1f0..d58a033 100644 --- a/pyo3-asyncio-macros/src/tokio.rs +++ b/pyo3-asyncio-macros/src/tokio.rs @@ -247,6 +247,8 @@ fn parse_knobs( #body } + pyo3::prepare_freethreaded_python(); + pyo3_asyncio::tokio::init( #rt .enable_all() diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index 0148533..f748155 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -159,6 +159,8 @@ async fn test_local_future_into_py() -> PyResult<()> { #[allow(deprecated)] fn main() -> pyo3::PyResult<()> { + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { // into_coroutine requires the 0.13 API pyo3_asyncio::try_init(py)?; diff --git a/pytests/test_async_std_run_forever.rs b/pytests/test_async_std_run_forever.rs index 335c182..2346466 100644 --- a/pytests/test_async_std_run_forever.rs +++ b/pytests/test_async_std_run_forever.rs @@ -9,6 +9,8 @@ fn dump_err(py: Python, e: PyErr) { } fn main() { + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { let asyncio = py.import("asyncio")?; diff --git a/pytests/test_tokio_current_thread_asyncio.rs b/pytests/test_tokio_current_thread_asyncio.rs index 866eab2..ce756ee 100644 --- a/pytests/test_tokio_current_thread_asyncio.rs +++ b/pytests/test_tokio_current_thread_asyncio.rs @@ -5,6 +5,8 @@ use pyo3::prelude::*; #[allow(deprecated)] fn main() -> pyo3::PyResult<()> { + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { // into_coroutine requires the 0.13 API pyo3_asyncio::try_init(py)?; diff --git a/pytests/test_tokio_current_thread_run_forever.rs b/pytests/test_tokio_current_thread_run_forever.rs index f2a72ad..641d018 100644 --- a/pytests/test_tokio_current_thread_run_forever.rs +++ b/pytests/test_tokio_current_thread_run_forever.rs @@ -1,6 +1,7 @@ mod tokio_run_forever; fn main() { + pyo3::prepare_freethreaded_python(); pyo3_asyncio::tokio::init_current_thread(); tokio_run_forever::test_main(); diff --git a/pytests/test_tokio_multi_thread_asyncio.rs b/pytests/test_tokio_multi_thread_asyncio.rs index ae68b0e..32adae7 100644 --- a/pytests/test_tokio_multi_thread_asyncio.rs +++ b/pytests/test_tokio_multi_thread_asyncio.rs @@ -5,6 +5,8 @@ use pyo3::prelude::*; #[allow(deprecated)] fn main() -> pyo3::PyResult<()> { + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { // into_coroutine requires the 0.13 API pyo3_asyncio::try_init(py)?; diff --git a/pytests/test_tokio_multi_thread_run_forever.rs b/pytests/test_tokio_multi_thread_run_forever.rs index 59d1ce3..423510b 100644 --- a/pytests/test_tokio_multi_thread_run_forever.rs +++ b/pytests/test_tokio_multi_thread_run_forever.rs @@ -1,6 +1,8 @@ mod tokio_run_forever; fn main() { + pyo3::prepare_freethreaded_python(); + pyo3_asyncio::tokio::init_multi_thread(); tokio_run_forever::test_main(); diff --git a/src/async_std.rs b/src/async_std.rs index 3718204..2b98304 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -144,6 +144,8 @@ pub fn get_current_loop(py: Python) -> PyResult<&PyAny> { /// # /// # use pyo3::prelude::*; /// # +/// # pyo3::prepare_freethreaded_python(); +/// # /// # Python::with_gil(|py| { /// # pyo3_asyncio::with_runtime(py, || { /// pyo3_asyncio::async_std::run_until_complete(py, async move { @@ -179,6 +181,9 @@ where /// # use pyo3::prelude::*; /// # /// fn main() { +/// // call this or use pyo3 0.14 "auto-initialize" feature +/// pyo3::prepare_freethreaded_python(); +/// /// Python::with_gil(|py| { /// pyo3_asyncio::async_std::run(py, async move { /// async_std::task::sleep(Duration::from_secs(1)).await; diff --git a/src/lib.rs b/src/lib.rs index 4548d51..6dcdac1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -181,6 +181,9 @@ fn create_future(event_loop: &PyAny) -> PyResult<&PyAny> { /// use pyo3::prelude::*; /// /// fn main() { +/// // Call this function or use pyo3's "auto-initialize" feature +/// pyo3::prepare_freethreaded_python(); +/// /// Python::with_gil(|py| { /// pyo3_asyncio::with_runtime(py, || { /// println!("PyO3 Asyncio Initialized!"); @@ -310,6 +313,9 @@ pub fn get_event_loop(py: Python) -> &PyAny { /// fn main() -> pyo3::PyResult<()> { /// use std::time::Duration; /// use pyo3::prelude::*; +/// +/// // call this or use pyo3 0.14 "auto-initialize" feature +/// pyo3::prepare_freethreaded_python(); /// /// Python::with_gil(|py| { /// pyo3_asyncio::with_runtime(py, || { diff --git a/src/testing.rs b/src/testing.rs index 0b43ef3..9e8bd0e 100644 --- a/src/testing.rs +++ b/src/testing.rs @@ -155,7 +155,7 @@ //! # ))] //! mod tests { //! use pyo3::prelude::*; -//! +//! //! # #[cfg(feature = "async-std-runtime")] //! #[pyo3_asyncio::async_std::test] //! async fn test_async_std_async_test_compiles() -> PyResult<()> { diff --git a/src/tokio.rs b/src/tokio.rs index 8540ed8..ea2e84d 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -220,6 +220,7 @@ pub fn init_current_thread_once() { /// # .build() /// # .expect("Couldn't build the runtime"); /// # +/// # pyo3::prepare_freethreaded_python(); /// # Python::with_gil(|py| { /// # pyo3_asyncio::with_runtime(py, || { /// # pyo3_asyncio::tokio::init_current_thread(); From 1c4f5209c97f046202806aca036b964e98a0eab3 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Fri, 6 Aug 2021 15:12:26 -0500 Subject: [PATCH 21/32] Added test for running asyncio.run multiple times, found new issue that needs to be documented --- Cargo.toml | 3 ++ pytests/test_async_std_asyncio.rs | 74 +++++++++++++++++++++++++++---- pytests/tokio_asyncio/mod.rs | 70 +++++++++++++++++++++++++---- src/async_std.rs | 9 ++-- src/generic.rs | 17 ++++--- src/tokio.rs | 9 ++-- 6 files changed, 149 insertions(+), 33 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c517d38..715268d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,9 @@ once_cell = "1.5" pyo3 = "0.14" pyo3-asyncio-macros = { path = "pyo3-asyncio-macros", version = "=0.14.0", optional = true } +[dev-dependencies] +pyo3 = { version = "0.14", features = ["macros"] } + [dependencies.async-std] version = "1.9" features = ["unstable"] diff --git a/pytests/test_async_std_asyncio.rs b/pytests/test_async_std_asyncio.rs index 4eea3a5..a1b9e35 100644 --- a/pytests/test_async_std_asyncio.rs +++ b/pytests/test_async_std_asyncio.rs @@ -3,7 +3,12 @@ mod common; use std::{rc::Rc, time::Duration}; use async_std::task; -use pyo3::{prelude::*, types::PyType, wrap_pyfunction}; +use pyo3::{ + prelude::*, + proc_macro::pymodule, + types::{IntoPyDict, PyType}, + wrap_pyfunction, wrap_pymodule, +}; #[pyfunction] #[allow(deprecated)] @@ -27,8 +32,9 @@ fn sleep<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { } #[pyo3_asyncio::async_std::test] -async fn test_into_coroutine() -> PyResult<()> { - let fut = Python::with_gil(|py| { +fn test_into_coroutine() -> PyResult<()> { + #[allow(deprecated)] + Python::with_gil(|py| { let sleeper_mod = PyModule::new(py, "rust_sleeper")?; sleeper_mod.add_wrapped(wrap_pyfunction!(sleep_into_coroutine))?; @@ -40,15 +46,21 @@ async fn test_into_coroutine() -> PyResult<()> { "test_into_coroutine_mod", )?; - pyo3_asyncio::async_std::into_future(test_mod.call_method1( + let fut = pyo3_asyncio::into_future(test_mod.call_method1( "sleep_for_1s", (sleeper_mod.getattr("sleep_into_coroutine")?,), - )?) - })?; + )?)?; - fut.await?; + pyo3_asyncio::async_std::run_until_complete( + pyo3_asyncio::get_event_loop(py), + async move { + fut.await?; + Ok(()) + }, + )?; - Ok(()) + Ok(()) + }) } #[pyo3_asyncio::async_std::test] @@ -189,6 +201,52 @@ async fn test_cancel() -> PyResult<()> { Ok(()) } +/// This module is implemented in Rust. +#[pymodule] +fn test_mod(_py: Python, m: &PyModule) -> PyResult<()> { + #![allow(deprecated)] + #[pyfn(m, "sleep")] + fn sleep(py: Python) -> PyResult<&PyAny> { + pyo3_asyncio::async_std::future_into_py(py, async move { + async_std::task::sleep(Duration::from_millis(500)).await; + Ok(Python::with_gil(|py| py.None())) + }) + } + + Ok(()) +} + +const TEST_CODE: &str = r#" +async def main(): + return await test_mod.sleep() + +asyncio.run(main()) +"#; + +#[pyo3_asyncio::async_std::test] +fn test_multiple_asyncio_run() -> PyResult<()> { + Python::with_gil(|py| { + pyo3_asyncio::async_std::run(py, async move { + async_std::task::sleep(Duration::from_millis(500)).await; + Ok(()) + })?; + pyo3_asyncio::async_std::run(py, async move { + async_std::task::sleep(Duration::from_millis(500)).await; + Ok(()) + })?; + + let d = [ + ("asyncio", py.import("asyncio")?.into()), + ("test_mod", wrap_pymodule!(test_mod)(py)), + ] + .into_py_dict(py); + + py.run(TEST_CODE, Some(d), None)?; + py.run(TEST_CODE, Some(d), None)?; + Ok(()) + }) +} + #[allow(deprecated)] fn main() -> pyo3::PyResult<()> { pyo3::prepare_freethreaded_python(); diff --git a/pytests/tokio_asyncio/mod.rs b/pytests/tokio_asyncio/mod.rs index 78bfdec..21339a9 100644 --- a/pytests/tokio_asyncio/mod.rs +++ b/pytests/tokio_asyncio/mod.rs @@ -1,6 +1,11 @@ use std::{rc::Rc, time::Duration}; -use pyo3::{prelude::*, types::PyType, wrap_pyfunction}; +use pyo3::{ + prelude::*, + proc_macro::pymodule, + types::{IntoPyDict, PyType}, + wrap_pyfunction, wrap_pymodule, +}; use crate::common; @@ -26,8 +31,9 @@ fn sleep<'p>(py: Python<'p>, secs: &'p PyAny) -> PyResult<&'p PyAny> { } #[pyo3_asyncio::tokio::test] -async fn test_into_coroutine() -> PyResult<()> { - let fut = Python::with_gil(|py| { +fn test_into_coroutine() -> PyResult<()> { + #[allow(deprecated)] + Python::with_gil(|py| { let sleeper_mod = PyModule::new(py, "rust_sleeper")?; sleeper_mod.add_wrapped(wrap_pyfunction!(sleep_into_coroutine))?; @@ -39,15 +45,18 @@ async fn test_into_coroutine() -> PyResult<()> { "test_into_coroutine_mod", )?; - pyo3_asyncio::tokio::into_future(test_mod.call_method1( + let fut = pyo3_asyncio::into_future(test_mod.call_method1( "sleep_for_1s", (sleeper_mod.getattr("sleep_into_coroutine")?,), - )?) - })?; + )?)?; - fut.await?; + pyo3_asyncio::tokio::run_until_complete(pyo3_asyncio::get_event_loop(py), async move { + fut.await?; + Ok(()) + })?; - Ok(()) + Ok(()) + }) } #[pyo3_asyncio::tokio::test] @@ -198,3 +207,48 @@ async fn test_cancel() -> PyResult<()> { Ok(()) } +/// This module is implemented in Rust. +#[pymodule] +fn test_mod(_py: Python, m: &PyModule) -> PyResult<()> { + #![allow(deprecated)] + #[pyfn(m, "sleep")] + fn sleep(py: Python) -> PyResult<&PyAny> { + pyo3_asyncio::async_std::future_into_py(py, async move { + async_std::task::sleep(Duration::from_millis(500)).await; + Ok(Python::with_gil(|py| py.None())) + }) + } + + Ok(()) +} + +const TEST_CODE: &str = r#" +async def main(): + return await test_mod.sleep() + +asyncio.run(main()) +"#; + +#[pyo3_asyncio::tokio::test] +fn test_multiple_asyncio_run() -> PyResult<()> { + Python::with_gil(|py| { + pyo3_asyncio::tokio::run(py, async move { + tokio::time::sleep(Duration::from_millis(500)).await; + Ok(()) + })?; + pyo3_asyncio::tokio::run(py, async move { + tokio::time::sleep(Duration::from_millis(500)).await; + Ok(()) + })?; + + let d = [ + ("asyncio", py.import("asyncio")?.into()), + ("test_mod", wrap_pymodule!(test_mod)(py)), + ] + .into_py_dict(py); + + py.run(TEST_CODE, Some(d), None)?; + py.run(TEST_CODE, Some(d), None)?; + Ok(()) + }) +} diff --git a/src/async_std.rs b/src/async_std.rs index 2b98304..b5f0835 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -134,7 +134,7 @@ pub fn get_current_loop(py: Python) -> PyResult<&PyAny> { /// [`run_forever`](`crate::run_forever`) /// /// # Arguments -/// * `py` - The current PyO3 GIL guard +/// * `event_loop` - The Python event loop that should run the future /// * `fut` - The future to drive to completion /// /// # Examples @@ -148,7 +148,8 @@ pub fn get_current_loop(py: Python) -> PyResult<&PyAny> { /// # /// # Python::with_gil(|py| { /// # pyo3_asyncio::with_runtime(py, || { -/// pyo3_asyncio::async_std::run_until_complete(py, async move { +/// # let event_loop = py.import("asyncio")?.call_method0("new_event_loop")?; +/// pyo3_asyncio::async_std::run_until_complete(event_loop, async move { /// async_std::task::sleep(Duration::from_secs(1)).await; /// Ok(()) /// })?; @@ -160,11 +161,11 @@ pub fn get_current_loop(py: Python) -> PyResult<&PyAny> { /// # .unwrap(); /// # }); /// ``` -pub fn run_until_complete(py: Python, fut: F) -> PyResult<()> +pub fn run_until_complete(event_loop: &PyAny, fut: F) -> PyResult<()> where F: Future> + Send + 'static, { - generic::run_until_complete::(py, fut) + generic::run_until_complete::(event_loop, fut) } /// Run the event loop until the given Future completes diff --git a/src/generic.rs b/src/generic.rs index 79074b3..4933c8b 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -4,8 +4,8 @@ use pyo3::{prelude::*, PyNativeType}; #[allow(deprecated)] use crate::{ - asyncio_get_event_loop, call_soon_threadsafe, close, create_future, dump_err, err::RustPanic, - get_event_loop, get_running_loop, into_future_with_loop, + asyncio, call_soon_threadsafe, close, create_future, dump_err, err::RustPanic, get_event_loop, + get_running_loop, into_future_with_loop, }; /// Generic utilities for a JoinError @@ -69,7 +69,7 @@ where /// [`run_forever`](`crate::run_forever`) /// /// # Arguments -/// * `py` - The current PyO3 GIL guard +/// * `event_loop` - The Python event loop that should run the future /// * `fut` - The future to drive to completion /// /// # Examples @@ -127,8 +127,9 @@ where /// # /// # Python::with_gil(|py| { /// # pyo3_asyncio::with_runtime(py, || { +/// # let event_loop = py.import("asyncio")?.call_method0("new_event_loop")?; /// # #[cfg(feature = "tokio-runtime")] -/// pyo3_asyncio::generic::run_until_complete::(py, async move { +/// pyo3_asyncio::generic::run_until_complete::(event_loop, async move { /// tokio::time::sleep(Duration::from_secs(1)).await; /// Ok(()) /// })?; @@ -140,13 +141,11 @@ where /// # .unwrap(); /// # }); /// ``` -pub fn run_until_complete(py: Python, fut: F) -> PyResult<()> +pub fn run_until_complete(event_loop: &PyAny, fut: F) -> PyResult<()> where R: Runtime, F: Future> + Send + 'static, { - let event_loop = asyncio_get_event_loop(py)?; - let coro = future_into_py_with_loop::(event_loop, async move { fut.await?; Ok(Python::with_gil(|py| py.None())) @@ -235,9 +234,9 @@ where R: Runtime, F: Future> + Send + 'static, { - let event_loop = asyncio_get_event_loop(py)?; + let event_loop = asyncio(py)?.call_method0("new_event_loop")?; - let result = run_until_complete::(py, fut); + let result = run_until_complete::(event_loop, fut); close(event_loop)?; diff --git a/src/tokio.rs b/src/tokio.rs index ea2e84d..d8db3fd 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -204,7 +204,7 @@ pub fn init_current_thread_once() { /// [`crate::run_forever`] /// /// # Arguments -/// * `py` - The current PyO3 GIL guard +/// * `event_loop` - The Python event loop that should run the future /// * `fut` - The future to drive to completion /// /// # Examples @@ -224,7 +224,8 @@ pub fn init_current_thread_once() { /// # Python::with_gil(|py| { /// # pyo3_asyncio::with_runtime(py, || { /// # pyo3_asyncio::tokio::init_current_thread(); -/// pyo3_asyncio::tokio::run_until_complete(py, async move { +/// # let event_loop = py.import("asyncio")?.call_method0("new_event_loop")?; +/// pyo3_asyncio::tokio::run_until_complete(event_loop, async move { /// tokio::time::sleep(Duration::from_secs(1)).await; /// Ok(()) /// })?; @@ -236,11 +237,11 @@ pub fn init_current_thread_once() { /// # .unwrap(); /// # }); /// ``` -pub fn run_until_complete(py: Python, fut: F) -> PyResult<()> +pub fn run_until_complete(event_loop: &PyAny, fut: F) -> PyResult<()> where F: Future> + Send + 'static, { - generic::run_until_complete::(py, fut) + generic::run_until_complete::(event_loop, fut) } /// Run the event loop until the given Future completes From 27ad604100250f08ebc23f893cd5e409ac6d1e98 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sat, 7 Aug 2021 08:00:37 -0500 Subject: [PATCH 22/32] Added PyO3 guide to README --- README.md | 290 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 289 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 10f55eb..cb253fc 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,295 @@ Type "help", "copyright", "credits" or "license" for more information. > > This restriction may be lifted in a future release. +## PyO3 Asyncio Primer + +If you are working with a Python library that makes use of async functions or wish to provide +Python bindings for an async Rust library, [`pyo3-asyncio`](https://github.com/awestlake87/pyo3-asyncio) +likely has the tools you need. It provides conversions between async functions in both Python and +Rust and was designed with first-class support for popular Rust runtimes such as +[`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). In addition, all async Python +code runs on the default `asyncio` event loop, so `pyo3-asyncio` should work just fine with existing +Python libraries. + +In the following sections, we'll give a general overview of `pyo3-asyncio` explaining how to call +async Python functions with PyO3, how to call async Rust functions from Python, and how to configure +your codebase to manage the runtimes of both. + +## Awaiting an Async Python Function in Rust + +Let's take a look at a dead simple async Python function: + +```python +# Sleep for 1 second +async def py_sleep(): + await asyncio.sleep(1) +``` + +**Async functions in Python are simply functions that return a `coroutine` object**. For our purposes, +we really don't need to know much about these `coroutine` objects. The key factor here is that calling +an `async` function is _just like calling a regular function_, the only difference is that we have +to do something special with the object that it returns. + +Normally in Python, that something special is the `await` keyword, but in order to await this +coroutine in Rust, we first need to convert it into Rust's version of a `coroutine`: a `Future`. +That's where `pyo3-asyncio` comes in. +[`pyo3_asyncio::into_future`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.into_future.html) +performs this conversion for us: + + +```rust no_run +use pyo3::prelude::*; + +#[pyo3_asyncio::tokio::main] +async fn main() -> PyResult<()> { + let future = Python::with_gil(|py| -> PyResult<_> { + // import the module containing the py_sleep function + let example = py.import("example")?; + + // calling the py_sleep method like a normal function + // returns a coroutine + let coroutine = example.call_method0("py_sleep")?; + + // convert the coroutine into a Rust future using the + // tokio runtime + pyo3_asyncio::tokio::into_future(coroutine) + })?; + + // await the future + future.await?; + + Ok(()) +} +``` + +> If you're interested in learning more about `coroutines` and `awaitables` in general, check out the +> [Python 3 `asyncio` docs](https://docs.python.org/3/library/asyncio-task.html) for more information. + +## Awaiting a Rust Future in Python + +Here we have the same async function as before written in Rust using the +[`async-std`](https://async.rs/) runtime: + +```rust +/// Sleep for 1 second +async fn rust_sleep() { + async_std::task::sleep(std::time::Duration::from_secs(1)).await; +} +``` + +Similar to Python, Rust's async functions also return a special object called a +`Future`: + +```rust compile_fail +let future = rust_sleep(); +``` + +We can convert this `Future` object into Python to make it `awaitable`. This tells Python that you +can use the `await` keyword with it. In order to do this, we'll call +[`pyo3_asyncio::async_std::future_into_py`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.future_into_py.html): + +```rust +use pyo3::prelude::*; + +async fn rust_sleep() { + async_std::task::sleep(std::time::Duration::from_secs(1)).await; +} + +#[pyfunction] +fn call_rust_sleep(py: Python) -> PyResult<&PyAny> { + pyo3_asyncio::async_std::future_into_py(py, async move { + rust_sleep().await; + Ok(Python::with_gil(|py| py.None())) + }) +} +``` + +In Python, we can call this pyo3 function just like any other async function: + +```python +from example import call_rust_sleep + +async def rust_sleep(): + await call_rust_sleep() +``` + +## Managing Event Loops + +Python's event loop requires some special treatment, especially regarding the main thread. Some of +Python's `asyncio` features, like proper signal handling, require control over the main thread, which +doesn't always play well with Rust. + +Luckily, Rust's event loops are pretty flexible and don't _need_ control over the main thread, so in +`pyo3-asyncio`, we decided the best way to handle Rust/Python interop was to just surrender the main +thread to Python and run Rust's event loops in the background. Unfortunately, since most event loop +implementations _prefer_ control over the main thread, this can still make some things awkward. + +### PyO3 Asyncio Initialization + +Because Python needs to control the main thread, we can't use the convenient proc macros from Rust +runtimes to handle the `main` function or `#[test]` functions. Instead, the initialization for PyO3 has to be done from the `main` function and the main +thread must block on [`pyo3_asyncio::run_forever`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.run_forever.html) or [`pyo3_asyncio::async_std::run_until_complete`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.run_until_complete.html). + +Because we have to block on one of those functions, we can't use [`#[async_std::main]`](https://docs.rs/async-std/latest/async_std/attr.main.html) or [`#[tokio::main]`](https://docs.rs/tokio/1.1.0/tokio/attr.main.html) +since it's not a good idea to make long blocking calls during an async function. + +> Internally, these `#[main]` proc macros are expanded to something like this: +> ```rust compile_fail +> fn main() { +> // your async main fn +> async fn _main_impl() { /* ... */ } +> Runtime::new().block_on(_main_impl()); +> } +> ``` +> Making a long blocking call inside the `Future` that's being driven by `block_on` prevents that +> thread from doing anything else and can spell trouble for some runtimes (also this will actually +> deadlock a single-threaded runtime!). Many runtimes have some sort of `spawn_blocking` mechanism +> that can avoid this problem, but again that's not something we can use here since we need it to +> block on the _main_ thread. + +For this reason, `pyo3-asyncio` provides its own set of proc macros to provide you with this +initialization. These macros are intended to mirror the initialization of `async-std` and `tokio` +while also satisfying the Python runtime's needs. + +Here's a full example of PyO3 initialization with the `async-std` runtime: +```rust no_run +use pyo3::prelude::*; + +#[pyo3_asyncio::async_std::main] +async fn main() -> PyResult<()> { + // PyO3 is initialized - Ready to go + + let fut = Python::with_gil(|py| -> PyResult<_> { + let asyncio = py.import("asyncio")?; + + // convert asyncio.sleep into a Rust Future + pyo3_asyncio::async_std::into_future( + asyncio.call_method1("sleep", (1.into_py(py),))? + ) + })?; + + fut.await?; + + Ok(()) +} +``` + +## PyO3 Asyncio in Cargo Tests + +The default Cargo Test harness does not currently allow test crates to provide their own `main` +function, so there doesn't seem to be a good way to allow Python to gain control over the main +thread. + +We can, however, override the default test harness and provide our own. `pyo3-asyncio` provides some +utilities to help us do just that! In the following sections, we will provide an overview for +constructing a Cargo integration test with `pyo3-asyncio` and adding your tests to it. + +### Main Test File +First, we need to create the test's main file. Although these tests are considered integration +tests, we cannot put them in the `tests` directory since that is a special directory owned by +Cargo. Instead, we put our tests in a `pytests` directory. + +> The name `pytests` is just a convention. You can name this folder anything you want in your own +> projects. + +We'll also want to provide the test's main function. Most of the functionality that the test harness needs is packed in the [`pyo3_asyncio::testing::main`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/testing/fn.main.html) function. This function will parse the test's CLI arguments, collect and pass the functions marked with [`#[pyo3_asyncio::async_std::test]`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/attr.test.html) or [`#[pyo3_asyncio::tokio::test]`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/tokio/attr.test.html) and pass them into the test harness for running and filtering. + +`pytests/test_example.rs` for the `tokio` runtime: +```rust +#[pyo3_asyncio::tokio::main] +async fn main() -> pyo3::PyResult<()> { + pyo3_asyncio::testing::main().await +} +``` + +`pytests/test_example.rs` for the `async-std` runtime: +```rust +#[pyo3_asyncio::async_std::main] +async fn main() -> pyo3::PyResult<()> { + pyo3_asyncio::testing::main().await +} +``` + +### Cargo Configuration +Next, we need to add our test file to the Cargo manifest by adding the following section to the +`Cargo.toml` + +```toml +[[test]] +name = "test_example" +path = "pytests/test_example.rs" +harness = false +``` + +Also add the `testing` and `attributes` features to the `pyo3-asyncio` dependency and select your preferred runtime: + +```toml +pyo3-asyncio = { version = "0.13", features = ["testing", "attributes", "async-std-runtime"] } +``` + +At this point, you should be able to run the test via `cargo test` + +### Adding Tests to the PyO3 Asyncio Test Harness + +We can add tests anywhere in the test crate with the runtime's corresponding `#[test]` attribute: + +For `async-std` use the [`pyo3_asyncio::async_std::test`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/attr.test.html) attribute: +```rust +mod tests { + use std::{time::Duration, thread}; + + use pyo3::prelude::*; + + // tests can be async + #[pyo3_asyncio::async_std::test] + async fn test_async_sleep() -> PyResult<()> { + async_std::task::sleep(Duration::from_secs(1)).await; + Ok(()) + } + + // they can also be synchronous + #[pyo3_asyncio::async_std::test] + fn test_blocking_sleep() -> PyResult<()> { + thread::sleep(Duration::from_secs(1)); + Ok(()) + } +} + +#[pyo3_asyncio::async_std::main] +async fn main() -> pyo3::PyResult<()> { + pyo3_asyncio::testing::main().await +} +``` + +For `tokio` use the [`pyo3_asyncio::tokio::test`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/tokio/attr.test.html) attribute: +```rust +mod tests { + use std::{time::Duration, thread}; + + use pyo3::prelude::*; + + // tests can be async + #[pyo3_asyncio::tokio::test] + async fn test_async_sleep() -> PyResult<()> { + tokio::time::sleep(Duration::from_secs(1)).await; + Ok(()) + } + + // they can also be synchronous + #[pyo3_asyncio::tokio::test] + fn test_blocking_sleep() -> PyResult<()> { + thread::sleep(Duration::from_secs(1)); + Ok(()) + } +} + +#[pyo3_asyncio::tokio::main] +async fn main() -> pyo3::PyResult<()> { + pyo3_asyncio::testing::main().await +} +``` + ## MSRV Currently the MSRV for this library is 1.46.0, _but_ if you don't need to use the `async-std-runtime` feature, you can use rust 1.45.0. -> `async-std` depends on `socket2` which fails to compile under 1.45.0. \ No newline at end of file +> `async-std` depends on `socket2` which fails to compile under 1.45.0. From ca89b5cafe774b9daa2a89e881978bdc89ec22af Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sat, 7 Aug 2021 11:50:47 -0500 Subject: [PATCH 23/32] Added a section about Event Loop references and thread-awareness to the README --- README.md | 204 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 196 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index cb253fc..4f952e2 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ Here we initialize the runtime, import Python's `asyncio` library and run the gi ```toml # Cargo.toml dependencies [dependencies] -pyo3 = { version = "0.13" } -pyo3-asyncio = { version = "0.13", features = ["attributes", "async-std-runtime"] } +pyo3 = { version = "0.14" } +pyo3-asyncio = { version = "0.14", features = ["attributes", "async-std-runtime"] } async-std = "1.9" ``` @@ -60,8 +60,8 @@ attribute. ```toml # Cargo.toml dependencies [dependencies] -pyo3 = { version = "0.13" } -pyo3-asyncio = { version = "0.13", features = ["attributes", "tokio-runtime"] } +pyo3 = { version = "0.14" } +pyo3-asyncio = { version = "0.14", features = ["attributes", "tokio-runtime"] } tokio = "1.4" ``` @@ -84,7 +84,7 @@ async fn main() -> PyResult<()> { } ``` -More details on the usage of this library can be found in the [API docs](https://awestlake87.github.io/pyo3-asyncio/master/doc). +More details on the usage of this library can be found in the [API docs](https://awestlake87.github.io/pyo3-asyncio/master/doc) and the primer below. ### PyO3 Native Rust Modules @@ -104,7 +104,7 @@ For `async-std`: ```toml [dependencies] pyo3 = { version = "0.13", features = ["extension-module"] } -pyo3-asyncio = { version = "0.13", features = ["async-std-runtime"] } +pyo3-asyncio = { version = "0.14", features = ["async-std-runtime"] } async-std = "1.9" ``` @@ -112,7 +112,7 @@ For `tokio`: ```toml [dependencies] pyo3 = { version = "0.13", features = ["extension-module"] } -pyo3-asyncio = { version = "0.13", features = ["tokio-runtime"] } +pyo3-asyncio = { version = "0.14", features = ["tokio-runtime"] } tokio = "1.4" ``` @@ -158,7 +158,6 @@ fn rust_sleep(py: Python) -> PyResult<&PyAny> { #[pymodule] fn my_async_module(py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; - Ok(()) } @@ -362,6 +361,195 @@ async fn main() -> PyResult<()> { } ``` +### Event Loop References and Thread-awareness + +One problem that arises when interacting with Python's asyncio library is that the functions we use to get a reference to the Python event loop can only be called in certain contexts. Since PyO3 Asyncio requires references to the event loop when performing conversions between Rust and Python, this is unfortunately something that you need to worry about. + +#### The Main Dilemma + +Python programs can have many independent event loop instances throughout the lifetime of the application (`asyncio.run` for example creates its own event loop each time it's called for instance), and they can even run concurrent with other event loops. The most correct method of obtaining a reference to the Python event loop is via `asyncio.get_running_loop`. + +`asyncio.get_running_loop` returns the event loop associated with the current OS thread. It can be used inside Python coroutines to spawn concurrent tasks, interact with timers, or in our case signal between Rust and Python. This is all well and good when we are operating on a Python thread, but what happens when we want to perform a PyO3 Asyncio conversion on a _Rust_ thread? Since the Rust thread is not associated with a Python event loop, `asyncio.get_running_loop` will fail. + +#### The Solution + +A really straightforward way of dealing with this problem is to pass a reference to the associated Python event loop for every conversion. That's why in PyO3 Asyncio, we introduced a new set of conversion functions that do just that: + +- `pyo3_asyncio::into_future_with_loop` - Convert a Python awaitable into a Rust future with the given asyncio event loop. +- `pyo3_asyncio::::future_into_py_with_loop` - Convert a Rust future into a Python awaitable with the given asyncio event loop. +- `pyo3_asyncio::::local_future_into_py_with_loop` - Convert a `!Send` Rust future into a Python awaitable with the given asyncio event loop. + +One clear disadvantage to this approach (besides the verbose naming) is that the Rust application has to explicitly track its references to the Python event loop. In native libraries, we can't make any assumptions about the underlying event loop, so the only reliable way to make sure our conversions work properly is to store a reference to the current event loop to use later on. + +```rust +use pyo3::prelude::*; + +#[pyfunction] +fn sleep(py: Python) -> PyResult<&PyAny> { + let current_loop = pyo3_asyncio::get_running_loop(py)?; + let loop_ref = PyObject::from(current_loop); + + // Convert the async move { } block to a Python awaitable + pyo3_asyncio::tokio::future_into_py_with_loop(current_loop, async move { + let py_sleep = Python::with_gil(|py| { + // Sometimes we need to call other async Python functions within + // this future. In order for this to work, we need to track the + // event loop from earlier. + pyo3_asyncio::into_future_with_loop( + loop_ref.as_ref(py), + py.import("asyncio")?.call_method1("sleep", (1,))? + ) + })?; + + py_sleep.await?; + + Ok(Python::with_gil(|py| py.None())) + }) +} + +#[pymodule] +fn my_mod(py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(sleep, m)?)?; + Ok(()) +} +``` + +> A naive solution to this tracking problem would be to cache a global reference to the asyncio event loop that all PyO3 Asyncio conversions can use. In fact this is what we did in PyO3 Asyncio `v0.13`. This works well for applications, but it soon became clear that this is not so ideal for libraries. Libraries usually have no direct control over how the event loop is managed, they're just expected to work with any event loop at any point in the application. This problem is compounded further when multiple event loops are used in the application since the global reference will only point to one. + +Another disadvantage to this explicit approach that is less obvious is that we can no longer call our `#[pyfunction] fn sleep` on a Rust runtime since `asyncio.get_running_loop` only works on Python threads! It's clear that we need a slightly more flexible approach. + +In order to detect the Python event loop at the callsite, we need something like `asyncio.get_running_loop` that works for _both Python and Rust_. In Python, `asyncio.get_running_loop` uses thread-local data to retrieve the event loop associated with the current thread. What we need in Rust is something that can retrieve the Python event loop associated with the current _task_. + +Enter `pyo3_asyncio::::get_current_loop`. This function first checks task-local data for a Python event loop, then falls back on `asyncio.get_running_loop` if no task-local event loop is found. This way both bases are convered. + +Now, all we need is a way to store the event loop in task-local data. Since this is a runtime-specific feature, you can find the following functions in each runtime module: + +- `pyo3_asyncio::::scope` - Store the event loop in task-local data when executing the given Future. +- `pyo3_asyncio::::scope_local` - Store the event loop in task-local data when executing the given `!Send` Future. + +With these new functions, we can make our previous example more correct: + +```rust no_run +use pyo3::prelude::*; + +#[pyfunction] +fn sleep(py: Python) -> PyResult<&PyAny> { + // get the current event loop through task-local data + // OR `asyncio.get_running_loop` + let current_loop = pyo3_asyncio::tokio::get_current_loop(py)?; + + pyo3_asyncio::tokio::future_into_py_with_loop( + current_loop, + pyo3_asyncio::tokio::scope(current_loop.into(), async move { + let py_sleep = Python::with_gil(|py| { + pyo3_asyncio::into_future_with_loop( + // Now we can get the current loop through task-local data + pyo3_asyncio::tokio::get_current_loop(py)?, + py.import("asyncio")?.call_method1("sleep", (1,))? + ) + })?; + + py_sleep.await?; + + Ok(Python::with_gil(|py| py.None())) + }) + ) +} + +#[pyfunction] +fn wrap_sleep(py: Python) -> PyResult<&PyAny> { + // get the current event loop through task-local data + // OR `asyncio.get_running_loop` + let current_loop = pyo3_asyncio::tokio::get_current_loop(py)?; + + pyo3_asyncio::tokio::future_into_py_with_loop( + current_loop, + pyo3_asyncio::tokio::scope(current_loop.into(), async move { + let py_sleep = Python::with_gil(|py| { + pyo3_asyncio::into_future_with_loop( + pyo3_asyncio::tokio::get_current_loop(py)?, + // We can also call sleep within a Rust task since the + // event loop is stored in task local data + sleep(py)? + ) + })?; + + py_sleep.await?; + + Ok(Python::with_gil(|py| py.None())) + }) + ) +} + +#[pymodule] +fn my_mod(py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(sleep, m)?)?; + m.add_function(wrap_pyfunction!(wrap_sleep, m)?)?; + Ok(()) +} +``` + +Even though this is more correct, it's clearly not more ergonomic. That's why we introduced a new set of functions with this functionality baked in: + +- `pyo3_asyncio::::into_future` - Convert a Python awaitable into a Rust future (using `pyo3_asyncio::::get_current_loop`) +- `pyo3_asyncio::::future_into_py` - Convert a Rust future into a Python awaitable (using `pyo3_asyncio::::get_current_loop` and `pyo3_asyncio::::scope` to set the task-local event loop for the given Rust future) +- `pyo3_asyncio::::local_future_into_py` - Convert a `!Send` Rust future into a Python awaitable (using `pyo3_asyncio::::get_current_loop` and `pyo3_asyncio::::scope_local` to set the task-local event loop for the given Rust future). + +__These are the functions that we recommend using__. With these functions, the previous example can be written like so: + +```rust +use pyo3::prelude::*; + +#[pyfunction] +fn sleep(py: Python) -> PyResult<&PyAny> { + pyo3_asyncio::tokio::future_into_py(py, async move { + let py_sleep = Python::with_gil(|py| { + pyo3_asyncio::tokio::into_future( + py.import("asyncio")?.call_method1("sleep", (1,))? + ) + })?; + + py_sleep.await?; + + Ok(Python::with_gil(|py| py.None())) + }) +} + +#[pyfunction] +fn wrap_sleep(py: Python) -> PyResult<&PyAny> { + pyo3_asyncio::tokio::future_into_py(py, async move { + let py_sleep = Python::with_gil(|py| { + pyo3_asyncio::tokio::into_future(sleep(py)?) + })?; + + py_sleep.await?; + + Ok(Python::with_gil(|py| py.None())) + }) +} + +#[pymodule] +fn my_mod(py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(sleep, m)?)?; + m.add_function(wrap_pyfunction!(wrap_sleep, m)?)?; + Ok(()) +} +``` + +### A Note for `v0.13` Users + +Hey guys, I realize that these are pretty major changes for `v0.14`, and I apologize in advance for having to modify the public API so much. I hope +the explanation above gives some much needed context and justification for all the breaking changes. + +Part of the reason why it's taken so long to push out a `v0.14` release is because I wanted to make sure we got this release right. There were a lot of issues with the `v0.13` release that I hadn't anticipated, and it's thanks to your feedback and patience that we've worked through these issues to get a more correct, more flexible version out there! + +This new release should address most the core issues that users have reported in the `v0.13` release, so I think we can expect more stability going forward. + +Also, a special thanks to @ShadowJonathan for helping with the design and review +of these changes! + +- @awestlake87 + ## PyO3 Asyncio in Cargo Tests The default Cargo Test harness does not currently allow test crates to provide their own `main` From 9daa361caadf15dd77879afb8f37ab3e09f15967 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sat, 7 Aug 2021 12:11:17 -0500 Subject: [PATCH 24/32] Made some README changes after proofread on GitHub --- README.md | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4f952e2..5b7aea7 100644 --- a/README.md +++ b/README.md @@ -363,23 +363,25 @@ async fn main() -> PyResult<()> { ### Event Loop References and Thread-awareness -One problem that arises when interacting with Python's asyncio library is that the functions we use to get a reference to the Python event loop can only be called in certain contexts. Since PyO3 Asyncio requires references to the event loop when performing conversions between Rust and Python, this is unfortunately something that you need to worry about. +One problem that arises when interacting with Python's asyncio library is that the functions we use to get a reference to the Python event loop can only be called in certain contexts. Since PyO3 Asyncio needs to interact with Python's event loop during conversions, the context of these conversions can matter a lot. + +> The core conversions we've mentioned so far in this guide should insulate you from these concerns in most cases, but in the event that they don't, this section should provide you with the information you need to solve these problems. #### The Main Dilemma -Python programs can have many independent event loop instances throughout the lifetime of the application (`asyncio.run` for example creates its own event loop each time it's called for instance), and they can even run concurrent with other event loops. The most correct method of obtaining a reference to the Python event loop is via `asyncio.get_running_loop`. +Python programs can have many independent event loop instances throughout the lifetime of the application (`asyncio.run` for example creates its own event loop each time it's called for instance), and they can even run concurrent with other event loops. For this reason, the most correct method of obtaining a reference to the Python event loop is via `asyncio.get_running_loop`. -`asyncio.get_running_loop` returns the event loop associated with the current OS thread. It can be used inside Python coroutines to spawn concurrent tasks, interact with timers, or in our case signal between Rust and Python. This is all well and good when we are operating on a Python thread, but what happens when we want to perform a PyO3 Asyncio conversion on a _Rust_ thread? Since the Rust thread is not associated with a Python event loop, `asyncio.get_running_loop` will fail. +`asyncio.get_running_loop` returns the event loop associated with the current OS thread. It can be used inside Python coroutines to spawn concurrent tasks, interact with timers, or in our case signal between Rust and Python. This is all well and good when we are operating on a Python thread, but since Rust threads are not associated with a Python event loop, `asyncio.get_running_loop` will fail when called on a Rust runtime. #### The Solution -A really straightforward way of dealing with this problem is to pass a reference to the associated Python event loop for every conversion. That's why in PyO3 Asyncio, we introduced a new set of conversion functions that do just that: +A really straightforward way of dealing with this problem is to pass a reference to the associated Python event loop for every conversion. That's why in `v0.14`, we introduced a new set of conversion functions that do just that: - `pyo3_asyncio::into_future_with_loop` - Convert a Python awaitable into a Rust future with the given asyncio event loop. - `pyo3_asyncio::::future_into_py_with_loop` - Convert a Rust future into a Python awaitable with the given asyncio event loop. - `pyo3_asyncio::::local_future_into_py_with_loop` - Convert a `!Send` Rust future into a Python awaitable with the given asyncio event loop. -One clear disadvantage to this approach (besides the verbose naming) is that the Rust application has to explicitly track its references to the Python event loop. In native libraries, we can't make any assumptions about the underlying event loop, so the only reliable way to make sure our conversions work properly is to store a reference to the current event loop to use later on. +One clear disadvantage to this approach (aside from the verbose naming) is that the Rust application has to explicitly track its references to the Python event loop. In native libraries, we can't make any assumptions about the underlying event loop, so the only reliable way to make sure our conversions work properly is to store a reference to the current event loop at the callsite to use later on. ```rust use pyo3::prelude::*; @@ -420,7 +422,7 @@ Another disadvantage to this explicit approach that is less obvious is that we c In order to detect the Python event loop at the callsite, we need something like `asyncio.get_running_loop` that works for _both Python and Rust_. In Python, `asyncio.get_running_loop` uses thread-local data to retrieve the event loop associated with the current thread. What we need in Rust is something that can retrieve the Python event loop associated with the current _task_. -Enter `pyo3_asyncio::::get_current_loop`. This function first checks task-local data for a Python event loop, then falls back on `asyncio.get_running_loop` if no task-local event loop is found. This way both bases are convered. +Enter `pyo3_asyncio::::get_current_loop`. This function first checks task-local data for a Python event loop, then falls back on `asyncio.get_running_loop` if no task-local event loop is found. This way both bases are covered. Now, all we need is a way to store the event loop in task-local data. Since this is a runtime-specific feature, you can find the following functions in each runtime module: @@ -440,6 +442,7 @@ fn sleep(py: Python) -> PyResult<&PyAny> { pyo3_asyncio::tokio::future_into_py_with_loop( current_loop, + // Store the current loop in task-local data pyo3_asyncio::tokio::scope(current_loop.into(), async move { let py_sleep = Python::with_gil(|py| { pyo3_asyncio::into_future_with_loop( @@ -464,6 +467,7 @@ fn wrap_sleep(py: Python) -> PyResult<&PyAny> { pyo3_asyncio::tokio::future_into_py_with_loop( current_loop, + // Store the current loop in task-local data pyo3_asyncio::tokio::scope(current_loop.into(), async move { let py_sleep = Python::with_gil(|py| { pyo3_asyncio::into_future_with_loop( @@ -491,11 +495,14 @@ fn my_mod(py: Python, m: &PyModule) -> PyResult<()> { Even though this is more correct, it's clearly not more ergonomic. That's why we introduced a new set of functions with this functionality baked in: -- `pyo3_asyncio::::into_future` - Convert a Python awaitable into a Rust future (using `pyo3_asyncio::::get_current_loop`) -- `pyo3_asyncio::::future_into_py` - Convert a Rust future into a Python awaitable (using `pyo3_asyncio::::get_current_loop` and `pyo3_asyncio::::scope` to set the task-local event loop for the given Rust future) -- `pyo3_asyncio::::local_future_into_py` - Convert a `!Send` Rust future into a Python awaitable (using `pyo3_asyncio::::get_current_loop` and `pyo3_asyncio::::scope_local` to set the task-local event loop for the given Rust future). +- `pyo3_asyncio::::into_future` + > Convert a Python awaitable into a Rust future (using `pyo3_asyncio::::get_current_loop`) +- `pyo3_asyncio::::future_into_py` + > Convert a Rust future into a Python awaitable (using `pyo3_asyncio::::get_current_loop` and `pyo3_asyncio::::scope` to set the task-local event loop for the given Rust future) +- `pyo3_asyncio::::local_future_into_py` + > Convert a `!Send` Rust future into a Python awaitable (using `pyo3_asyncio::::get_current_loop` and `pyo3_asyncio::::scope_local` to set the task-local event loop for the given Rust future). -__These are the functions that we recommend using__. With these functions, the previous example can be written like so: +__These are the functions that we recommend using__. With these functions, the previous example can be rewritten to be more compact: ```rust use pyo3::prelude::*; @@ -545,10 +552,10 @@ Part of the reason why it's taken so long to push out a `v0.14` release is becau This new release should address most the core issues that users have reported in the `v0.13` release, so I think we can expect more stability going forward. -Also, a special thanks to @ShadowJonathan for helping with the design and review +Also, a special thanks to [@ShadowJonathan](https://github.com/ShadowJonathan) for helping with the design and review of these changes! -- @awestlake87 +- [@awestlake87](https://github.com/awestlake87) ## PyO3 Asyncio in Cargo Tests From 390a785c34204fe88b33b6cc2b5b5054a67e45fc Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sat, 7 Aug 2021 13:07:55 -0500 Subject: [PATCH 25/32] Added example for non-standard Python event loops --- README.md | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/README.md b/README.md index 5b7aea7..ac65738 100644 --- a/README.md +++ b/README.md @@ -361,6 +361,121 @@ async fn main() -> PyResult<()> { } ``` +### Non-standard Python Event Loops + +Python allows you to use alternatives to the default `asyncio` event loop. One +popular alternative is `uvloop`. In `v0.13` using non-standard event loops was +a bit of an ordeal, but in `v0.14` it's trivial. + +#### Using `uvloop` in a PyO3 Asyncio Native Extensions + +```toml +# Cargo.toml + +[lib] +name = "my_async_module" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.14", features = ["extension-module", "auto-initialize"] } +pyo3-asyncio = { version = "0.14", features = ["tokio-runtime"] } +async-std = "1.9" +tokio = "1.4" +``` + +```rust +//! lib.rs + +use pyo3::{prelude::*, wrap_pyfunction}; + +#[pyfunction] +fn rust_sleep(py: Python) -> PyResult<&PyAny> { + pyo3_asyncio::tokio::future_into_py(py, async { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + Ok(Python::with_gil(|py| py.None())) + }) +} + +#[pymodule] +fn my_async_module(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; + + Ok(()) +} +``` + +```bash +$ cargo build --release && mv target/release/libmy_async_module.so my_async_module.so + Compiling pyo3-asyncio-lib v0.1.0 (pyo3-asyncio-lib) + Finished release [optimized] target(s) in 1.00s +$ PYTHONPATH=target/release/ python3 +Python 3.8.8 (default, Apr 13 2021, 19:58:26) +[GCC 7.3.0] :: Anaconda, Inc. on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import asyncio +>>> import uvloop +>>> +>>> import my_async_module +>>> +>>> uvloop.install() +>>> +>>> async def main(): +... await my_async_module.rust_sleep() +... +>>> asyncio.run(main()) +>>> +``` + +#### Using `uvloop` in Rust Applications + +Using `uvloop` in Rust applications is a bit trickier, but it's still possible +with relatively few modifications. + +> Unfortunately, we can't make use of the `#[pyo3_asyncio::::main]` attribute with non-standard event loops. This is because the `#[pyo3_asyncio::::main]` proc macro has to interact with the Python +event loop before we can install the `uvloop` policy. + +```toml +[dependencies] +async-std = "1.9" +pyo3 = "0.14" +pyo3-asyncio = { version = "0.14", features = ["async-std-runtime"] } +``` + +```rust +//! main.rs + +use pyo3::{prelude::*, types::PyType}; + +fn main() -> PyResult<()> { + pyo3::prepare_freethreaded_python(); + + Python::with_gil(|py| { + let uvloop = py.import("uvloop")?; + uvloop.call_method0("install")?; + + // store a reference for the assertion + let uvloop = PyObject::from(uvloop); + + pyo3_asyncio::async_std::run(py, async move { + // verify that we are on a uvloop.Loop + Python::with_gil(|py| -> PyResult<()> { + assert!(uvloop + .as_ref(py) + .getattr("Loop")? + .downcast::() + .unwrap() + .is_instance(pyo3_asyncio::async_std::get_current_loop(py)?)?); + Ok(()) + })?; + + async_std::task::sleep(std::time::Duration::from_secs(1)).await; + + Ok(()) + }) + }) +} +``` + ### Event Loop References and Thread-awareness One problem that arises when interacting with Python's asyncio library is that the functions we use to get a reference to the Python event loop can only be called in certain contexts. Since PyO3 Asyncio needs to interact with Python's event loop during conversions, the context of these conversions can matter a lot. From 4399932c62d815441dd9a74b3fbb570ac907da2b Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sat, 7 Aug 2021 13:27:47 -0500 Subject: [PATCH 26/32] Reordered README a bit to include the Quickstart in the Primer, documented caveat for asyncio.run --- README.md | 84 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index ac65738..853991b 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,25 @@ This library can give spurious failures during finalization prior to PyO3 release `v0.13.2`. Make sure your PyO3 dependency is up-to-date! +## PyO3 Asyncio Primer + +If you are working with a Python library that makes use of async functions or wish to provide +Python bindings for an async Rust library, [`pyo3-asyncio`](https://github.com/awestlake87/pyo3-asyncio) +likely has the tools you need. It provides conversions between async functions in both Python and +Rust and was designed with first-class support for popular Rust runtimes such as +[`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). In addition, all async Python +code runs on the default `asyncio` event loop, so `pyo3-asyncio` should work just fine with existing +Python libraries. + +In the following sections, we'll give a general overview of `pyo3-asyncio` explaining how to call +async Python functions with PyO3, how to call async Rust functions from Python, and how to configure +your codebase to manage the runtimes of both. + ## Quickstart +Here are some examples to get you started right away! A more detailed breakdown +of the concepts in these examples can be found in the following sections. + ### Rust Applications Here we initialize the runtime, import Python's `asyncio` library and run the given future to completion using Python's default `EventLoop` and `async-std`. Inside the future, we convert `asyncio` sleep into a Rust future and await it. @@ -177,31 +194,17 @@ Python 3.8.5 (default, Jan 27 2021, 15:41:15) [GCC 9.3.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import asyncio +>>> >>> from my_async_module import rust_sleep >>> +>>> async def main(): +>>> await rust_sleep() +>>> >>> # should sleep for 1s ->>> asyncio.get_event_loop().run_until_complete(rust_sleep()) +>>> asyncio.run(main()) >>> ``` -> Note that we are using `EventLoop.run_until_complete` here instead of the newer `asyncio.run`. That is because `asyncio.run` will set up its own internal event loop that `pyo3_asyncio` will not be aware of. For this reason, running `pyo3_asyncio` conversions through `asyncio.run` is not currently supported. -> -> This restriction may be lifted in a future release. - -## PyO3 Asyncio Primer - -If you are working with a Python library that makes use of async functions or wish to provide -Python bindings for an async Rust library, [`pyo3-asyncio`](https://github.com/awestlake87/pyo3-asyncio) -likely has the tools you need. It provides conversions between async functions in both Python and -Rust and was designed with first-class support for popular Rust runtimes such as -[`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). In addition, all async Python -code runs on the default `asyncio` event loop, so `pyo3-asyncio` should work just fine with existing -Python libraries. - -In the following sections, we'll give a general overview of `pyo3-asyncio` explaining how to call -async Python functions with PyO3, how to call async Rust functions from Python, and how to configure -your codebase to manage the runtimes of both. - ## Awaiting an Async Python Function in Rust Let's take a look at a dead simple async Python function: @@ -361,6 +364,49 @@ async fn main() -> PyResult<()> { } ``` +### A Note About `asyncio.run` + +In Python 3.7+, the recommended way to run a top-level coroutine with `asyncio` +is with `asyncio.run`. In `v0.13` we recommended against using this function due to initialization issues, but in `v0.14` it's perfectly valid to use this function... with a caveat. + +Since our Rust <--> Python conversions require a reference to the Python event loop, this poses a problem. Imagine we have a PyO3 Asyncio module that defines +a `rust_sleep` function like in previous examples. You might rightfully assume that you can call pass this directly into `asyncio.run` like this: + +```python +import asyncio + +from my_async_module import rust_sleep + +asyncio.run(rust_sleep()) +``` + +You might be surprised to find out that this throws an error: +``` +Traceback (most recent call last): + File "", line 1, in +RuntimeError: no running event loop +``` + +What's happening here is that we are calling `rust_sleep` _before_ the future is +actually running on the event loop created by `asyncio.run`. This is counter-intuitive, but expected behaviour, and unfortunately there doesn't seem to be a good way of solving this problem within PyO3 Asyncio itself. + +However, we can make this example work with a simple workaround: + +```python +import asyncio + +from my_async_module import rust_sleep + +# Calling main will just construct the coroutine that later calls rust_sleep. +# - This ensures that rust_sleep will be called when the event loop is running, +# not before. +async def main(): + await rust_sleep() + +# Run the main() coroutine at the top-level instead +asyncio.run(main()) +``` + ### Non-standard Python Event Loops Python allows you to use alternatives to the default `asyncio` event loop. One From c79cbc3fd5a1975a862b9a8e35825aef347e7ed4 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sat, 7 Aug 2021 13:29:22 -0500 Subject: [PATCH 27/32] Forgot to add bash as language for traceback error to satisfy rustdoc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 853991b..e9da327 100644 --- a/README.md +++ b/README.md @@ -381,7 +381,7 @@ asyncio.run(rust_sleep()) ``` You might be surprised to find out that this throws an error: -``` +```bash Traceback (most recent call last): File "", line 1, in RuntimeError: no running event loop From 140db44a3a795aae929bce11118250e442e90e73 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sat, 7 Aug 2021 22:35:19 -0500 Subject: [PATCH 28/32] Added the migration guide to the README --- README.md | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e9da327..7b0ff43 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,7 @@ > PyO3 Asyncio is a _brand new_ part of the broader PyO3 ecosystem. Feel free to open any issues for feature requests or bugfixes for this crate. -## Known Problems - -This library can give spurious failures during finalization prior to PyO3 release `v0.13.2`. Make sure your PyO3 dependency is up-to-date! +__If you're a new-comer, the best way to get started is to read through the primer below! For `v0.13` users I highly recommend reading through the [migration section](#migrating-from-013-to-014) to get a general idea of what's changed in `v0.14`.__ ## PyO3 Asyncio Primer @@ -833,6 +831,103 @@ async fn main() -> pyo3::PyResult<()> { } ``` +## Migrating from 0.13 to 0.14 + +So what's changed from `v0.13` to `v0.14`? + +Well, a lot actually. There were some pretty major flaws in the initialization behaviour of `v0.13`. While it would have been nicer to address these issues without changing the public API, I decided it'd be better to break some of the old API rather than completely change the underlying behaviour of the existing functions. I realize this is going to be a bit of a headache, so hopefully this section will help you through it. + +To make things a bit easier, I decided to keep most of the old API alongside the new one (with some deprecation warnings to encourage users to move away from it). It should be possible to use the `v0.13` API alongside the newer `v0.14` API, which should allow you to upgrade your application piecemeal rather than all at once. + +__Before you get started, I personally recommend taking a look at [Event Loop References and Thread-awareness](#event-loop-references-and-thread-awareness) in order to get a better grasp on the motivation behind these changes and the nuance involved in using the new conversions.__ + +### 0.14 Highlights +- Tokio initialization is now lazy. + - No configuration necessary if you're using the multithreaded scheduler + - Calls to `pyo3_asyncio::tokio::init_multithread` or `pyo3_asyncio::tokio::init_multithread_once` can just be removed. + - Calls to `pyo3_asyncio::tokio::init_current_thread` or `pyo3_asyncio::tokio::init_current_thread_once` require some special attention. + - Custom runtime configuration is done by passing a `tokio::runtime::Builder` into `pyo3_asyncio::tokio::init` instead of a `tokio::runtime::Runtime` +- A new, more correct set of functions has been added to replace the `v0.13` conversions. + - `pyo3_asyncio::into_future_with_loop` + - `pyo3_asyncio::::future_into_py_with_loop` + - `pyo3_asyncio::::local_future_into_py_with_loop` + - `pyo3_asyncio::::into_future` + - `pyo3_asyncio::::future_into_py` + - `pyo3_asyncio::::local_future_into_py` + - `pyo3_asyncio::::get_current_loop` +- The `ThreadPoolExecutor` is no longer configured automatically at the start. + - Fortunately, this doesn't seem to have much effect on `v0.13` code, it just means that it's now possible to configure the executor manually as you see fit. + +### Upgrading Your Code to 0.14 + +1. Fix PyO3 0.14 initialization. + - PyO3 0.14 feature gated its automatic initialization behaviour behind "auto-initialize". You can either enable the "auto-initialize" behaviour in your project or add a call to `pyo3::prepare_freethreaded_python()` to the start of your program. + - If you're using the `#[pyo3_asyncio::::main]` proc macro attributes, then you can skip this step. `#[pyo3_asyncio::::main]` will call `pyo3::prepare_freethreaded_python()` at the start regardless of your project's "auto-initialize" feature. +2. Fix the tokio initialization. + - Calls to `pyo3_asyncio::tokio::init_multithread` or `pyo3_asyncio::tokio::init_multithread_once` can just be removed. + - If you're using the current thread scheduler, you'll need to manually spawn the thread that it runs on during initialization: + ```rust no_run + let mut builder = tokio::runtime::Builder::new_current_thread(); + builder.enable_all(); + + pyo3_asyncio::tokio::init(builder); + std::thread::spawn(move || { + pyo3_asyncio::tokio::get_runtime().block_on( + futures::future::pending::<()>() + ); + }); + ``` + - Custom `tokio::runtime::Builder` configs can be passed into `pyo3_asyncio::tokio::init`. The `tokio::runtime::Runtime` will be lazily instantiated on the first call to `pyo3_asyncio::tokio::get_runtime()` +3. If you're using `pyo3_asyncio::run_forever` in your application, you should switch to a more manual approach. + > `run_forever` is not the recommended way of running an event loop in Python, so it might be a good idea to move away from it. This function would have needed to change for `0.14`, but since it's considered an edge case, it was decided that users could just manually call it if they need to. + ```rust + use pyo3::prelude::*; + + fn main() -> PyResult<()> { + pyo3::prepare_freethreaded_python(); + + Python::with_gil(|py| { + let asyncio = py.import("asyncio")?; + + let event_loop = asyncio.call_method0("new_event_loop")?; + asyncio.call_method1("set_event_loop", (event_loop,))?; + + let event_loop_hdl = PyObject::from(event_loop); + + pyo3_asyncio::tokio::get_runtime().spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + + // Stop the event loop manually + Python::with_gil(|py| { + event_loop_hdl + .as_ref(py) + .call_method1( + "call_soon_threadsafe", + (event_loop_hdl + .as_ref(py) + .getattr("stop") + .unwrap(),), + ) + .unwrap(); + }) + }); + + event_loop.call_method0("run_forever")?; + Ok(()) + }) + } + ``` +4. Replace conversions with their newer counterparts. + > You may encounter some issues regarding the usage of `get_running_loop` vs `get_event_loop`. For more details on these newer conversions and how they should be used see [Event Loop References and Thread-awareness](#event-loop-references-and-thread-awareness). + - Replace `pyo3_asyncio::into_future` with `pyo3_asyncio::::into_future` + - Replace `pyo3_asyncio::::into_coroutine` with `pyo3_asyncio::::future_into_py` + - Replace `pyo3_asyncio::get_event_loop` with `pyo3_asyncio::::get_current_loop` +5. After all conversions have been replaced with their `v0.14` counterparts, `pyo3_asyncio::try_init` can safely be removed. + +## Known Problems + +This library can give spurious failures during finalization prior to PyO3 release `v0.13.2`. Make sure your PyO3 dependency is up-to-date! + ## MSRV Currently the MSRV for this library is 1.46.0, _but_ if you don't need to use the `async-std-runtime` feature, you can use rust 1.45.0. From dd6f2cc7838a2990ed15447ff9ef9f93fa51423d Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sat, 7 Aug 2021 22:43:19 -0500 Subject: [PATCH 29/32] Added a small note about supporting v0.13 API through v0.15 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7b0ff43..218ef21 100644 --- a/README.md +++ b/README.md @@ -924,6 +924,8 @@ __Before you get started, I personally recommend taking a look at [Event Loop Re - Replace `pyo3_asyncio::get_event_loop` with `pyo3_asyncio::::get_current_loop` 5. After all conversions have been replaced with their `v0.14` counterparts, `pyo3_asyncio::try_init` can safely be removed. +> The `v0.13` API will likely still be supported in version `v0.15`, but no solid guarantees after that point. + ## Known Problems This library can give spurious failures during finalization prior to PyO3 release `v0.13.2`. Make sure your PyO3 dependency is up-to-date! From 56eb606b3b142b1d48c9ef6d3cbd0e692fb07888 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sun, 8 Aug 2021 16:38:10 -0500 Subject: [PATCH 30/32] Restructured the docs a bit, integrated changes from PyO3 guide PR --- README.md | 361 ++++--------------------------------------------- src/lib.rs | 196 +++++++++++++++++++++++++++ src/testing.rs | 63 ++++----- 3 files changed, 245 insertions(+), 375 deletions(-) diff --git a/README.md b/README.md index 218ef21..1fe5056 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,12 @@ In the following sections, we'll give a general overview of `pyo3-asyncio` expla async Python functions with PyO3, how to call async Rust functions from Python, and how to configure your codebase to manage the runtimes of both. -## Quickstart +### Quickstart Here are some examples to get you started right away! A more detailed breakdown of the concepts in these examples can be found in the following sections. -### Rust Applications +#### Rust Applications Here we initialize the runtime, import Python's `asyncio` library and run the given future to completion using Python's default `EventLoop` and `async-std`. Inside the future, we convert `asyncio` sleep into a Rust future and await it. @@ -101,7 +101,7 @@ async fn main() -> PyResult<()> { More details on the usage of this library can be found in the [API docs](https://awestlake87.github.io/pyo3-asyncio/master/doc) and the primer below. -### PyO3 Native Rust Modules +#### PyO3 Native Rust Modules PyO3 Asyncio can also be used to write native modules with async functions. @@ -118,7 +118,7 @@ Make your project depend on `pyo3` with the `extension-module` feature enabled a For `async-std`: ```toml [dependencies] -pyo3 = { version = "0.13", features = ["extension-module"] } +pyo3 = { version = "0.14", features = ["extension-module"] } pyo3-asyncio = { version = "0.14", features = ["async-std-runtime"] } async-std = "1.9" ``` @@ -126,7 +126,7 @@ async-std = "1.9" For `tokio`: ```toml [dependencies] -pyo3 = { version = "0.13", features = ["extension-module"] } +pyo3 = { version = "0.14", features = ["extension-module"] } pyo3-asyncio = { version = "0.14", features = ["tokio-runtime"] } tokio = "1.4" ``` @@ -175,19 +175,15 @@ fn my_async_module(py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; Ok(()) } - ``` -Build your module and rename `libmy_async_module.so` to `my_async_module.so` -```bash -cargo build --release && mv target/release/libmy_async_module.so target/release/my_async_module.so -``` - -Now, point your `PYTHONPATH` to the directory containing `my_async_module.so`, then you'll be able -to import and use it: +You can build your module with maturin (see the [Using Rust in Python](https://pyo3.rs/main/#using-rust-from-python) section in the PyO3 guide for setup instructions). After that you should be able to run the Python REPL to try it out. ```bash -$ PYTHONPATH=target/release python3 +maturin develop && python3 +🔗 Found pyo3 bindings +🐍 Found CPython 3.8 at python3 + Finished dev [unoptimized + debuginfo] target(s) in 0.04s Python 3.8.5 (default, Jan 27 2021, 15:41:15) [GCC 9.3.0] on linux Type "help", "copyright", "credits" or "license" for more information. @@ -203,7 +199,7 @@ Type "help", "copyright", "credits" or "license" for more information. >>> ``` -## Awaiting an Async Python Function in Rust +### Awaiting an Async Python Function in Rust Let's take a look at a dead simple async Python function: @@ -253,7 +249,7 @@ async fn main() -> PyResult<()> { > If you're interested in learning more about `coroutines` and `awaitables` in general, check out the > [Python 3 `asyncio` docs](https://docs.python.org/3/library/asyncio-task.html) for more information. -## Awaiting a Rust Future in Python +### Awaiting a Rust Future in Python Here we have the same async function as before written in Rust using the [`async-std`](https://async.rs/) runtime: @@ -362,7 +358,7 @@ async fn main() -> PyResult<()> { } ``` -### A Note About `asyncio.run` +#### A Note About `asyncio.run` In Python 3.7+, the recommended way to run a top-level coroutine with `asyncio` is with `asyncio.run`. In `v0.13` we recommended against using this function due to initialization issues, but in `v0.14` it's perfectly valid to use this function... with a caveat. @@ -381,7 +377,8 @@ asyncio.run(rust_sleep()) You might be surprised to find out that this throws an error: ```bash Traceback (most recent call last): - File "", line 1, in + File "example.py", line 5, in + asyncio.run(rust_sleep()) RuntimeError: no running event loop ``` @@ -405,7 +402,7 @@ async def main(): asyncio.run(main()) ``` -### Non-standard Python Event Loops +#### Non-standard Python Event Loops Python allows you to use alternatives to the default `asyncio` event loop. One popular alternative is `uvloop`. In `v0.13` using non-standard event loops was @@ -421,7 +418,7 @@ name = "my_async_module" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.14", features = ["extension-module", "auto-initialize"] } +pyo3 = { version = "0.14", features = ["extension-module"] } pyo3-asyncio = { version = "0.14", features = ["tokio-runtime"] } async-std = "1.9" tokio = "1.4" @@ -449,10 +446,10 @@ fn my_async_module(_py: Python, m: &PyModule) -> PyResult<()> { ``` ```bash -$ cargo build --release && mv target/release/libmy_async_module.so my_async_module.so - Compiling pyo3-asyncio-lib v0.1.0 (pyo3-asyncio-lib) - Finished release [optimized] target(s) in 1.00s -$ PYTHONPATH=target/release/ python3 +$ maturin develop && python3 +🔗 Found pyo3 bindings +🐍 Found CPython 3.8 at python3 + Finished dev [unoptimized + debuginfo] target(s) in 0.04s Python 3.8.8 (default, Apr 13 2021, 19:58:26) [GCC 7.3.0] :: Anaconda, Inc. on linux Type "help", "copyright", "credits" or "license" for more information. @@ -475,7 +472,7 @@ Type "help", "copyright", "credits" or "license" for more information. Using `uvloop` in Rust applications is a bit trickier, but it's still possible with relatively few modifications. -> Unfortunately, we can't make use of the `#[pyo3_asyncio::::main]` attribute with non-standard event loops. This is because the `#[pyo3_asyncio::::main]` proc macro has to interact with the Python +Unfortunately, we can't make use of the `#[pyo3_asyncio::::main]` attribute with non-standard event loops. This is because the `#[pyo3_asyncio::::main]` proc macro has to interact with the Python event loop before we can install the `uvloop` policy. ```toml @@ -520,316 +517,9 @@ fn main() -> PyResult<()> { } ``` -### Event Loop References and Thread-awareness - -One problem that arises when interacting with Python's asyncio library is that the functions we use to get a reference to the Python event loop can only be called in certain contexts. Since PyO3 Asyncio needs to interact with Python's event loop during conversions, the context of these conversions can matter a lot. - -> The core conversions we've mentioned so far in this guide should insulate you from these concerns in most cases, but in the event that they don't, this section should provide you with the information you need to solve these problems. - -#### The Main Dilemma - -Python programs can have many independent event loop instances throughout the lifetime of the application (`asyncio.run` for example creates its own event loop each time it's called for instance), and they can even run concurrent with other event loops. For this reason, the most correct method of obtaining a reference to the Python event loop is via `asyncio.get_running_loop`. - -`asyncio.get_running_loop` returns the event loop associated with the current OS thread. It can be used inside Python coroutines to spawn concurrent tasks, interact with timers, or in our case signal between Rust and Python. This is all well and good when we are operating on a Python thread, but since Rust threads are not associated with a Python event loop, `asyncio.get_running_loop` will fail when called on a Rust runtime. - -#### The Solution - -A really straightforward way of dealing with this problem is to pass a reference to the associated Python event loop for every conversion. That's why in `v0.14`, we introduced a new set of conversion functions that do just that: - -- `pyo3_asyncio::into_future_with_loop` - Convert a Python awaitable into a Rust future with the given asyncio event loop. -- `pyo3_asyncio::::future_into_py_with_loop` - Convert a Rust future into a Python awaitable with the given asyncio event loop. -- `pyo3_asyncio::::local_future_into_py_with_loop` - Convert a `!Send` Rust future into a Python awaitable with the given asyncio event loop. - -One clear disadvantage to this approach (aside from the verbose naming) is that the Rust application has to explicitly track its references to the Python event loop. In native libraries, we can't make any assumptions about the underlying event loop, so the only reliable way to make sure our conversions work properly is to store a reference to the current event loop at the callsite to use later on. - -```rust -use pyo3::prelude::*; - -#[pyfunction] -fn sleep(py: Python) -> PyResult<&PyAny> { - let current_loop = pyo3_asyncio::get_running_loop(py)?; - let loop_ref = PyObject::from(current_loop); - - // Convert the async move { } block to a Python awaitable - pyo3_asyncio::tokio::future_into_py_with_loop(current_loop, async move { - let py_sleep = Python::with_gil(|py| { - // Sometimes we need to call other async Python functions within - // this future. In order for this to work, we need to track the - // event loop from earlier. - pyo3_asyncio::into_future_with_loop( - loop_ref.as_ref(py), - py.import("asyncio")?.call_method1("sleep", (1,))? - ) - })?; - - py_sleep.await?; - - Ok(Python::with_gil(|py| py.None())) - }) -} - -#[pymodule] -fn my_mod(py: Python, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(sleep, m)?)?; - Ok(()) -} -``` - -> A naive solution to this tracking problem would be to cache a global reference to the asyncio event loop that all PyO3 Asyncio conversions can use. In fact this is what we did in PyO3 Asyncio `v0.13`. This works well for applications, but it soon became clear that this is not so ideal for libraries. Libraries usually have no direct control over how the event loop is managed, they're just expected to work with any event loop at any point in the application. This problem is compounded further when multiple event loops are used in the application since the global reference will only point to one. - -Another disadvantage to this explicit approach that is less obvious is that we can no longer call our `#[pyfunction] fn sleep` on a Rust runtime since `asyncio.get_running_loop` only works on Python threads! It's clear that we need a slightly more flexible approach. - -In order to detect the Python event loop at the callsite, we need something like `asyncio.get_running_loop` that works for _both Python and Rust_. In Python, `asyncio.get_running_loop` uses thread-local data to retrieve the event loop associated with the current thread. What we need in Rust is something that can retrieve the Python event loop associated with the current _task_. - -Enter `pyo3_asyncio::::get_current_loop`. This function first checks task-local data for a Python event loop, then falls back on `asyncio.get_running_loop` if no task-local event loop is found. This way both bases are covered. - -Now, all we need is a way to store the event loop in task-local data. Since this is a runtime-specific feature, you can find the following functions in each runtime module: - -- `pyo3_asyncio::::scope` - Store the event loop in task-local data when executing the given Future. -- `pyo3_asyncio::::scope_local` - Store the event loop in task-local data when executing the given `!Send` Future. - -With these new functions, we can make our previous example more correct: - -```rust no_run -use pyo3::prelude::*; - -#[pyfunction] -fn sleep(py: Python) -> PyResult<&PyAny> { - // get the current event loop through task-local data - // OR `asyncio.get_running_loop` - let current_loop = pyo3_asyncio::tokio::get_current_loop(py)?; - - pyo3_asyncio::tokio::future_into_py_with_loop( - current_loop, - // Store the current loop in task-local data - pyo3_asyncio::tokio::scope(current_loop.into(), async move { - let py_sleep = Python::with_gil(|py| { - pyo3_asyncio::into_future_with_loop( - // Now we can get the current loop through task-local data - pyo3_asyncio::tokio::get_current_loop(py)?, - py.import("asyncio")?.call_method1("sleep", (1,))? - ) - })?; - - py_sleep.await?; - - Ok(Python::with_gil(|py| py.None())) - }) - ) -} - -#[pyfunction] -fn wrap_sleep(py: Python) -> PyResult<&PyAny> { - // get the current event loop through task-local data - // OR `asyncio.get_running_loop` - let current_loop = pyo3_asyncio::tokio::get_current_loop(py)?; - - pyo3_asyncio::tokio::future_into_py_with_loop( - current_loop, - // Store the current loop in task-local data - pyo3_asyncio::tokio::scope(current_loop.into(), async move { - let py_sleep = Python::with_gil(|py| { - pyo3_asyncio::into_future_with_loop( - pyo3_asyncio::tokio::get_current_loop(py)?, - // We can also call sleep within a Rust task since the - // event loop is stored in task local data - sleep(py)? - ) - })?; - - py_sleep.await?; - - Ok(Python::with_gil(|py| py.None())) - }) - ) -} - -#[pymodule] -fn my_mod(py: Python, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(sleep, m)?)?; - m.add_function(wrap_pyfunction!(wrap_sleep, m)?)?; - Ok(()) -} -``` - -Even though this is more correct, it's clearly not more ergonomic. That's why we introduced a new set of functions with this functionality baked in: - -- `pyo3_asyncio::::into_future` - > Convert a Python awaitable into a Rust future (using `pyo3_asyncio::::get_current_loop`) -- `pyo3_asyncio::::future_into_py` - > Convert a Rust future into a Python awaitable (using `pyo3_asyncio::::get_current_loop` and `pyo3_asyncio::::scope` to set the task-local event loop for the given Rust future) -- `pyo3_asyncio::::local_future_into_py` - > Convert a `!Send` Rust future into a Python awaitable (using `pyo3_asyncio::::get_current_loop` and `pyo3_asyncio::::scope_local` to set the task-local event loop for the given Rust future). - -__These are the functions that we recommend using__. With these functions, the previous example can be rewritten to be more compact: - -```rust -use pyo3::prelude::*; - -#[pyfunction] -fn sleep(py: Python) -> PyResult<&PyAny> { - pyo3_asyncio::tokio::future_into_py(py, async move { - let py_sleep = Python::with_gil(|py| { - pyo3_asyncio::tokio::into_future( - py.import("asyncio")?.call_method1("sleep", (1,))? - ) - })?; - - py_sleep.await?; - - Ok(Python::with_gil(|py| py.None())) - }) -} - -#[pyfunction] -fn wrap_sleep(py: Python) -> PyResult<&PyAny> { - pyo3_asyncio::tokio::future_into_py(py, async move { - let py_sleep = Python::with_gil(|py| { - pyo3_asyncio::tokio::into_future(sleep(py)?) - })?; - - py_sleep.await?; - - Ok(Python::with_gil(|py| py.None())) - }) -} - -#[pymodule] -fn my_mod(py: Python, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(sleep, m)?)?; - m.add_function(wrap_pyfunction!(wrap_sleep, m)?)?; - Ok(()) -} -``` - -### A Note for `v0.13` Users - -Hey guys, I realize that these are pretty major changes for `v0.14`, and I apologize in advance for having to modify the public API so much. I hope -the explanation above gives some much needed context and justification for all the breaking changes. - -Part of the reason why it's taken so long to push out a `v0.14` release is because I wanted to make sure we got this release right. There were a lot of issues with the `v0.13` release that I hadn't anticipated, and it's thanks to your feedback and patience that we've worked through these issues to get a more correct, more flexible version out there! - -This new release should address most the core issues that users have reported in the `v0.13` release, so I think we can expect more stability going forward. - -Also, a special thanks to [@ShadowJonathan](https://github.com/ShadowJonathan) for helping with the design and review -of these changes! - -- [@awestlake87](https://github.com/awestlake87) - -## PyO3 Asyncio in Cargo Tests - -The default Cargo Test harness does not currently allow test crates to provide their own `main` -function, so there doesn't seem to be a good way to allow Python to gain control over the main -thread. - -We can, however, override the default test harness and provide our own. `pyo3-asyncio` provides some -utilities to help us do just that! In the following sections, we will provide an overview for -constructing a Cargo integration test with `pyo3-asyncio` and adding your tests to it. - -### Main Test File -First, we need to create the test's main file. Although these tests are considered integration -tests, we cannot put them in the `tests` directory since that is a special directory owned by -Cargo. Instead, we put our tests in a `pytests` directory. - -> The name `pytests` is just a convention. You can name this folder anything you want in your own -> projects. - -We'll also want to provide the test's main function. Most of the functionality that the test harness needs is packed in the [`pyo3_asyncio::testing::main`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/testing/fn.main.html) function. This function will parse the test's CLI arguments, collect and pass the functions marked with [`#[pyo3_asyncio::async_std::test]`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/attr.test.html) or [`#[pyo3_asyncio::tokio::test]`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/tokio/attr.test.html) and pass them into the test harness for running and filtering. - -`pytests/test_example.rs` for the `tokio` runtime: -```rust -#[pyo3_asyncio::tokio::main] -async fn main() -> pyo3::PyResult<()> { - pyo3_asyncio::testing::main().await -} -``` - -`pytests/test_example.rs` for the `async-std` runtime: -```rust -#[pyo3_asyncio::async_std::main] -async fn main() -> pyo3::PyResult<()> { - pyo3_asyncio::testing::main().await -} -``` - -### Cargo Configuration -Next, we need to add our test file to the Cargo manifest by adding the following section to the -`Cargo.toml` - -```toml -[[test]] -name = "test_example" -path = "pytests/test_example.rs" -harness = false -``` - -Also add the `testing` and `attributes` features to the `pyo3-asyncio` dependency and select your preferred runtime: - -```toml -pyo3-asyncio = { version = "0.13", features = ["testing", "attributes", "async-std-runtime"] } -``` - -At this point, you should be able to run the test via `cargo test` - -### Adding Tests to the PyO3 Asyncio Test Harness - -We can add tests anywhere in the test crate with the runtime's corresponding `#[test]` attribute: - -For `async-std` use the [`pyo3_asyncio::async_std::test`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/attr.test.html) attribute: -```rust -mod tests { - use std::{time::Duration, thread}; - - use pyo3::prelude::*; - - // tests can be async - #[pyo3_asyncio::async_std::test] - async fn test_async_sleep() -> PyResult<()> { - async_std::task::sleep(Duration::from_secs(1)).await; - Ok(()) - } - - // they can also be synchronous - #[pyo3_asyncio::async_std::test] - fn test_blocking_sleep() -> PyResult<()> { - thread::sleep(Duration::from_secs(1)); - Ok(()) - } -} - -#[pyo3_asyncio::async_std::main] -async fn main() -> pyo3::PyResult<()> { - pyo3_asyncio::testing::main().await -} -``` - -For `tokio` use the [`pyo3_asyncio::tokio::test`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/tokio/attr.test.html) attribute: -```rust -mod tests { - use std::{time::Duration, thread}; - - use pyo3::prelude::*; - - // tests can be async - #[pyo3_asyncio::tokio::test] - async fn test_async_sleep() -> PyResult<()> { - tokio::time::sleep(Duration::from_secs(1)).await; - Ok(()) - } - - // they can also be synchronous - #[pyo3_asyncio::tokio::test] - fn test_blocking_sleep() -> PyResult<()> { - thread::sleep(Duration::from_secs(1)); - Ok(()) - } -} - -#[pyo3_asyncio::tokio::main] -async fn main() -> pyo3::PyResult<()> { - pyo3_asyncio::testing::main().await -} -``` +### Additional Information +- Managing event loop references can be tricky with pyo3-asyncio. See [Event Loop References](https://docs.rs/pyo3-asyncio/#event-loop-references) in the API docs to get a better intuition for how event loop references are managed in this library. +- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-asyncio/testing) ## Migrating from 0.13 to 0.14 @@ -855,6 +545,7 @@ __Before you get started, I personally recommend taking a look at [Event Loop Re - `pyo3_asyncio::::future_into_py` - `pyo3_asyncio::::local_future_into_py` - `pyo3_asyncio::::get_current_loop` +- `pyo3_asyncio::try_init` is no longer required if you're only using `0.14` conversions - The `ThreadPoolExecutor` is no longer configured automatically at the start. - Fortunately, this doesn't seem to have much effect on `v0.13` code, it just means that it's now possible to configure the executor manually as you see fit. diff --git a/src/lib.rs b/src/lib.rs index 0b32c7d..4df2840 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,202 @@ //! means that Cargo's default test harness will no longer work since it doesn't provide a method of //! overriding the main function to add our event loop initialization and finalization. //! +//! ## Event Loop References +//! +//! One problem that arises when interacting with Python's asyncio library is that the functions we use to get a reference to the Python event loop can only be called in certain contexts. Since PyO3 Asyncio needs to interact with Python's event loop during conversions, the context of these conversions can matter a lot. +//! +//! > The core conversions we've mentioned so far in this guide should insulate you from these concerns in most cases, but in the event that they don't, this section should provide you with the information you need to solve these problems. +//! +//! ### The Main Dilemma +//! +//! Python programs can have many independent event loop instances throughout the lifetime of the application (`asyncio.run` for example creates its own event loop each time it's called for instance), and they can even run concurrent with other event loops. For this reason, the most correct method of obtaining a reference to the Python event loop is via `asyncio.get_running_loop`. +//! +//! `asyncio.get_running_loop` returns the event loop associated with the current OS thread. It can be used inside Python coroutines to spawn concurrent tasks, interact with timers, or in our case signal between Rust and Python. This is all well and good when we are operating on a Python thread, but since Rust threads are not associated with a Python event loop, `asyncio.get_running_loop` will fail when called on a Rust runtime. +//! +//! ### The Solution +//! +//! A really straightforward way of dealing with this problem is to pass a reference to the associated Python event loop for every conversion. That's why in `v0.14`, we introduced a new set of conversion functions that do just that: +//! +//! - `pyo3_asyncio::into_future_with_loop` - Convert a Python awaitable into a Rust future with the given asyncio event loop. +//! - `pyo3_asyncio::::future_into_py_with_loop` - Convert a Rust future into a Python awaitable with the given asyncio event loop. +//! - `pyo3_asyncio::::local_future_into_py_with_loop` - Convert a `!Send` Rust future into a Python awaitable with the given asyncio event loop. +//! +//! One clear disadvantage to this approach (aside from the verbose naming) is that the Rust application has to explicitly track its references to the Python event loop. In native libraries, we can't make any assumptions about the underlying event loop, so the only reliable way to make sure our conversions work properly is to store a reference to the current event loop at the callsite to use later on. +//! +//! ```rust +//! use pyo3::prelude::*; +//! +//! #[pyfunction] +//! fn sleep(py: Python) -> PyResult<&PyAny> { +//! let current_loop = pyo3_asyncio::get_running_loop(py)?; +//! let loop_ref = PyObject::from(current_loop); +//! +//! // Convert the async move { } block to a Python awaitable +//! pyo3_asyncio::tokio::future_into_py_with_loop(current_loop, async move { +//! let py_sleep = Python::with_gil(|py| { +//! // Sometimes we need to call other async Python functions within +//! // this future. In order for this to work, we need to track the +//! // event loop from earlier. +//! pyo3_asyncio::into_future_with_loop( +//! loop_ref.as_ref(py), +//! py.import("asyncio")?.call_method1("sleep", (1,))? +//! ) +//! })?; +//! +//! py_sleep.await?; +//! +//! Ok(Python::with_gil(|py| py.None())) +//! }) +//! } +//! +//! #[pymodule] +//! fn my_mod(py: Python, m: &PyModule) -> PyResult<()> { +//! m.add_function(wrap_pyfunction!(sleep, m)?)?; +//! Ok(()) +//! } +//! ``` +//! +//! > A naive solution to this tracking problem would be to cache a global reference to the asyncio event loop that all PyO3 Asyncio conversions can use. In fact this is what we did in PyO3 Asyncio `v0.13`. This works well for applications, but it soon became clear that this is not so ideal for libraries. Libraries usually have no direct control over how the event loop is managed, they're just expected to work with any event loop at any point in the application. This problem is compounded further when multiple event loops are used in the application since the global reference will only point to one. +//! +//! Another disadvantage to this explicit approach that is less obvious is that we can no longer call our `#[pyfunction] fn sleep` on a Rust runtime since `asyncio.get_running_loop` only works on Python threads! It's clear that we need a slightly more flexible approach. +//! +//! In order to detect the Python event loop at the callsite, we need something like `asyncio.get_running_loop` that works for _both Python and Rust_. In Python, `asyncio.get_running_loop` uses thread-local data to retrieve the event loop associated with the current thread. What we need in Rust is something that can retrieve the Python event loop associated with the current _task_. +//! +//! Enter `pyo3_asyncio::::get_current_loop`. This function first checks task-local data for a Python event loop, then falls back on `asyncio.get_running_loop` if no task-local event loop is found. This way both bases are covered. +//! +//! Now, all we need is a way to store the event loop in task-local data. Since this is a runtime-specific feature, you can find the following functions in each runtime module: +//! +//! - `pyo3_asyncio::::scope` - Store the event loop in task-local data when executing the given Future. +//! - `pyo3_asyncio::::scope_local` - Store the event loop in task-local data when executing the given `!Send` Future. +//! +//! With these new functions, we can make our previous example more correct: +//! +//! ```rust no_run +//! use pyo3::prelude::*; +//! +//! #[pyfunction] +//! fn sleep(py: Python) -> PyResult<&PyAny> { +//! // get the current event loop through task-local data +//! // OR `asyncio.get_running_loop` +//! let current_loop = pyo3_asyncio::tokio::get_current_loop(py)?; +//! +//! pyo3_asyncio::tokio::future_into_py_with_loop( +//! current_loop, +//! // Store the current loop in task-local data +//! pyo3_asyncio::tokio::scope(current_loop.into(), async move { +//! let py_sleep = Python::with_gil(|py| { +//! pyo3_asyncio::into_future_with_loop( +//! // Now we can get the current loop through task-local data +//! pyo3_asyncio::tokio::get_current_loop(py)?, +//! py.import("asyncio")?.call_method1("sleep", (1,))? +//! ) +//! })?; +//! +//! py_sleep.await?; +//! +//! Ok(Python::with_gil(|py| py.None())) +//! }) +//! ) +//! } +//! +//! #[pyfunction] +//! fn wrap_sleep(py: Python) -> PyResult<&PyAny> { +//! // get the current event loop through task-local data +//! // OR `asyncio.get_running_loop` +//! let current_loop = pyo3_asyncio::tokio::get_current_loop(py)?; +//! +//! pyo3_asyncio::tokio::future_into_py_with_loop( +//! current_loop, +//! // Store the current loop in task-local data +//! pyo3_asyncio::tokio::scope(current_loop.into(), async move { +//! let py_sleep = Python::with_gil(|py| { +//! pyo3_asyncio::into_future_with_loop( +//! pyo3_asyncio::tokio::get_current_loop(py)?, +//! // We can also call sleep within a Rust task since the +//! // event loop is stored in task local data +//! sleep(py)? +//! ) +//! })?; +//! +//! py_sleep.await?; +//! +//! Ok(Python::with_gil(|py| py.None())) +//! }) +//! ) +//! } +//! +//! #[pymodule] +//! fn my_mod(py: Python, m: &PyModule) -> PyResult<()> { +//! m.add_function(wrap_pyfunction!(sleep, m)?)?; +//! m.add_function(wrap_pyfunction!(wrap_sleep, m)?)?; +//! Ok(()) +//! } +//! ``` +//! +//! Even though this is more correct, it's clearly not more ergonomic. That's why we introduced a new set of functions with this functionality baked in: +//! +//! - `pyo3_asyncio::::into_future` +//! > Convert a Python awaitable into a Rust future (using `pyo3_asyncio::::get_current_loop`) +//! - `pyo3_asyncio::::future_into_py` +//! > Convert a Rust future into a Python awaitable (using `pyo3_asyncio::::get_current_loop` and `pyo3_asyncio::::scope` to set the task-local event loop for the given Rust future) +//! - `pyo3_asyncio::::local_future_into_py` +//! > Convert a `!Send` Rust future into a Python awaitable (using `pyo3_asyncio::::get_current_loop` and `pyo3_asyncio::::scope_local` to set the task-local event loop for the given Rust future). +//! +//! __These are the functions that we recommend using__. With these functions, the previous example can be rewritten to be more compact: +//! +//! ```rust +//! use pyo3::prelude::*; +//! +//! #[pyfunction] +//! fn sleep(py: Python) -> PyResult<&PyAny> { +//! pyo3_asyncio::tokio::future_into_py(py, async move { +//! let py_sleep = Python::with_gil(|py| { +//! pyo3_asyncio::tokio::into_future( +//! py.import("asyncio")?.call_method1("sleep", (1,))? +//! ) +//! })?; +//! +//! py_sleep.await?; +//! +//! Ok(Python::with_gil(|py| py.None())) +//! }) +//! } +//! +//! #[pyfunction] +//! fn wrap_sleep(py: Python) -> PyResult<&PyAny> { +//! pyo3_asyncio::tokio::future_into_py(py, async move { +//! let py_sleep = Python::with_gil(|py| { +//! pyo3_asyncio::tokio::into_future(sleep(py)?) +//! })?; +//! +//! py_sleep.await?; +//! +//! Ok(Python::with_gil(|py| py.None())) +//! }) +//! } +//! +//! #[pymodule] +//! fn my_mod(py: Python, m: &PyModule) -> PyResult<()> { +//! m.add_function(wrap_pyfunction!(sleep, m)?)?; +//! m.add_function(wrap_pyfunction!(wrap_sleep, m)?)?; +//! Ok(()) +//! } +//! ``` +//! +//! ## A Note for `v0.13` Users +//! +//! Hey guys, I realize that these are pretty major changes for `v0.14`, and I apologize in advance for having to modify the public API so much. I hope +//! the explanation above gives some much needed context and justification for all the breaking changes. +//! +//! Part of the reason why it's taken so long to push out a `v0.14` release is because I wanted to make sure we got this release right. There were a lot of issues with the `v0.13` release that I hadn't anticipated, and it's thanks to your feedback and patience that we've worked through these issues to get a more correct, more flexible version out there! +//! +//! This new release should address most the core issues that users have reported in the `v0.13` release, so I think we can expect more stability going forward. +//! +//! Also, a special thanks to [@ShadowJonathan](https://github.com/ShadowJonathan) for helping with the design and review +//! of these changes! +//! +//! - [@awestlake87](https://github.com/awestlake87) +//! //! ## Rust's Event Loop //! //! Currently only the async-std and Tokio runtimes are supported by this crate. diff --git a/src/testing.rs b/src/testing.rs index 9e8bd0e..84cc3fc 100644 --- a/src/testing.rs +++ b/src/testing.rs @@ -18,45 +18,34 @@ //! overriding the default test harness can be quite different from what you're used to doing for //! integration tests, so these next sections will walk you through this process. //! -//! ### Main Test File +//! ## Main Test File //! First, we need to create the test's main file. Although these tests are considered integration //! tests, we cannot put them in the `tests` directory since that is a special directory owned by -//! Cargo. Instead, we put our tests in a `pytests` directory, although the name `pytests` is just -//! a convention. +//! Cargo. Instead, we put our tests in a `pytests` directory. //! -//! We'll also want to provide the test's main function. Most of the functionality that the test -//! harness needs is packed in this module's [`main`](crate::testing::main) function. It will parse -//! the test's CLI arguments, collect and pass the functions marked with -//! [`#[pyo3_asyncio::async_std::test]`](crate::async_std::test) or -//! [`#[pyo3_asyncio::tokio::test]`](crate::tokio::test) and pass them into the test harness for -//! running and filtering. +//! > The name `pytests` is just a convention. You can name this folder anything you want in your own +//! > projects. +//! +//! We'll also want to provide the test's main function. Most of the functionality that the test harness needs is packed in the [`pyo3_asyncio::testing::main`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/testing/fn.main.html) function. This function will parse the test's CLI arguments, collect and pass the functions marked with [`#[pyo3_asyncio::async_std::test]`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/attr.test.html) or [`#[pyo3_asyncio::tokio::test]`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/tokio/attr.test.html) and pass them into the test harness for running and filtering. //! //! `pytests/test_example.rs` for the `tokio` runtime: -//! ``` -//! # #[cfg(all(feature = "tokio-runtime", feature = "attributes"))] +//! ```rust //! #[pyo3_asyncio::tokio::main] //! async fn main() -> pyo3::PyResult<()> { //! pyo3_asyncio::testing::main().await //! } -//! # -//! # #[cfg(not(all(feature = "tokio-runtime", feature = "attributes")))] -//! # fn main() {} //! ``` //! //! `pytests/test_example.rs` for the `async-std` runtime: -//! ``` -//! # #[cfg(all(feature = "async-std-runtime", feature = "attributes"))] +//! ```rust //! #[pyo3_asyncio::async_std::main] //! async fn main() -> pyo3::PyResult<()> { //! pyo3_asyncio::testing::main().await //! } -//! # -//! # #[cfg(not(all(feature = "async-std-runtime", feature = "attributes")))] -//! # fn main() {} //! ``` //! -//! ### Cargo Configuration -//! Next, we need to add our test file to the Cargo manifest. Add the following section to your +//! ## Cargo Configuration +//! Next, we need to add our test file to the Cargo manifest by adding the following section to the //! `Cargo.toml` //! //! ```toml @@ -66,34 +55,33 @@ //! harness = false //! ``` //! -//! Also, add the `testing` and `attributes` features to `pyo3-asyncio` and select your preferred -//! runtime: +//! Also add the `testing` and `attributes` features to the `pyo3-asyncio` dependency and select your preferred runtime: //! //! ```toml -//! [dependencies] //! pyo3-asyncio = { version = "0.13", features = ["testing", "attributes", "async-std-runtime"] } //! ``` //! -//! At this point you should be able to run the test via `cargo test` +//! At this point, you should be able to run the test via `cargo test` //! //! ### Adding Tests to the PyO3 Asyncio Test Harness //! //! We can add tests anywhere in the test crate with the runtime's corresponding `#[test]` attribute: //! -//! For `async-std` use the [`pyo3_asyncio::async_std::test`](crate::async_std::test) attribute: -//! ``` -//! # #[cfg(all(feature = "async-std-runtime", feature = "attributes"))] +//! For `async-std` use the [`pyo3_asyncio::async_std::test`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/attr.test.html) attribute: +//! ```rust //! mod tests { //! use std::{time::Duration, thread}; //! //! use pyo3::prelude::*; //! +//! // tests can be async //! #[pyo3_asyncio::async_std::test] //! async fn test_async_sleep() -> PyResult<()> { //! async_std::task::sleep(Duration::from_secs(1)).await; //! Ok(()) //! } //! +//! // they can also be synchronous //! #[pyo3_asyncio::async_std::test] //! fn test_blocking_sleep() -> PyResult<()> { //! thread::sleep(Duration::from_secs(1)); @@ -101,29 +89,27 @@ //! } //! } //! -//! # #[cfg(all(feature = "async-std-runtime", feature = "attributes"))] //! #[pyo3_asyncio::async_std::main] //! async fn main() -> pyo3::PyResult<()> { //! pyo3_asyncio::testing::main().await //! } -//! # #[cfg(not(all(feature = "async-std-runtime", feature = "attributes")))] -//! # fn main() {} //! ``` //! -//! For `tokio` use the [`pyo3_asyncio::tokio::test`](crate::tokio::test) attribute: -//! ``` -//! # #[cfg(all(feature = "tokio-runtime", feature = "attributes"))] +//! For `tokio` use the [`pyo3_asyncio::tokio::test`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/tokio/attr.test.html) attribute: +//! ```rust //! mod tests { //! use std::{time::Duration, thread}; //! //! use pyo3::prelude::*; //! +//! // tests can be async //! #[pyo3_asyncio::tokio::test] //! async fn test_async_sleep() -> PyResult<()> { //! tokio::time::sleep(Duration::from_secs(1)).await; //! Ok(()) //! } //! +//! // they can also be synchronous //! #[pyo3_asyncio::tokio::test] //! fn test_blocking_sleep() -> PyResult<()> { //! thread::sleep(Duration::from_secs(1)); @@ -131,21 +117,18 @@ //! } //! } //! -//! # #[cfg(all(feature = "tokio-runtime", feature = "attributes"))] //! #[pyo3_asyncio::tokio::main] //! async fn main() -> pyo3::PyResult<()> { //! pyo3_asyncio::testing::main().await //! } -//! # #[cfg(not(all(feature = "tokio-runtime", feature = "attributes")))] -//! # fn main() {} //! ``` //! -//! ### Lib Tests +//! ## Lib Tests //! //! Unfortunately, as we mentioned at the beginning, these utilities will only run in integration //! tests and doc tests. Running lib tests are out of the question since we need control over the -//! main function. You can however perform compilation checks for lib tests. This is unfortunately -//! much more useful in doc tests than it is for lib tests, but the option is there if you want it. +//! main function. You can however perform compilation checks for lib tests. This is much more +//! useful in doc tests than it is for lib tests, but the option is there if you want it. //! //! `my-crate/src/lib.rs` //! ``` From 9244cc42b2f56f3076dfa2df527b25834b432c87 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sun, 8 Aug 2021 16:39:56 -0500 Subject: [PATCH 31/32] Changed Additional Information links to point to gh-pages docs instead of stable --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1fe5056..7a75ea8 100644 --- a/README.md +++ b/README.md @@ -518,8 +518,8 @@ fn main() -> PyResult<()> { ``` ### Additional Information -- Managing event loop references can be tricky with pyo3-asyncio. See [Event Loop References](https://docs.rs/pyo3-asyncio/#event-loop-references) in the API docs to get a better intuition for how event loop references are managed in this library. -- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-asyncio/testing) +- Managing event loop references can be tricky with pyo3-asyncio. See [Event Loop References](https://awestlake87.github.io/pyo3-asyncio/master/doc/#event-loop-references) in the API docs to get a better intuition for how event loop references are managed in this library. +- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://awestlake87.github.io/pyo3-asyncio/master/doc/testing) ## Migrating from 0.13 to 0.14 From f167f308a8aba4e396fb8777a9d2a89dae85a0a7 Mon Sep 17 00:00:00 2001 From: Andrew J Westlake Date: Sun, 8 Aug 2021 16:59:23 -0500 Subject: [PATCH 32/32] Added links to the migration guide on every deprecation note --- README.md | 4 ++-- src/async_std.rs | 2 +- src/generic.rs | 2 +- src/lib.rs | 24 ++++++++++++++++++------ src/tokio.rs | 2 +- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7a75ea8..afca8a3 100644 --- a/README.md +++ b/README.md @@ -518,8 +518,8 @@ fn main() -> PyResult<()> { ``` ### Additional Information -- Managing event loop references can be tricky with pyo3-asyncio. See [Event Loop References](https://awestlake87.github.io/pyo3-asyncio/master/doc/#event-loop-references) in the API docs to get a better intuition for how event loop references are managed in this library. -- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://awestlake87.github.io/pyo3-asyncio/master/doc/testing) +- Managing event loop references can be tricky with pyo3-asyncio. See [Event Loop References](https://awestlake87.github.io/pyo3-asyncio/master/doc/pyo3_asyncio/#event-loop-references) in the API docs to get a better intuition for how event loop references are managed in this library. +- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://awestlake87.github.io/pyo3-asyncio/master/doc/pyo3_asyncio/testing) ## Migrating from 0.13 to 0.14 diff --git a/src/async_std.rs b/src/async_std.rs index b5f0835..0612047 100644 --- a/src/async_std.rs +++ b/src/async_std.rs @@ -230,7 +230,7 @@ where /// ``` #[deprecated( since = "0.14.0", - note = "Use the pyo3_asyncio::async_std::future_into_py instead" + note = "Use the pyo3_asyncio::async_std::future_into_py instead\n\t\t(see the [migration guide](https://github.com/awestlake87/pyo3-asyncio/#migrating-from-013-to-014) for more details)" )] #[allow(deprecated)] pub fn into_coroutine(py: Python, fut: F) -> PyResult diff --git a/src/generic.rs b/src/generic.rs index 4933c8b..6f15537 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -660,7 +660,7 @@ where /// ``` #[deprecated( since = "0.14.0", - note = "Use the pyo3_asyncio::generic::future_into_py instead" + note = "Use the pyo3_asyncio::generic::future_into_py instead\n\t\t(see the [migration guide](https://github.com/awestlake87/pyo3-asyncio/#migrating-from-013-to-014) for more details)" )] #[allow(deprecated)] pub fn into_coroutine(py: Python, fut: F) -> PyResult diff --git a/src/lib.rs b/src/lib.rs index 4df2840..c283191 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -391,7 +391,7 @@ fn create_future(event_loop: &PyAny) -> PyResult<&PyAny> { /// ``` #[deprecated( since = "0.14.0", - note = "Use the pyo3_asyncio::async_std::run or pyo3_asyncio::tokio::run instead" + note = "Use the pyo3_asyncio::async_std::run or pyo3_asyncio::tokio::run instead\n\t\t(see the [migration guide](https://github.com/awestlake87/pyo3-asyncio/#migrating-from-013-to-014) for more details)" )] #[allow(deprecated)] pub fn with_runtime(py: Python, f: F) -> PyResult @@ -431,7 +431,10 @@ fn close(event_loop: &PyAny) -> PyResult<()> { /// - Must be called before any other pyo3-asyncio functions. /// - Calling `try_init` a second time returns `Ok(())` and does nothing. /// > In future versions this may return an `Err`. -#[deprecated(since = "0.14.0")] +#[deprecated( + since = "0.14.0", + note = "see the [migration guide](https://github.com/awestlake87/pyo3-asyncio/#migrating-from-013-to-014) for more details" +)] pub fn try_init(py: Python) -> PyResult<()> { CACHED_EVENT_LOOP.get_or_try_init(|| -> PyResult { let event_loop = asyncio_get_event_loop(py)?; @@ -483,7 +486,10 @@ pub fn get_running_loop(py: Python) -> PyResult<&PyAny> { } /// Get a reference to the Python event loop cached by `try_init` (0.13 behaviour) -#[deprecated(since = "0.14.0")] +#[deprecated( + since = "0.14.0", + note = "see the [migration guide](https://github.com/awestlake87/pyo3-asyncio/#migrating-from-013-to-014) for more details" +)] pub fn get_event_loop(py: Python) -> &PyAny { CACHED_EVENT_LOOP.get().expect(EXPECT_INIT).as_ref(py) } @@ -539,7 +545,10 @@ pub fn get_event_loop(py: Python) -> &PyAny { /// } /// # #[cfg(not(feature = "async-std-runtime"))] /// # fn main() {} -#[deprecated(since = "0.14.0")] +#[deprecated( + since = "0.14.0", + note = "see the [migration guide](https://github.com/awestlake87/pyo3-asyncio/#migrating-from-013-to-014) for more details" +)] #[allow(deprecated)] pub fn run_forever(py: Python) -> PyResult<()> { if let Err(e) = get_event_loop(py).call_method0("run_forever") { @@ -554,7 +563,10 @@ pub fn run_forever(py: Python) -> PyResult<()> { } /// Shutdown the event loops and perform any necessary cleanup -#[deprecated(since = "0.14.0")] +#[deprecated( + since = "0.14.0", + note = "see the [migration guide](https://github.com/awestlake87/pyo3-asyncio/#migrating-from-013-to-014) for more details" +)] pub fn try_close(py: Python) -> PyResult<()> { if let Some(exec) = EXECUTOR.get() { // Shutdown the executor and wait until all threads are cleaned up @@ -751,7 +763,7 @@ pub fn into_future_with_loop( /// ``` #[deprecated( since = "0.14.0", - note = "Use pyo3_asyncio::async_std::into_future or pyo3_asyncio::tokio::into_future" + note = "Use pyo3_asyncio::async_std::into_future or pyo3_asyncio::tokio::into_future instead\n (see the [migration guide](https://github.com/awestlake87/pyo3-asyncio/#migrating-from-013-to-014) for more details)" )] #[allow(deprecated)] pub fn into_future(awaitable: &PyAny) -> PyResult> + Send> { diff --git a/src/tokio.rs b/src/tokio.rs index 47a648d..af92e2f 100644 --- a/src/tokio.rs +++ b/src/tokio.rs @@ -243,7 +243,7 @@ where /// ``` #[deprecated( since = "0.14.0", - note = "Use the pyo3_asyncio::tokio::future_into_py instead" + note = "Use pyo3_asyncio::tokio::future_into_py instead\n (see the [migration guide](https://github.com/awestlake87/pyo3-asyncio/#migrating-from-013-to-014) for more details)" )] #[allow(deprecated)] pub fn into_coroutine(py: Python, fut: F) -> PyResult