Update PyO3 to 0.21 (#147)

* upgrade pyo3

* pyo3: transition away from gil-refs api

* windows bits

mitmproxy-rs/Cargo.toml

@@ -18,9 +18,9 @@ anyhow = { version = "1.0.86", features = ["backtrace"] }
 data-encoding = "2.6.0"
 log = "0.4.18"
 once_cell = "1"
-pyo3 = { version = "0.20.3", features = ["abi3", "abi3-py310", "extension-module", "anyhow"] }
-pyo3-asyncio = { version = "0.20", features = ["tokio-runtime"] }
-pyo3-log = "0.9.0"
+pyo3 = { version = "0.21", features = ["abi3", "abi3-py310", "extension-module", "anyhow"] }
+pyo3-asyncio-0-21 = { version = "0.21", features = ["tokio-runtime"] }
+pyo3-log = "0.10.0"
 rand_core = { version = "0.6.4", features = ["getrandom"] }
 tokio = { version = "1.38", features = ["macros", "net", "rt-multi-thread", "sync"] }
 boringtun = "0.6"

mitmproxy-rs/src/lib.rs

@@ -32,7 +32,7 @@ fn init_logger() -> PyResult<()> {
 
 #[allow(unused)]
 #[pymodule]
-pub fn mitmproxy_rs(py: Python, m: &PyModule) -> PyResult<()> {
+pub fn mitmproxy_rs(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     // set up the Rust logger to send messages to the Python logger
     init_logger()?;
 
@@ -63,9 +63,9 @@ pub fn mitmproxy_rs(py: Python, m: &PyModule) -> PyResult<()> {
 
     // Import platform-specific modules here so that missing dependencies are raising immediately.
     #[cfg(target_os = "macos")]
-    py.import("mitmproxy_macos")?;
+    py.import_bound("mitmproxy_macos")?;
     #[cfg(windows)]
-    py.import("mitmproxy_windows")?;
+    py.import_bound("mitmproxy_windows")?;
 
     Ok(())
 }

mitmproxy-rs/src/process_info.rs

@@ -66,7 +66,7 @@ pub fn executable_icon(path: PathBuf) -> Result<PyObject> {
         let mut icon_cache = windows::icons::ICON_CACHE.lock().unwrap();
         let png_bytes = icon_cache.get_png(path)?;
         Ok(Python::with_gil(|py| {
-            pyo3::types::PyBytes::new(py, png_bytes).to_object(py)
+            pyo3::types::PyBytes::new_bound(py, png_bytes).to_object(py)
         }))
     }
     #[cfg(not(windows))]

mitmproxy-rs/src/server/base.rs

@@ -23,9 +23,9 @@ impl Server {
         }
     }
 
-    pub fn wait_closed<'p>(&self, py: Python<'p>) -> PyResult<&'p PyAny> {
+    pub fn wait_closed<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
         let mut receiver = self.shutdown_done.resubscribe();
-        pyo3_asyncio::tokio::future_into_py(py, async move {
+        pyo3_asyncio_0_21::tokio::future_into_py(py, async move {
             receiver.recv().await.ok();
             Ok(())
         })

mitmproxy-rs/src/server/local_redirector.rs

@@ -58,7 +58,7 @@ impl LocalRedirector {
         self.server.close()
     }
 
-    pub fn wait_closed<'p>(&self, py: Python<'p>) -> PyResult<&'p PyAny> {
+    pub fn wait_closed<'p>(&self, py: Python<'p>) -> PyResult<Bound<'p, PyAny>> {
         self.server.wait_closed(py)
     }
 
@@ -79,18 +79,18 @@ pub fn start_local_redirector(
     py: Python<'_>,
     handle_tcp_stream: PyObject,
     handle_udp_stream: PyObject,
-) -> PyResult<&PyAny> {
+) -> PyResult<Bound<PyAny>> {
     #[cfg(windows)]
     {
         let executable_path: PathBuf = py
-            .import("mitmproxy_windows")?
+            .import_bound("mitmproxy_windows")?
             .call_method0("executable_path")?
             .extract()?;
         if !executable_path.exists() {
             return Err(anyhow::anyhow!("{} does not exist", executable_path.display()).into());
         }
         let conf = WindowsConf { executable_path };
-        pyo3_asyncio::tokio::future_into_py(py, async move {
+        pyo3_asyncio_0_21::tokio::future_into_py(py, async move {
             let (server, conf_tx) =
                 Server::init(conf, handle_tcp_stream, handle_udp_stream).await?;
 
@@ -103,9 +103,9 @@ pub fn start_local_redirector(
         if destination_path.exists() {
             log::info!("Using existing mitmproxy redirector app.");
         } else {
-            let filename = py.import("mitmproxy_macos")?.filename()?;
+            let filename = py.import_bound("mitmproxy_macos")?.filename()?;
 
-            let source_path = Path::new(filename)
+            let source_path = Path::new(filename.to_str()?)
                 .parent()
                 .ok_or_else(|| anyhow::anyhow!("invalid path"))?
                 .join("Mitmproxy Redirector.app.tar");
@@ -120,7 +120,7 @@ pub fn start_local_redirector(
             archive.unpack(destination_path.parent().unwrap())?;
         }
         let conf = MacosConf;
-        pyo3_asyncio::tokio::future_into_py(py, async move {
+        pyo3_asyncio_0_21::tokio::future_into_py(py, async move {
             let (server, conf_tx) =
                 Server::init(conf, handle_tcp_stream, handle_udp_stream).await?;
             Ok(LocalRedirector::new(server, conf_tx))

mitmproxy-rs/src/server/udp.rs

@@ -36,7 +36,7 @@ impl UdpServer {
     ///
     /// This coroutine will yield once pending data has been flushed and all server tasks have
     /// successfully terminated after calling the `Server.close` method.
-    pub fn wait_closed<'p>(&self, py: Python<'p>) -> PyResult<&'p PyAny> {
+    pub fn wait_closed<'p>(&self, py: Python<'p>) -> PyResult<Bound<'p, PyAny>> {
         self.server.wait_closed(py)
     }
 
@@ -61,10 +61,10 @@ pub fn start_udp_server(
     host: String,
     port: u16,
     handle_udp_stream: PyObject,
-) -> PyResult<&PyAny> {
+) -> PyResult<Bound<PyAny>> {
     let conf = UdpConf { host, port };
     let handle_tcp_stream = py.None();
-    pyo3_asyncio::tokio::future_into_py(py, async move {
+    pyo3_asyncio_0_21::tokio::future_into_py(py, async move {
         let (server, local_addr) = Server::init(conf, handle_tcp_stream, handle_udp_stream).await?;
         Ok(UdpServer { server, local_addr })
     })

mitmproxy-rs/src/server/wireguard.rs

@@ -38,7 +38,7 @@ impl WireGuardServer {
     ///
     /// This coroutine will yield once pending data has been flushed and all server tasks have
     /// successfully terminated after calling the `Server.close` method.
-    pub fn wait_closed<'p>(&self, py: Python<'p>) -> PyResult<&'p PyAny> {
+    pub fn wait_closed<'p>(&self, py: Python<'p>) -> PyResult<Bound<'p, PyAny>> {
         self.server.wait_closed(py)
     }
 
@@ -69,7 +69,7 @@ pub fn start_wireguard_server(
     peer_public_keys: Vec<String>,
     handle_tcp_stream: PyObject,
     handle_udp_stream: PyObject,
-) -> PyResult<&PyAny> {
+) -> PyResult<Bound<PyAny>> {
     let private_key = string_to_key(private_key)?;
     let peer_public_keys = peer_public_keys
         .into_iter()
@@ -81,7 +81,7 @@ pub fn start_wireguard_server(
         private_key,
         peer_public_keys,
     };
-    pyo3_asyncio::tokio::future_into_py(py, async move {
+    pyo3_asyncio_0_21::tokio::future_into_py(py, async move {
         let (server, local_addr) = Server::init(conf, handle_tcp_stream, handle_udp_stream).await?;
         Ok(WireGuardServer { server, local_addr })
     })

mitmproxy-rs/src/stream.rs

@@ -37,15 +37,15 @@ pub struct Stream {
 
 /// Do *not* hold the GIL while accessing.
 static EMPTY_BYTES: Lazy<Py<PyBytes>> =
-    Lazy::new(|| Python::with_gil(|py| PyBytes::new(py, &[]).into_py(py)));
+    Lazy::new(|| Python::with_gil(|py| PyBytes::new_bound(py, &[]).unbind()));
 
 #[pymethods]
 impl Stream {
     /// Read up to `n` bytes of a TCP stream, or a single UDP packet (`n` is ignored for UDP).
     ///
     /// Return an empty `bytes` object if the connection was closed
     /// or the server has been shut down.
-    fn read<'p>(&self, py: Python<'p>, n: u32) -> PyResult<&'p PyAny> {
+    fn read<'py>(&self, py: Python<'py>, n: u32) -> PyResult<Bound<'py, PyAny>> {
         match self.state {
             StreamState::Open | StreamState::HalfClosed => {
                 let (tx, rx) = oneshot::channel();
@@ -54,16 +54,16 @@ impl Stream {
                     .send(TransportCommand::ReadData(self.connection_id, n, tx))
                     .ok(); // if this fails tx is dropped and rx.await will error.
 
-                pyo3_asyncio::tokio::future_into_py(py, async move {
+                pyo3_asyncio_0_21::tokio::future_into_py(py, async move {
                     if let Ok(data) = rx.await {
-                        Python::with_gil(|py| Ok(PyBytes::new(py, &data).into_py(py)))
+                        Python::with_gil(|py| Ok(PyBytes::new_bound(py, &data).unbind()))
                     } else {
                         Ok(EMPTY_BYTES.clone())
                     }
                 })
             }
             StreamState::Closed => {
-                pyo3_asyncio::tokio::future_into_py(py, async move { Ok(EMPTY_BYTES.clone()) })
+                pyo3_asyncio_0_21::tokio::future_into_py(py, async move { Ok(EMPTY_BYTES.clone()) })
             }
         }
     }
@@ -90,14 +90,14 @@ impl Stream {
     ///
     /// Raises:
     ///     OSError if the stream is closed or the server has been shut down.
-    fn drain<'p>(&self, py: Python<'p>) -> PyResult<&'p PyAny> {
+    fn drain<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
         let (tx, rx) = oneshot::channel();
 
         self.command_tx
             .send(TransportCommand::DrainWriter(self.connection_id, tx))
             .map_err(event_queue_unavailable)?;
 
-        pyo3_asyncio::tokio::future_into_py(py, async move {
+        pyo3_asyncio_0_21::tokio::future_into_py(py, async move {
             rx.await
                 .map_err(|_| PyOSError::new_err("connection closed"))
         })
@@ -146,8 +146,8 @@ impl Stream {
     }
 
     /// Wait until the stream is closed (currently a no-op).
-    fn wait_closed<'p>(&self, py: Python<'p>) -> PyResult<&'p PyAny> {
-        pyo3_asyncio::tokio::future_into_py(py, std::future::ready(Ok(())))
+    fn wait_closed<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        pyo3_asyncio_0_21::tokio::future_into_py(py, std::future::ready(Ok(())))
     }
 
     /// Query the stream for details of the underlying network connection.
@@ -165,9 +165,9 @@ impl Stream {
         match name.as_str() {
             "transport_protocol" => {
                 if self.connection_id.is_tcp() {
-                    return Ok(PyObject::from(intern!(py, "tcp")));
+                    return Ok(intern!(py, "tcp").to_object(py));
                 } else {
-                    return Ok(PyObject::from(intern!(py, "udp")));
+                    return Ok(intern!(py, "udp").to_object(py));
                 }
             }
             "peername" => return Ok(socketaddr_to_py(py, self.peername)),

mitmproxy-rs/src/task.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 
 use anyhow::{Context, Result};
 use pyo3::prelude::*;
-use pyo3_asyncio::TaskLocals;
+use pyo3_asyncio_0_21::TaskLocals;
 use tokio::sync::{broadcast, mpsc, Mutex};
 
 use mitmproxy::messages::{TransportCommand, TransportEvent};
@@ -31,8 +31,8 @@ impl PyInteropTask {
     ) -> Result<Self> {
         // Note: The current asyncio event loop needs to be determined here on the main thread.
         let locals = Python::with_gil(|py| -> Result<TaskLocals, PyErr> {
-            let py_loop = pyo3_asyncio::tokio::get_current_loop(py)?.into_py(py);
-            TaskLocals::new(py_loop.as_ref(py)).copy_context(py)
+            let py_loop = pyo3_asyncio_0_21::tokio::get_current_loop(py)?;
+            TaskLocals::new(py_loop).copy_context(py)
         })
         .context("failed to get python task locals")?;
 
@@ -92,7 +92,7 @@ impl PyInteropTask {
                                 };
 
                                 // convert Python awaitable into Rust Future
-                                let future = pyo3_asyncio::into_future_with_locals(&self.locals, coro.as_ref(py))?;
+                                let future = pyo3_asyncio_0_21::into_future_with_locals(&self.locals, coro.into_bound(py))?;
 
                                 // run Future on a new Tokio task
                                 let handle = {

mitmproxy-rs/src/udp_client.rs

@@ -23,8 +23,8 @@ pub fn open_udp_connection(
     host: String,
     port: u16,
     local_addr: Option<(String, u16)>,
-) -> PyResult<&PyAny> {
-    pyo3_asyncio::tokio::future_into_py(py, async move {
+) -> PyResult<Bound<PyAny>> {
+    pyo3_asyncio_0_21::tokio::future_into_py(py, async move {
         let socket = udp_connect(host, port, local_addr).await?;
 
         let peername = socket.peer_addr()?;

mitmproxy-rs/src/util.rs

@@ -73,8 +73,8 @@ pub fn add_cert(py: Python<'_>, pem: String) -> PyResult<()> {
             .take_while(|&line| line != "-----END CERTIFICATE-----")
             .collect::<String>();
 
-        let filename = py.import("mitmproxy_rs")?.filename()?;
-        let executable_path = std::path::Path::new(filename)
+        let filename = py.import_bound("mitmproxy_rs")?.filename()?;
+        let executable_path = std::path::Path::new(filename.to_str()?)
             .parent()
             .ok_or_else(|| anyhow!("invalid path"))?
             .join("macos-certificate-truster.app");