[WIP] Switch from Rust-cpython to PyO3

[ci skip-build-wheels]

src/rust/engine/Cargo.toml

@@ -138,6 +138,7 @@ num_enum = "0.4"
 parking_lot = "0.11"
 petgraph = "0.5"
 process_execution = { path = "process_execution" }
+pyo3 = "0.15"
 rand = "0.8"
 regex = "1"
 reqwest = { version = "0.11", default_features = false, features = ["stream", "rustls-tls"] }

src/rust/engine/src/externs/mod.rs

@@ -23,25 +23,26 @@ use std::fmt;
 use crate::interning::Interns;
 use crate::python::{Failure, Key, TypeId, Value};
 
-use cpython::{
-  py_class, CompareOp, FromPyObject, ObjectProtocol, PyBool, PyBytes, PyClone, PyDict, PyErr,
-  PyObject, PyResult as CPyResult, PyTuple, PyType, Python, PythonObject, ToPyObject,
-};
+use cpython::ObjectProtocol;
 use lazy_static::lazy_static;
+use pyo3::basic::CompareOp;
+use pyo3::prelude::*;
+use pyo3::types::{PyBool, PyBytes, PyDict, PyTuple};
+use pyo3::ToPyObject;
 
 use logging::PythonLogLevel;
 
-pub fn equals(py: Python, h1: &PyObject, h2: &PyObject) -> bool {
+pub fn equals(h1: &PyAny, h2: &PyAny) -> bool {
   // NB: Although it does not precisely align with Python's definition of equality, we ban matches
   // between non-equal types to avoid legacy behavior like `assert True == 1`, which is very
   // surprising in interning, and would likely be surprising anywhere else in the engine where we
   // compare things.
-  if h1.get_type(py) != h2.get_type(py) {
+  if h1.get_type() != h2.get_type() {
     return false;
   }
-  h1.rich_compare(py, h2, CompareOp::Eq)
+  h1.rich_compare(h2, CompareOp::Eq)
     .unwrap()
-    .cast_as::<PyBool>(py)
+    .cast_as::<PyBool>()
     .unwrap()
     .is_true()
 }
@@ -55,14 +56,10 @@ pub fn store_tuple(py: Python, values: Vec<Value>) -> Value {
 }
 
 /// Store a slice containing 2-tuples of (key, value) as a Python dictionary.
-pub fn store_dict(py: Python, keys_and_values: Vec<(Value, Value)>) -> Result<Value, PyErr> {
+pub fn store_dict(py: Python, keys_and_values: Vec<(Value, Value)>) -> PyResult<Value> {
   let dict = PyDict::new(py);
   for (k, v) in keys_and_values {
-    dict.set_item(
-      py,
-      k.consume_into_py_object(py),
-      v.consume_into_py_object(py),
-    )?;
+    dict.set_item(k.consume_into_py_object(py), v.consume_into_py_object(py))?;
   }
   Ok(Value::from(dict.into_object()))
 }
@@ -72,31 +69,31 @@ pub fn store_bytes(py: Python, bytes: &[u8]) -> Value {
   Value::from(PyBytes::new(py, bytes).into_object())
 }
 
-/// Store an buffer of utf8 bytes to pass to Python. This will end up as a Python `str`.
+/// Store a buffer of utf8 bytes to pass to Python. This will end up as a Python `str`.
 pub fn store_utf8(py: Python, utf8: &str) -> Value {
-  Value::from(utf8.to_py_object(py).into_object())
+  Value::from(utf8.to_object(py).into_object())
 }
 
 pub fn store_u64(py: Python, val: u64) -> Value {
-  Value::from(val.to_py_object(py).into_object())
+  Value::from(val.to_object(py).into_object())
 }
 
 pub fn store_i64(py: Python, val: i64) -> Value {
-  Value::from(val.to_py_object(py).into_object())
+  Value::from(val.to_object(py).into_object())
 }
 
 pub fn store_bool(py: Python, val: bool) -> Value {
-  Value::from(val.to_py_object(py).into_object())
+  Value::from(val.to_object(py).into_object())
 }
 
 ///
 /// Gets an attribute of the given value as the given type.
 ///
-pub fn getattr<T>(value: &PyObject, field: &str) -> Result<T, String>
+pub fn getattr<T>(value: &cpython::PyObject, field: &str) -> Result<T, String>
 where
-  for<'a> T: FromPyObject<'a>,
+  for<'a> T: cpython::FromPyObject<'a>,
 {
-  let gil = Python::acquire_gil();
+  let gil = cpython::Python::acquire_gil();
   let py = gil.python();
   value
     .getattr(py, field)
@@ -115,8 +112,8 @@ where
 ///
 /// Collect the Values contained within an outer Python Iterable PyObject.
 ///
-pub fn collect_iterable(value: &PyObject) -> Result<Vec<PyObject>, String> {
-  let gil = Python::acquire_gil();
+pub fn collect_iterable(value: &cpython::PyObject) -> Result<Vec<cpython::PyObject>, String> {
+  let gil = cpython::Python::acquire_gil();
   let py = gil.python();
   match value.iter(py) {
     Ok(py_iter) => py_iter
@@ -140,10 +137,10 @@ pub fn collect_iterable(value: &PyObject) -> Result<Vec<PyObject>, String> {
   }
 }
 
-pub fn getattr_from_frozendict(value: &PyObject, field: &str) -> BTreeMap<String, String> {
+pub fn getattr_from_frozendict(value: &cpython::PyObject, field: &str) -> BTreeMap<String, String> {
   let frozendict = getattr(value, field).unwrap();
-  let pydict: PyDict = getattr(&frozendict, "_data").unwrap();
-  let gil = Python::acquire_gil();
+  let pydict: cpython::PyDict = getattr(&frozendict, "_data").unwrap();
+  let gil = cpython::Python::acquire_gil();
   let py = gil.python();
   pydict
     .items(py)
@@ -152,7 +149,11 @@ pub fn getattr_from_frozendict(value: &PyObject, field: &str) -> BTreeMap<String
     .collect()
 }
 
-pub fn getattr_as_optional_string(py: Python, value: &PyObject, field: &str) -> Option<String> {
+pub fn getattr_as_optional_string(
+  py: cpython::Python,
+  value: &cpython::PyObject,
+  field: &str,
+) -> Option<String> {
   let v = value.getattr(py, field).unwrap();
   if v.is_none(py) {
     return None;
@@ -162,8 +163,8 @@ pub fn getattr_as_optional_string(py: Python, value: &PyObject, field: &str) ->
   Some(v.extract(py).unwrap())
 }
 
-pub fn val_to_str(obj: &PyObject) -> String {
-  let gil = Python::acquire_gil();
+pub fn val_to_str(obj: &cpython::PyObject) -> String {
+  let gil = cpython::Python::acquire_gil();
   let py = gil.python();
 
   if *obj == py.None() {
@@ -174,7 +175,7 @@ pub fn val_to_str(obj: &PyObject) -> String {
   pystring.to_string(py).unwrap().into_owned()
 }
 
-pub fn val_to_log_level(obj: &PyObject) -> Result<log::Level, String> {
+pub fn val_to_log_level(obj: &cpython::PyObject) -> Result<log::Level, String> {
   let res: Result<PythonLogLevel, String> = getattr(obj, "_level").and_then(|n: u64| {
     n.try_into()
       .map_err(|e: num_enum::TryFromPrimitiveError<_>| {
@@ -186,39 +187,43 @@ pub fn val_to_log_level(obj: &PyObject) -> Result<log::Level, String> {
 
 /// Link to the Pants docs using the current version of Pants.
 pub fn doc_url(py: Python, slug: &str) -> String {
-  let docutil = py.import("pants.util.docutil").unwrap();
-  docutil
-    .call(py, "doc_url", (slug,), None)
-    .unwrap()
-    .extract(py)
-    .unwrap()
+  let docutil_module = py.import("pants.util.docutil").unwrap();
+  let doc_url_func = docutil_module.getattr("doc_url").unwrap();
+  doc_url_func.call1((slug,)).unwrap().extract().unwrap()
 }
 
-pub fn create_exception(py: Python, msg: &str) -> Value {
-  Value::from(PyErr::new::<cpython::exc::Exception, _>(py, msg).instance(py))
+pub fn create_exception(py: cpython::Python, msg: &str) -> Value {
+  Value::from(cpython::PyErr::new::<cpython::exc::Exception, _>(py, msg).instance(py))
 }
 
-pub fn call_method0(py: Python, value: &PyObject, method: &str) -> Result<PyObject, PyErr> {
-  value.call_method(py, method, PyTuple::new(py, &[]), None)
+pub fn call_method0(
+  py: cpython::Python,
+  value: &cpython::PyObject,
+  method: &str,
+) -> Result<cpython::PyObject, cpython::PyErr> {
+  value.call_method(py, method, cpython::PyTuple::new(py, &[]), None)
 }
 
-pub fn call_function<T: AsRef<PyObject>>(func: T, args: &[Value]) -> Result<PyObject, PyErr> {
-  let func: &PyObject = func.as_ref();
-  let arg_handles: Vec<PyObject> = args.iter().map(|v| v.clone().into()).collect();
-  let gil = Python::acquire_gil();
-  let args_tuple = PyTuple::new(gil.python(), &arg_handles);
+pub fn call_function<T: AsRef<cpython::PyObject>>(
+  func: T,
+  args: &[Value],
+) -> Result<cpython::PyObject, cpython::PyErr> {
+  let func: &cpython::PyObject = func.as_ref();
+  let arg_handles: Vec<cpython::PyObject> = args.iter().map(|v| v.clone().into()).collect();
+  let gil = cpython::Python::acquire_gil();
+  let args_tuple = cpython::PyTuple::new(gil.python(), &arg_handles);
   func.call(gil.python(), args_tuple, None)
 }
 
 pub fn generator_send(generator: &Value, arg: &Value) -> Result<GeneratorResponse, Failure> {
-  let gil = Python::acquire_gil();
+  let gil = cpython::Python::acquire_gil();
   let py = gil.python();
   let selectors = py.import("pants.engine.internals.selectors").unwrap();
   let response = selectors
     .call(
       py,
       "native_engine_generator_send",
-      (generator as &PyObject, arg as &PyObject),
+      (generator as &cpython::PyObject, arg as &cpython::PyObject),
       None,
     )
     .map_err(|py_err| Failure::from_py_err_with_gil(py, py_err))?;
@@ -253,15 +258,15 @@ pub fn generator_send(generator: &Value, arg: &Value) -> Result<GeneratorRespons
 /// those configured in types::Types.
 pub fn unsafe_call(py: Python, type_id: TypeId, args: &[Value]) -> Value {
   let py_type = type_id.as_py_type(py);
-  let arg_handles: Vec<PyObject> = args.iter().map(|v| v.clone().into()).collect();
+  let arg_handles: Vec<cpython::PyObject> = args.iter().map(|v| v.clone().into()).collect();
   let args_tuple = PyTuple::new(py, &arg_handles);
   py_type
-    .call(py, args_tuple, None)
+    .call(args_tuple, None)
     .map(Value::from)
     .unwrap_or_else(|e| {
       panic!(
         "Core type constructor `{}` failed: {:?}",
-        py_type.name(py),
+        py_type.name().unwrap(),
         e
       );
     })

src/rust/engine/src/interning.rs

@@ -5,8 +5,9 @@ use std::collections::HashMap;
 use std::hash;
 use std::sync::atomic;
 
-use cpython::{ObjectProtocol, PyErr, PyType, Python, ToPyObject};
 use parking_lot::{Mutex, RwLock};
+use pyo3::prelude::*;
+use pyo3::types::PyType;
 
 use crate::externs;
 use crate::python::{Fnv, Key, Value};
@@ -51,10 +52,10 @@ impl Interns {
     Interns::default()
   }
 
-  pub fn key_insert(&self, py: Python, v: Value) -> Result<Key, PyErr> {
+  pub fn key_insert(&self, py: Python, v: Value) -> PyResult<Key> {
     let (intern_key, type_id) = {
-      let obj = v.to_py_object(py).into();
-      (InternKey(v.hash(py)?, obj), (&v.get_type(py)).into())
+      let obj = v.to_object(py).into();
+      (InternKey(v.hash()?, obj), (&v.get_type()).into())
     };
 
     py.allow_threads(|| {
@@ -99,8 +100,7 @@ impl Eq for InternKey {}
 
 impl PartialEq for InternKey {
   fn eq(&self, other: &InternKey) -> bool {
-    let gil = Python::acquire_gil();
-    externs::equals(gil.python(), &self.1, &other.1)
+    Python::with_gil(|py| externs::equals(self.1.into_ref(py), other.1.into_ref(py)))
   }
 }