Convert ndarray scalars to Span<T> using the buffer protocol

src/CSnakes.Runtime/CPython/Buffer.cs

@@ -0,0 +1,60 @@
+using CSnakes.Runtime.Python;
+using System.Runtime.InteropServices;
+
+namespace CSnakes.Runtime.CPython;
+internal unsafe partial class CPythonAPI
+{
+    private const int PyBUF_SIMPLE = 0;
+    private const int PyBUF_WRITABLE = 0x0001;
+    private const int PyBUF_FORMAT = 0x0004;
+    private const int PyBUF_ND = 0x0008;
+    private const int PyBUF_STRIDES = (0x0010 | PyBUF_ND);
+    private const int PyBUF_C_CONTIGUOUS = (0x0020 | PyBUF_STRIDES);
+    private const int PyBUF_F_CONTIGUOUS = (0x0040 | PyBUF_STRIDES);
+    private const int PyBUF_ANY_CONTIGUOUS = (0x0080 | PyBUF_STRIDES);
+
+    [StructLayout(LayoutKind.Sequential)]
+    internal struct Py_buffer
+    {
+        public IntPtr buf;
+        public IntPtr obj;
+        public nint len;
+        public Int32 @readonly;
+        public nint itemsize;
+        public char* format;
+        public Int32 ndim;
+        public nint* shape;
+        public nint* strides;
+        public nint* suboffsets;
+        public nint* @internal;
+    }
+
+    public static bool IsBuffer(PyObject p)
+    {
+        return PyObject_CheckBuffer(p) == 1;
+    }
+
+    internal static Py_buffer GetBuffer(PyObject p)
+    {
+        Py_buffer view = default;
+        if (PyObject_GetBuffer(p, &view, PyBUF_FORMAT) != 0)
+        {
+            throw PyObject.ThrowPythonExceptionAsClrException();
+        }
+        return view;
+    }
+
+    internal static void ReleaseBuffer(Py_buffer view)
+    {
+        PyBuffer_Release(&view);
+    }
+
+    [LibraryImport(PythonLibraryName)]
+    private static partial int PyObject_CheckBuffer(PyObject ob);
+
+    [LibraryImport(PythonLibraryName)]
+    private static partial int PyObject_GetBuffer(PyObject ob, Py_buffer* view, int flags);
+
+    [LibraryImport(PythonLibraryName)]
+    private static partial void PyBuffer_Release(Py_buffer* view);
+}

src/CSnakes.Runtime/PyObjectTypeConverter.KnownTypes.cs

@@ -9,4 +9,5 @@ internal partial class PyObjectTypeConverter
     static readonly Type dictionaryType = typeof(IReadOnlyDictionary<,>);
     static readonly Type pyObjectType = typeof(PyObject);
     static readonly Type generatorIteratorType = typeof(IGeneratorIterator<,,>);
+    static readonly Type bufferType = typeof(IPyBuffer);
 }

src/CSnakes.Runtime/PyObjectTypeConverter.cs

@@ -34,6 +34,11 @@ public static object PyObjectToManagedType(PyObject pyObject, Type destinationTy
             return ConvertToList(pyObject, destinationType);
         }
 
+        if (CPythonAPI.IsBuffer(pyObject) && destinationType.IsAssignableTo(bufferType))
+        {
+            return new PyBuffer(pyObject);
+        }
+
         throw new InvalidCastException($"Attempting to cast {destinationType} from {pyObject.GetPythonType()}");
     }
 

src/CSnakes.Runtime/Python/IPyBuffer.cs

@@ -0,0 +1,12 @@
+namespace CSnakes.Runtime.Python;
+public interface IPyBuffer
+{
+    Int64 Length { get; }
+
+    bool Scalar { get; }
+
+    Span<Int32> AsInt32Scalar();
+    Span<Int64> AsInt64Scalar();
+    Span<UInt32> AsUInt32Scalar();
+    Span<UInt64> AsUInt64Scalar();
+}

src/CSnakes.Runtime/Python/PyBuffer.cs

@@ -0,0 +1,65 @@
+using CSnakes.Runtime.CPython;
+
+namespace CSnakes.Runtime.Python;
+internal class PyBuffer : IPyBuffer
+{
+    private readonly CPythonAPI.Py_buffer _buffer;
+
+    public PyBuffer(PyObject exporter)
+    {
+        _buffer = CPythonAPI.GetBuffer(exporter);
+    }
+
+    public void Dispose()
+    {
+        CPythonAPI.ReleaseBuffer(_buffer);
+    }
+
+    public Int64 Length => _buffer.len;
+
+    public bool Scalar => _buffer.ndim == 0;
+
+    private unsafe void EnsureFormat(char format)
+    {
+        if (!_buffer.format[0].Equals(format))
+        {
+            throw new InvalidOperationException($"Buffer is not a {format}, it is {_buffer.format[0]}");
+        }
+    }
+
+    private void EnsureScalar()
+    {
+        if (!Scalar)
+        {
+            throw new InvalidOperationException("Buffer is not a scalar");
+        }
+    }
+
+    public unsafe Span<Int32> AsInt32Scalar()
+    {
+        EnsureScalar();
+        EnsureFormat('l'); // TODO: i is also valid
+        return new Span<Int32>((void*)_buffer.buf, (int)(Length / 4));
+    }
+
+    public unsafe Span<UInt32> AsUInt32Scalar()
+    {
+        EnsureScalar();
+        EnsureFormat('L'); // TODO: I is also valid
+        return new Span<UInt32>((void*)_buffer.buf, (int)(Length / 4));
+    }
+
+    public unsafe Span<Int64> AsInt64Scalar()
+    {
+        EnsureScalar();
+        EnsureFormat('q');
+        return new Span<Int64>((void*)_buffer.buf, (int)(Length / 8));
+    }
+
+    public unsafe Span<UInt64> AsUInt64Scalar()
+    {
+        EnsureScalar();
+        EnsureFormat('Q');
+        return new Span<UInt64>((void*)_buffer.buf, (int)(Length / 8));
+    }
+}

src/CSnakes.Tests/GeneratedSignatureTests.cs

@@ -36,9 +36,9 @@ public class GeneratedSignatureTests(TestEnvironment testEnv) : IClassFixture<Te
     [InlineData("def hello(a: int = 2147483648) -> None:\n ...\n", "void Hello(long a = 2147483648L)")]
     [InlineData("def hello(a: Optional[int] = None) -> None:\n ...\n", "void Hello(long? a = null)")]
     [InlineData("def hello(a: typing.List[int], b: typing.Dict[str, int]) -> typing.Tuple[str, int]:\n ...\n", "public (string, long) Hello(IReadOnlyList<long> a, IReadOnlyDictionary<string, long> b)")]
+    [InlineData("def hello() -> Buffer:\n ...\n", "IPyBuffer Hello()")]
     public void TestGeneratedSignature(string code, string expected)
     {
-
         var tempName = string.Format("{0}_{1:N}", "test", Guid.NewGuid().ToString("N"));
         File.WriteAllText(Path.Combine(testEnv.TempDir, $"{tempName}.py"), code);
 

src/CSnakes.Tests/TypeReflectionTests.cs

@@ -70,7 +70,7 @@ private static void ParsingTestInternal(string pythonType, string expectedType)
         var result = PythonParser.PythonTypeDefinitionTokenizer.TryParse(tokens);
         Assert.True(result.HasValue);
         Assert.NotNull(result.Value);
-        var reflectedType = TypeReflection.AsPredefinedType(result.Value);
+        var reflectedType = TypeReflection.AsPredefinedType(result.Value, TypeReflection.ConversionDirection.FromPython);
         Assert.Equal(expectedType, reflectedType.ToString());
     }
 }

src/CSnakes/Reflection/ArgumentReflection.cs

@@ -22,8 +22,8 @@ public class ArgumentReflection
         TypeSyntax reflectedType = parameter.ParameterType switch
         {
             PythonFunctionParameterType.Star => ArrayPyObject,
-            PythonFunctionParameterType.DoubleStar => TypeReflection.AsPredefinedType(DictStrAny),
-            PythonFunctionParameterType.Normal => TypeReflection.AsPredefinedType(parameter.Type),
+            PythonFunctionParameterType.DoubleStar => TypeReflection.AsPredefinedType(DictStrAny, TypeReflection.ConversionDirection.ToPython),
+            PythonFunctionParameterType.Normal => TypeReflection.AsPredefinedType(parameter.Type, TypeReflection.ConversionDirection.ToPython),
             _ => throw new NotImplementedException()
         };
 

src/CSnakes/Reflection/MethodReflection.cs

@@ -28,7 +28,7 @@ public static MethodDefinition FromMethod(PythonFunctionDefinition function, str
         }
         else
         {
-            returnSyntax = TypeReflection.AsPredefinedType(returnPythonType);
+            returnSyntax = TypeReflection.AsPredefinedType(returnPythonType, TypeReflection.ConversionDirection.FromPython);
         }
 
         // Step 3: Build arguments
@@ -107,7 +107,7 @@ public static MethodDefinition FromMethod(PythonFunctionDefinition function, str
         ReturnStatementSyntax returnExpression = returnSyntax switch
         {
             TypeSyntax s when s is PredefinedTypeSyntax p && p.Keyword.IsKind(SyntaxKind.VoidKeyword) => ReturnStatement(null),
-            TypeSyntax s when s is IdentifierNameSyntax => ReturnStatement(IdentifierName("__result_pyObject")),
+            TypeSyntax s when s is IdentifierNameSyntax && ((IdentifierNameSyntax)s)?.Identifier.ValueText == "PyObject" => ReturnStatement(IdentifierName("__result_pyObject")),
             _ => ProcessMethodWithReturnType(returnSyntax, parameterGenericArgs)
         };
 

src/CSnakes/Reflection/TypeReflection.cs

@@ -2,66 +2,77 @@
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
 using CSnakes.Parser.Types;
+using System.ComponentModel;
 
 namespace CSnakes.Reflection;
 
 public static class TypeReflection
 {
-    public static TypeSyntax AsPredefinedType(PythonTypeSpec pythonType)
+    public enum ConversionDirection
+    {
+        ToPython,
+        FromPython
+    }
+
+    public static TypeSyntax AsPredefinedType(PythonTypeSpec pythonType, ConversionDirection direction)
     {
         // If type is an alias, e.g. "list[int]", "list[float]", etc.
         if (pythonType.HasArguments())
         {
             // Get last occurrence of ] in pythonType
             return pythonType.Name.Replace("typing.", "") switch
             {
-                "list" => CreateListType(pythonType.Arguments[0]),
-                "List" => CreateListType(pythonType.Arguments[0]),
-                "Tuple" => CreateTupleType(pythonType.Arguments),
-                "tuple" => CreateTupleType(pythonType.Arguments),
-                "dict" => CreateDictionaryType(pythonType.Arguments[0], pythonType.Arguments[1]),
-                "Dict" => CreateDictionaryType(pythonType.Arguments[0], pythonType.Arguments[1]),
-                "Mapping" => CreateDictionaryType(pythonType.Arguments[0], pythonType.Arguments[1]),
-                "Sequence" => CreateListType(pythonType.Arguments[0]),
-                "Optional" => AsPredefinedType(pythonType.Arguments[0]),
-                "Generator" => CreateGeneratorType(pythonType.Arguments[0], pythonType.Arguments[1], pythonType.Arguments[2]),
+                "list" => CreateListType(pythonType.Arguments[0], direction),
+                "List" => CreateListType(pythonType.Arguments[0], direction),
+                "Tuple" => CreateTupleType(pythonType.Arguments, direction),
+                "tuple" => CreateTupleType(pythonType.Arguments, direction),
+                "dict" => CreateDictionaryType(pythonType.Arguments[0], pythonType.Arguments[1], direction),
+                "Dict" => CreateDictionaryType(pythonType.Arguments[0], pythonType.Arguments[1], direction),
+                "Mapping" => CreateDictionaryType(pythonType.Arguments[0], pythonType.Arguments[1], direction),
+                "Sequence" => CreateListType(pythonType.Arguments[0], direction),
+                "Optional" => AsPredefinedType(pythonType.Arguments[0], direction),
+                "Generator" => CreateGeneratorType(pythonType.Arguments[0], pythonType.Arguments[1], pythonType.Arguments[2], direction),
+
                 // Todo more types... see https://docs.python.org/3/library/stdtypes.html#standard-generic-classes
                 _ => SyntaxFactory.ParseTypeName("PyObject"),
             };
         }
-        return pythonType.Name switch
+        return (pythonType.Name, direction) switch
         {
-            "int" => SyntaxFactory.PredefinedType(SyntaxFactory.Token(SyntaxKind.LongKeyword)),
-            "str" => SyntaxFactory.PredefinedType(SyntaxFactory.Token(SyntaxKind.StringKeyword)),
-            "float" => SyntaxFactory.PredefinedType(SyntaxFactory.Token(SyntaxKind.DoubleKeyword)),
-            "bool" => SyntaxFactory.PredefinedType(SyntaxFactory.Token(SyntaxKind.BoolKeyword)),
-            "bytes" => SyntaxFactory.ParseTypeName("byte[]"),
+            ("int", _) => SyntaxFactory.PredefinedType(SyntaxFactory.Token(SyntaxKind.LongKeyword)),
+            ("str", _) => SyntaxFactory.PredefinedType(SyntaxFactory.Token(SyntaxKind.StringKeyword)),
+            ("float", _) => SyntaxFactory.PredefinedType(SyntaxFactory.Token(SyntaxKind.DoubleKeyword)),
+            ("bool", _) => SyntaxFactory.PredefinedType(SyntaxFactory.Token(SyntaxKind.BoolKeyword)),
+            ("bytes", _) => SyntaxFactory.ParseTypeName("byte[]"),
+            ("Buffer", ConversionDirection.FromPython) => SyntaxFactory.ParseTypeName("IPyBuffer"),
+            ("typing.Buffer", ConversionDirection.FromPython) => SyntaxFactory.ParseTypeName("IPyBuffer"),
+            ("collections.abc.Buffer", ConversionDirection.FromPython) => SyntaxFactory.ParseTypeName("IPyBuffer"),
             _ => SyntaxFactory.ParseTypeName("PyObject"),
         };
     }
 
-    private static TypeSyntax CreateDictionaryType(PythonTypeSpec keyType, PythonTypeSpec valueType) => CreateGenericType("IReadOnlyDictionary", [
-            AsPredefinedType(keyType),
-            AsPredefinedType(valueType)
+    private static TypeSyntax CreateDictionaryType(PythonTypeSpec keyType, PythonTypeSpec valueType, ConversionDirection direction) => CreateGenericType("IReadOnlyDictionary", [
+            AsPredefinedType(keyType, direction),
+            AsPredefinedType(valueType, direction)
             ]);
 
-    private static TypeSyntax CreateGeneratorType(PythonTypeSpec yieldType, PythonTypeSpec sendType, PythonTypeSpec returnType) => CreateGenericType("IGeneratorIterator", [
-            AsPredefinedType(yieldType),
-            AsPredefinedType(sendType),
-            AsPredefinedType(returnType)
+    private static TypeSyntax CreateGeneratorType(PythonTypeSpec yieldType, PythonTypeSpec sendType, PythonTypeSpec returnType, ConversionDirection direction) => CreateGenericType("IGeneratorIterator", [
+            AsPredefinedType(yieldType, direction),
+            AsPredefinedType(sendType, direction),
+            AsPredefinedType(returnType, direction)
             ]);
 
-    private static TypeSyntax CreateTupleType(PythonTypeSpec[] tupleTypes)
+    private static TypeSyntax CreateTupleType(PythonTypeSpec[] tupleTypes, ConversionDirection direction)
     {
         if (tupleTypes.Length == 1)
         {
-            return CreateGenericType("ValueTuple", tupleTypes.Select(AsPredefinedType));
+            return CreateGenericType("ValueTuple", tupleTypes.Select(t => AsPredefinedType(t, direction)));
         }
 
         IEnumerable<TupleElementSyntax> tupleTypeSyntaxGroups = tupleTypes.Select((x, i) => new { Index = i, Value = x })
             .GroupBy(x => x.Index / 7)
             .Select(x => x.Select(v => v.Value))
-            .Select(typeSpecs => typeSpecs.Select(AsPredefinedType))
+            .Select(typeSpecs => typeSpecs.Select(t => AsPredefinedType(t, direction)))
             .SelectMany(item => item.Select(SyntaxFactory.TupleElement));
 
         return SyntaxFactory.TupleType(
@@ -70,7 +81,7 @@ private static TypeSyntax CreateTupleType(PythonTypeSpec[] tupleTypes)
             SyntaxFactory.Token(SyntaxKind.CloseParenToken));
     }
 
-    private static TypeSyntax CreateListType(PythonTypeSpec genericOf) => CreateGenericType("IReadOnlyList", [AsPredefinedType(genericOf)]);
+    private static TypeSyntax CreateListType(PythonTypeSpec genericOf, ConversionDirection direction) => CreateGenericType("IReadOnlyList", [AsPredefinedType(genericOf, direction)]);
 
     internal static TypeSyntax CreateGenericType(string typeName, IEnumerable<TypeSyntax> genericArguments) =>
         SyntaxFactory.GenericName(

src/Integration.Tests/BufferTests.cs

@@ -0,0 +1,24 @@
+namespace Integration.Tests;
+
+public class BufferTests : IntegrationTestBase
+{
+    [Fact]
+    public void TestSimpleBuffer()
+    {
+        // SKip if < Python 3.12
+        if (new Version(Env.Version.Split(' ')[0]) < new Version(3, 11, 0))
+        {
+            return;
+        }
+        var testModule = Env.TestBuffer();
+        var bufferObject = testModule.TestSimpleBuffer();
+        Assert.Equal(20, bufferObject.Length); // 5 * sizeof(int)
+        Assert.True(bufferObject.Scalar);
+
+        // Check the buffer contents
+        Span<Int32> result = bufferObject.AsInt32Scalar();
+        Assert.Equal((int)1, result[0]);
+        Assert.Equal((int)5, result[4]);
+
+    }
+}

src/Integration.Tests/Integration.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <IsPackable>false</IsPackable>
@@ -10,6 +10,7 @@
     <None Remove="python\requirements.txt" />
     <None Remove="python\test_args.py" />
     <None Remove="python\test_basic.py" />
+    <None Remove="python\test_buffer.py" />
     <None Remove="python\test_defaults.py" />
     <None Remove="python\test_dependency.py" />
     <None Remove="python\test_dicts.py" />
@@ -29,6 +30,9 @@
     <AdditionalFiles Include="python\test_basic.py">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </AdditionalFiles>
+    <AdditionalFiles Include="python\test_buffer.py">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </AdditionalFiles>
     <AdditionalFiles Include="python\test_defaults.py">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </AdditionalFiles>

src/Integration.Tests/python/requirements.txt

@@ -1 +1,3 @@
-httpx
+httpx
+numpy
+typing-extensions

src/Integration.Tests/python/test_buffer.py

@@ -0,0 +1,10 @@
+try:
+    from collections.abc import Buffer
+except ImportError:
+    from typing_extensions import Buffer
+
+import numpy as np
+
+
+def test_simple_buffer() -> Buffer:
+    return np.array([1, 2, 3, 4, 5], dtype=np.int32)