Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert ndarray scalars to Span<T> using the buffer protocol #187

Merged
merged 55 commits into from
Sep 18, 2024
Merged
Changes from 1 commit
Commits
Show all changes
55 commits
Select commit Hold shift + click to select a range
b778aa8
Add numpy as a dependency for integration tests
tonybaloney Sep 5, 2024
73a9e3d
Expose ndarray scalars using the Python Buffer protocol
tonybaloney Sep 5, 2024
082c9b4
Add float scalars and test
tonybaloney Sep 5, 2024
7ffeb1a
Update src/CSnakes.Runtime/Python/IPyBuffer.cs
tonybaloney Sep 5, 2024
5cae82f
Update src/CSnakes.Runtime/Python/PyBuffer.cs
tonybaloney Sep 5, 2024
d339e8d
Update src/CSnakes.Runtime/Python/PyBuffer.cs
tonybaloney Sep 5, 2024
9dc0ddc
Update src/CSnakes.Runtime/Python/PyBuffer.cs
tonybaloney Sep 5, 2024
cb54930
Update src/CSnakes.Runtime/CPython/Buffer.cs
tonybaloney Sep 5, 2024
9e1f0c1
Update src/CSnakes.Runtime/CPython/Buffer.cs
tonybaloney Sep 5, 2024
0693a1b
Marshall format into a string
tonybaloney Sep 5, 2024
7f52a09
Add more conversions
tonybaloney Sep 5, 2024
6130c13
Using single-point precision for the float32 test
tonybaloney Sep 5, 2024
be317a8
Add byte-order and alignment accessor
tonybaloney Sep 5, 2024
9225cb7
Add the common format string types to the class
tonybaloney Sep 5, 2024
f44a778
Fix the struct sequence and add a 2D test
tonybaloney Sep 5, 2024
ff706ef
Add support for 2D vectors using the performance community extensions
tonybaloney Sep 6, 2024
31bab09
Add a test to verify that is writable
tonybaloney Sep 6, 2024
1840a78
Add some length and size checks
tonybaloney Sep 6, 2024
ec1f92f
Platform independent marshalling
tonybaloney Sep 6, 2024
d4077ab
Remove the Python 3.12 checks
tonybaloney Sep 6, 2024
8ee47fa
Add int8 int16 and readonly span
tonybaloney Sep 6, 2024
514454e
Use concrete readonly span implementations
tonybaloney Sep 6, 2024
8e4db0b
Test bytes and bytearray
tonybaloney Sep 6, 2024
deb3fde
Add a gIL to the initial buffer fetch and releases
tonybaloney Sep 6, 2024
e201c90
Explicitly request C Contiguous arrays incase someone tries to give t…
tonybaloney Sep 6, 2024
3f11ee0
Update Buffer.cs
tonybaloney Sep 6, 2024
80d9e7e
Update Buffer.cs
tonybaloney Sep 6, 2024
5d2333e
Update PyBuffer.cs
tonybaloney Sep 6, 2024
5d3d2c2
Update PyBuffer.cs
tonybaloney Sep 6, 2024
3cc7f97
Test ndarray transposition (non-contiguous array) errors, slicings an…
tonybaloney Sep 6, 2024
547db87
Merge branch 'numpy_tests' of https://github.com/tonybaloney/CSnakes …
tonybaloney Sep 6, 2024
75c0abb
Simplify types in API. Add bool, nint and unint conversions
tonybaloney Sep 16, 2024
d93295d
Apply suggestions from code review
tonybaloney Sep 16, 2024
49e06ac
Fix byte order enum parsing
tonybaloney Sep 16, 2024
c4f7948
Platform agnostic implementations
tonybaloney Sep 16, 2024
dabe33a
Add documentation
tonybaloney Sep 16, 2024
aacea48
Merge branch 'main' into numpy_tests
tonybaloney Sep 16, 2024
71d91f0
Make buffer span methods generic and move the convenience methods to …
tonybaloney Sep 17, 2024
3718b78
Merge branch 'numpy_tests' of https://github.com/tonybaloney/CSnakes …
tonybaloney Sep 17, 2024
ecf8100
Simplify names
tonybaloney Sep 17, 2024
f9f4bfa
Rename extension class
tonybaloney Sep 17, 2024
d6369b3
Tightening up the code a little
aaronpowell Sep 17, 2024
ec1d3c9
Update docs/buffers.md
tonybaloney Sep 17, 2024
d889bbb
Make a big red box with the warning about editing values
tonybaloney Sep 17, 2024
2c18f85
Merge branch 'numpy_tests' of https://github.com/tonybaloney/CSnakes …
tonybaloney Sep 17, 2024
aa110f1
Add remark about contiguous arrays
tonybaloney Sep 17, 2024
befea5c
Rebase from main
tonybaloney Sep 18, 2024
144e30c
Reference Tensor<T>
RussKie Sep 18, 2024
6bb3666
Merge pull request #213 from RussKie/numpy_tests
tonybaloney Sep 18, 2024
8b4dc70
Implement tensor spans
tonybaloney Sep 18, 2024
cdea255
Add extension methods
tonybaloney Sep 18, 2024
835ae1d
Add API type
tonybaloney Sep 18, 2024
3b652ea
Merge remote-tracking branch 'origin/main' into numpy_tests
tonybaloney Sep 18, 2024
f9ea39a
try LD_PRELOAD On tests
tonybaloney Sep 18, 2024
8471a7a
Use generic lib name
tonybaloney Sep 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Simplify types in API. Add bool, nint and unint conversions
  • Loading branch information
tonybaloney committed Sep 16, 2024

Verified

This commit was signed with the committer’s verified signature.
sstanculeanu Sorin Stanculeanu
commit 75c0abbdf58fe3f8f0e69324d519011be5170760
30 changes: 29 additions & 1 deletion src/CSnakes.Runtime/Python/IPyBuffer.cs
Original file line number Diff line number Diff line change
@@ -3,11 +3,28 @@
namespace CSnakes.Runtime.Python;
public interface IPyBuffer
{
/// <summary>
/// The bit length of the buffer.
/// </summary>
long Length { get; }

/// <summary>
/// The number of dimensions in the buffer. For a scalar, this is 1.
/// </summary>
int Dimensions { get; }

bool Scalar { get; }
/// <summary>
/// Indicates if the buffer is a scalar (single dimension array).
/// </summary>
bool IsScalar { get; }

/// <summary>
/// Gets the item type of the values in the buffer.
/// </summary>
/// <returns></returns>
Type GetItemType();

Span<bool> AsBoolSpan();
Span<byte> AsByteSpan();
Span<sbyte> AsSByteSpan();
Span<short> AsInt16Span();
@@ -18,7 +35,10 @@ public interface IPyBuffer
Span<ulong> AsUInt64Span();
Span<float> AsFloatSpan();
Span<double> AsDoubleSpan();
Span<nint> AsIntPtrSpan();
Span<nuint> AsUIntPtrSpan();

ReadOnlySpan<bool> AsBoolReadOnlySpan();
ReadOnlySpan<byte> AsByteReadOnlySpan();
ReadOnlySpan<sbyte> AsSByteReadOnlySpan();
ReadOnlySpan<short> AsInt16ReadOnlySpan();
@@ -29,7 +49,10 @@ public interface IPyBuffer
ReadOnlySpan<ulong> AsUInt64ReadOnlySpan();
ReadOnlySpan<float> AsFloatReadOnlySpan();
ReadOnlySpan<double> AsDoubleReadOnlySpan();
ReadOnlySpan<nint> AsIntPtrReadOnlySpan();
ReadOnlySpan<nuint> AsUIntPtrReadOnlySpan();

Span2D<bool> AsBoolSpan2D();
Span2D<byte> AsByteSpan2D();
Span2D<sbyte> AsSByteSpan2D();
Span2D<short> AsInt16Span2D();
@@ -40,7 +63,10 @@ public interface IPyBuffer
Span2D<ulong> AsUInt64Span2D();
Span2D<float> AsFloatSpan2D();
Span2D<double> AsDoubleSpan2D();
Span2D<nint> AsIntPtrSpan2D();
Span2D<nuint> AsUIntPtrSpan2D();

ReadOnlySpan2D<bool> AsBoolReadOnlySpan2D();
ReadOnlySpan2D<byte> AsByteReadOnlySpan2D();
ReadOnlySpan2D<sbyte> AsSByteReadOnlySpan2D();
ReadOnlySpan2D<short> AsInt16ReadOnlySpan2D();
@@ -51,4 +77,6 @@ public interface IPyBuffer
ReadOnlySpan2D<ulong> AsUInt64ReadOnlySpan2D();
ReadOnlySpan2D<float> AsFloatReadOnlySpan2D();
ReadOnlySpan2D<double> AsDoubleReadOnlySpan2D();
ReadOnlySpan2D<nint> AsIntPtrReadOnlySpan2D();
ReadOnlySpan2D<nuint> AsUIntPtrReadOnlySpan2D();
}
92 changes: 75 additions & 17 deletions src/CSnakes.Runtime/Python/PyBuffer.cs
Original file line number Diff line number Diff line change
@@ -70,11 +70,57 @@ public void Dispose()

public long Length => _buffer.len;

public bool Scalar => _isScalar;
public bool IsScalar => _isScalar;

public bool IsReadOnly => _buffer.@readonly == 1;

public int Dimensions => _buffer.ndim;
public int Dimensions => _buffer.ndim == 0 ? 1 : _buffer.ndim;

public Type GetItemType()
{
// The format string contains the type of the buffer, normally in the first
// position, but the first character can also be the byte order.
for (int i = 0; i < _format.Length; i++)
{
if (Enum.IsDefined(typeof(Format), (int)_format[i]))
{
var format = (Format)_format[i];
switch (format) {
case Format.Bool:
return typeof(bool);
case Format.Char:
return typeof(sbyte);
case Format.UChar:
return typeof(byte);
case Format.Short:
return typeof(short);
case Format.UShort:
return typeof(ushort);
case Format.Int:
return typeof(int);
case Format.UInt:
return typeof(uint);
case Format.Long:
return typeof(int);
case Format.ULong:
return typeof(uint);
case Format.LongLong:
return typeof(long);
case Format.ULongLong:
return typeof(ulong);
case Format.Float:
return typeof(float);
case Format.Double:
return typeof(double);
case Format.SizeT:
return typeof(nuint);
case Format.SSizeT:
return typeof(nint);
}
}
}
throw new InvalidOperationException($"Unknown format {_format}");
}

private ByteOrder GetByteOrder()
{
@@ -100,7 +146,7 @@ private void EnsureFormat(char format)

private void EnsureScalar()
{
if (!Scalar)
if (!IsScalar)
{
throw new InvalidOperationException("Buffer is not a scalar");
}
@@ -147,23 +193,26 @@ private unsafe Span<T> AsSpan<T>(Format format, Format nixFormat, bool allowRead
return new Span<T>((void*)_buffer.buf, (int)(Length / sizeof(T)));
}

public Span<bool> AsBoolSpan() => AsSpan<bool>(Format.Bool, Format.Bool);
public Span<byte> AsByteSpan() => AsSpan<byte>(Format.UChar, Format.UChar);
public Span<sbyte> AsSByteSpan() => AsSpan<sbyte>(Format.Char, Format.Char);

public Span<Int16> AsInt16Span() => AsSpan<Int16>(Format.Short, Format.Short);
public Span<UInt16> AsUInt16Span() => AsSpan<UInt16>(Format.UShort, Format.UShort);
public Span<short> AsInt16Span() => AsSpan<short>(Format.Short, Format.Short);
public Span<ushort> AsUInt16Span() => AsSpan<ushort>(Format.UShort, Format.UShort);

public Span<Int32> AsInt32Span() => AsSpan<Int32>(Format.Long, Format.Int);
public Span<int> AsInt32Span() => AsSpan<int>(Format.Long, Format.Int);

public Span<UInt32> AsUInt32Span() => AsSpan<UInt32>(Format.ULong, Format.UInt);
public Span<uint> AsUInt32Span() => AsSpan<uint>(Format.ULong, Format.UInt);

public Span<Int64> AsInt64Span() => AsSpan<Int64>(Format.LongLong, Format.Long);
public Span<long> AsInt64Span() => AsSpan<long>(Format.LongLong, Format.Long);

public Span<UInt64> AsUInt64Span() => AsSpan<UInt64>(Format.ULongLong, Format.ULong);
public Span<ulong> AsUInt64Span() => AsSpan<ulong>(Format.ULongLong, Format.ULong);

public Span<float> AsFloatSpan() => AsSpan<float>(Format.Float, Format.Float);

public Span<double> AsDoubleSpan() => AsSpan<double>(Format.Double, Format.Double);
public Span<nint> AsIntPtrSpan() => AsSpan<nint>(Format.SSizeT, Format.SSizeT);
public Span<nuint> AsUIntPtrSpan() => AsSpan<nuint>(Format.SizeT, Format.SizeT);

private unsafe ReadOnlySpan<T> AsReadOnlySpan<T>(Format format, Format nixFormat) where T : unmanaged
{
@@ -186,16 +235,19 @@ private unsafe ReadOnlySpan<T> AsReadOnlySpan<T>(Format format, Format nixFormat
return new ReadOnlySpan<T>((void*)_buffer.buf, (int)(Length / sizeof(T)));
}

public ReadOnlySpan<bool> AsBoolReadOnlySpan() => AsReadOnlySpan<bool>(Format.Bool, Format.Bool);
public ReadOnlySpan<byte> AsByteReadOnlySpan() => AsReadOnlySpan<byte>(Format.UChar, Format.UChar);
public ReadOnlySpan<sbyte> AsSByteReadOnlySpan() => AsReadOnlySpan<sbyte>(Format.Char, Format.Char);
public ReadOnlySpan<Int16> AsInt16ReadOnlySpan() => AsReadOnlySpan<Int16>(Format.Short, Format.Short);
public ReadOnlySpan<UInt16> AsUInt16ReadOnlySpan() => AsReadOnlySpan<UInt16>(Format.UShort, Format.UShort);
public ReadOnlySpan<Int32> AsInt32ReadOnlySpan() => AsReadOnlySpan<Int32>(Format.Long, Format.Int);
public ReadOnlySpan<UInt32> AsUInt32ReadOnlySpan() => AsReadOnlySpan<UInt32>(Format.ULong, Format.UInt);
public ReadOnlySpan<Int64> AsInt64ReadOnlySpan() => AsReadOnlySpan<Int64>(Format.LongLong, Format.Long);
public ReadOnlySpan<UInt64> AsUInt64ReadOnlySpan() => AsReadOnlySpan<UInt64>(Format.ULongLong, Format.ULong);
public ReadOnlySpan<short> AsInt16ReadOnlySpan() => AsReadOnlySpan<short>(Format.Short, Format.Short);
public ReadOnlySpan<ushort> AsUInt16ReadOnlySpan() => AsReadOnlySpan<ushort>(Format.UShort, Format.UShort);
public ReadOnlySpan<int> AsInt32ReadOnlySpan() => AsReadOnlySpan<int>(Format.Long, Format.Int);
public ReadOnlySpan<uint> AsUInt32ReadOnlySpan() => AsReadOnlySpan<uint>(Format.ULong, Format.UInt);
public ReadOnlySpan<long> AsInt64ReadOnlySpan() => AsReadOnlySpan <long>(Format.LongLong, Format.Long);
public ReadOnlySpan<ulong> AsUInt64ReadOnlySpan() => AsReadOnlySpan<ulong>(Format.ULongLong, Format.ULong);
public ReadOnlySpan<float> AsFloatReadOnlySpan() => AsReadOnlySpan<float>(Format.Float, Format.Float);
public ReadOnlySpan<double> AsDoubleReadOnlySpan() => AsReadOnlySpan<double>(Format.Double, Format.Double);
public ReadOnlySpan<nint> AsIntPtrReadOnlySpan() => AsReadOnlySpan<nint>(Format.SSizeT, Format.SSizeT);
public ReadOnlySpan<nuint> AsUIntPtrReadOnlySpan() => AsReadOnlySpan<nuint>(Format.SizeT, Format.SizeT);


private unsafe Span2D<T> As2DSpan<T>(Format format, Format nixFormat) where T : unmanaged
@@ -228,6 +280,7 @@ private unsafe Span2D<T> As2DSpan<T>(Format format, Format nixFormat) where T :
);
}

public Span2D<bool> AsBoolSpan2D() => As2DSpan<bool>(Format.Bool, Format.Bool);
public Span2D<byte> AsByteSpan2D() => As2DSpan<byte>(Format.UChar, Format.UChar);
public Span2D<sbyte> AsSByteSpan2D() => As2DSpan<sbyte>(Format.Char, Format.Char);
public Span2D<short> AsInt16Span2D() => As2DSpan<short>(Format.Short, Format.Short);
@@ -243,6 +296,8 @@ private unsafe Span2D<T> As2DSpan<T>(Format format, Format nixFormat) where T :
public Span2D<float> AsFloatSpan2D() => As2DSpan<float>(Format.Float, Format.Float);

public Span2D<double> AsDoubleSpan2D() => As2DSpan<double>(Format.Double, Format.Double);
public Span2D<nint> AsIntPtrSpan2D() => As2DSpan<nint>(Format.SSizeT, Format.SSizeT);
public Span2D<nuint> AsUIntPtrSpan2D() => As2DSpan<nuint>(Format.SizeT, Format.SizeT);

private unsafe ReadOnlySpan2D<T> AsReadOnly2DSpan<T>(Format format, Format nixFormat) where T : unmanaged
{
@@ -270,14 +325,17 @@ private unsafe ReadOnlySpan2D<T> AsReadOnly2DSpan<T>(Format format, Format nixFo
);
}

public ReadOnlySpan2D<bool> AsBoolReadOnlySpan2D() => AsReadOnly2DSpan<bool>(Format.Bool, Format.Bool);
public ReadOnlySpan2D<byte> AsByteReadOnlySpan2D() => AsReadOnly2DSpan<byte>(Format.UChar, Format.UChar);
public ReadOnlySpan2D<sbyte> AsSByteReadOnlySpan2D() => AsReadOnly2DSpan<sbyte>(Format.Char, Format.Char);
public ReadOnlySpan2D<Int16> AsInt16ReadOnlySpan2D() => AsReadOnly2DSpan<Int16>(Format.Short, Format.Short);
public ReadOnlySpan2D<UInt16> AsUInt16ReadOnlySpan2D() => AsReadOnly2DSpan<UInt16>(Format.UShort, Format.UShort);
public ReadOnlySpan2D<short> AsInt16ReadOnlySpan2D() => AsReadOnly2DSpan<short>(Format.Short, Format.Short);
public ReadOnlySpan2D<ushort> AsUInt16ReadOnlySpan2D() => AsReadOnly2DSpan<ushort>(Format.UShort, Format.UShort);
public ReadOnlySpan2D<int> AsInt32ReadOnlySpan2D() => AsReadOnly2DSpan<int>(Format.Long, Format.Int);
public ReadOnlySpan2D<uint> AsUInt32ReadOnlySpan2D() => AsReadOnly2DSpan<uint>(Format.ULong, Format.UInt);
public ReadOnlySpan2D<long> AsInt64ReadOnlySpan2D() => AsReadOnly2DSpan<long>(Format.LongLong, Format.Long);
public ReadOnlySpan2D<ulong> AsUInt64ReadOnlySpan2D() => AsReadOnly2DSpan<ulong>(Format.ULongLong, Format.ULong);
public ReadOnlySpan2D<float> AsFloatReadOnlySpan2D() => AsReadOnly2DSpan<float>(Format.Float, Format.Float);
public ReadOnlySpan2D<double> AsDoubleReadOnlySpan2D() => AsReadOnly2DSpan<double>(Format.Double, Format.Double);
public ReadOnlySpan2D<nint> AsIntPtrReadOnlySpan2D() => AsReadOnly2DSpan<nint>(Format.SSizeT, Format.SSizeT);
public ReadOnlySpan2D<nuint> AsUIntPtrReadOnlySpan2D() => AsReadOnly2DSpan<nuint>(Format.SizeT, Format.SizeT);
}
Loading
Loading