8000 implements buffer interface for .NET arrays of primitive types · pythonnet/pythonnet@00aed52 · GitHub
[go: up one dir, main page]
Skip to content

Commit 00aed52

Browse files
lostmsulostmsu
committed
implements buffer interface for .NET arrays of primitive types
fixes losttech/Gradient#27
1 parent 1e32d8c commit 00aed52

File tree

9 files changed

+218
-3
lines changed

9 files changed

+218
-3
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
1515
- Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355])
1616
- `PyType` - a wrapper for Python type objects, that also permits creating new heap types from `TypeSpec`
1717
- Improved exception handling:
18-
- exceptions can now be converted with codecs
19-
- `InnerException` and `__cause__` are propagated properly
18+
- exceptions can now be converted with codecs
19+
- `InnerException` and `__cause__` are propagated properly
20+
- .NET arrays implement Python buffer protocol
21+
2022

2123
### Changed
2224
- Drop support for Python 2, 3.4, and 3.5

‎src/embed_tests/TestExample.cs renamed to ‎src/embed_tests/NumPyTests.cs

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,17 @@
22
using System.Collections.Generic;
33
using NUnit.Framework;
44
using Python.Runtime;
5+
using Python.Runtime.Codecs;
56

67
namespace Python.EmbeddingTest
78
{
8-
public class TestExample
9+
public class NumPyTests
910
{
1011
[OneTimeSetUp]
1112
public void SetUp()
1213
{
1314
PythonEngine.Initialize();
15+
TupleCodec<ValueTuple>.Register();
1416
}
1517

1618
[OneTimeTearDown]
@@ -49,5 +51,23 @@ public void TestReadme()
4951

5052
Assert.AreEqual("[ 6. 10. 12.]", (a * b).ToString().Replace(" ", " "));
5153
}
54+
55+
[Test]
56+
public void MultidimensionalNumPyArray()
57+
{
58+
PyObject np;
59+
try {
60+
np = Py.Import("numpy");
61+
} catch (PythonException) {
62+
Assert.Inconclusive("Numpy or dependency not installed");
63+
return;
64+
}
65+
66+
var array = new[,] { { 1, 2 }, { 3, 4 } };
67+
var ndarray = np.InvokeMethod("asarray", array.ToPython());
68+
Assert.AreEqual((2,2), ndarray.GetAttr("shape").As<(int,int)>());
69+
Assert.AreEqual(1, ndarray[(0, 0).ToPython()].InvokeMethod("__int__").As<int>());
70+
Assert.AreEqual(array[1, 0], ndarray[(1, 0).ToPython()].InvokeMethod("__int__").As<int>());
71+
}
5272
}
5373
}

‎src/embed_tests/TestPyBuffer.cs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
using System;
12
using System.Text;
23
using NUnit.Framework;
34
using Python.Runtime;
5+
using Python.Runtime.Codecs;
46

57
namespace Python.EmbeddingTest {
68
class TestPyBuffer
@@ -9,6 +11,7 @@ class TestPyBuffer
911
public void SetUp()
1012
{
1113
PythonEngine.Initialize();
14+
TupleCodec<ValueTuple>.Register();
1215
}
1316

1417
[OneTimeTearDown]
@@ -64,5 +67,15 @@ public void TestBufferRead()
6467
}
6568
}
6669
}
70+
71+
[Test]
72+
public void ArrayHasBuffer()
73+
{
74+
var array = new[,] {{1, 2}, {3,4}};
75+
var memoryView = PythonEngine.Eval("memoryview");
76+
var mem = memoryView.Invoke(array.ToPython());
77+
Assert.AreEqual(1, mem[(0, 0).ToPython()].As<int>());
78+
Assert.AreEqual(array[1,0], mem[(1, 0).ToPython()].As<int>());
79+
}
6780
}
6881
}

‎src/runtime/Util.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ internal static class Util
1010
internal const string MinimalPythonVersionRequired =
1111
"Only Python 3.5 or newer is supported";
1212

13+
internal static bool CLongIs32Bit() => Runtime.IsWindows || Runtime.Is32Bit;
14+
1315
internal static Int64 ReadCLong(IntPtr tp, int offset)
1416
{
1517
// On Windows, a C long is always 32 bits.

‎src/runtime/arrayobject.cs

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
using System;
22
using System.Collections;
3+
using System.Collections.Generic;
4+
using System.Runtime.InteropServices;
35

46
namespace Python.Runtime
57
{
@@ -366,5 +368,165 @@ public static int sq_contains(IntPtr ob, IntPtr v)
366368

367369
return 0;
368370
}
371+
372+
#region Buffer protocol
373+
static int GetBuffer(BorrowedReference obj, out Py_buffer buffer, PyBUF flags)
374+
{
375+
buffer = default;
376+
377+
if (flags == PyBUF.SIMPLE)
378+
{
379+
Exceptions.SetError(Exceptions.BufferError, "SIMPLE not implemented");
380+
return -1;
381+
}
382+
if ((flags & PyBUF.F_CONTIGUOUS) == PyBUF.F_CONTIGUOUS)
383+
{
384+
Exceptions.SetError(Exceptions.BufferError, "only C-contiguous supported");
385+
return -1;
386+
}
387+
var self = (Array)((CLRObject)GetManagedObject(obj)).inst;
388+
Type itemType = self.GetType().GetElementType();
389+
390+
bool formatRequested = (flags & PyBUF.FORMATS) != 0;
391+
string format = GetFormat(itemType);
392+
if (formatRequested && format is null)
393+
{
394+
Exceptions.SetError(Exceptions.BufferError, "unsupported element type: " + itemType.Name);
395+
return -1;
396+
}
397+
GCHandle gcHandle;
398+
try
399+
{
400+
gcHandle = GCHandle.Alloc(self, GCHandleType.Pinned);
401+
} catch (ArgumentException ex)
402+
{
403+
Exceptions.SetError(Exceptions.BufferError, ex.Message);
404+
return -1;
405+
}
406+
407+
int itemSize = Marshal.SizeOf(itemType);
408+
IntPtr[] shape = GetShape(self);
409+
IntPtr[] strides = GetStrides(shape, itemSize);
410+
buffer = new Py_buffer
411+
{
412+
buf = gcHandle.AddrOfPinnedObject(),
413+
obj = Runtime.SelfIncRef(obj.DangerousGetAddress()),
414+
len = (IntPtr)(self.LongLength*itemSize),
415+
itemsize = (IntPtr)itemSize,
416+
_readonly = false,
417+
ndim = self.Rank,
418+
format = format,
419+
shape = ToUnmanaged(shape),
420+
strides = (flags & PyBUF.STRIDES) == PyBUF.STRIDES ? ToUnmanaged(strides) : IntPtr.Zero,
421+
suboffsets = IntPtr.Zero,
422+
_internal = (IntPtr)gcHandle,
423+
};
424+
425+
return 0;
426+
}
427+
static void ReleaseBuffer(BorrowedReference obj, ref Py_buffer buffer)
428+
{
429+
if (buffer._internal == IntPtr.Zero) return;
430+
431+
UnmanagedFree(ref buffer.shape);
432+
UnmanagedFree(ref buffer.strides);
433+
UnmanagedFree(ref buffer.suboffsets);
434+
435+
var gcHandle = (GCHandle)buffer._internal;
436+
gcHandle.Free();
437+
buffer._internal = IntPtr.Zero;
438+
}
439+
440+
static IntPtr[] GetStrides(IntPtr[] shape, long itemSize)
441+
{
442+
var result = new IntPtr[shape.Length];
443+
result[shape.Length - 1] = new IntPtr(itemSize);
444+
for (int dim = shape.Length - 2; dim >= 0; dim--)
445+
{
446+
itemSize *= shape[dim + 1].ToInt64();
447+
result[dim] = new IntPtr(itemSize);
448+
}
449+
return result;
450+
}
451+
static IntPtr[] GetShape(Array array)
452+
{
453+
var result = new IntPtr[array.Rank];
454+
for (int i = 0; i < result.Length; i++)
455+
result[i] = (IntPtr)array.GetLongLength(i);
456+
return result;
457+
}
458+
459+
static void UnmanagedFree(ref IntPtr address)
460+
{
461+
if (address == IntPtr.Zero) return;
462+
463+
Marshal.FreeHGlobal(address);
464+
address = IntPtr.Zero;
465+
}
466+
static unsafe IntPtr ToUnmanaged<T>(T[] array) where T : unmanaged
467+
{
468+
IntPtr result = Marshal.AllocHGlobal(checked(Marshal.SizeOf(typeof(T)) * array.Length));
469+
fixed (T* ptr = array)
470+
{
471+
var @out = (T*)result;
472+
for (int i = 0; i < array.Length; i++)
473+
@out[i] = ptr[i];
474+
}
475+
return result;
476+
}
477+
478+
static readonly Dictionary<Type, string> ItemFormats = new Dictionary<Type, string>
479+
{
480+
[typeof(byte)] = "B",
481+
[typeof(sbyte)] = "b",
482+
483+
[typeof(bool)] = "?",
484+
485+
[typeof(short)] = "h",
486+
[typeof(ushort)] = "H",
487+
[typeof(int)] = Util.CLongIs32Bit() ? "l" : "i",
488+
[typeof(uint)] = Util.CLongIs32Bit() ? "L" : "I",
489+
[typeof(long)] = Util.CLongIs32Bit() ? "q" : "l",
490+
[typeof(ulong)] = Util.CLongIs32Bit() ? "Q": "L",
491+
492+
[typeof(IntPtr)] = "n",
493+
[typeof(UIntPtr)] = "N",
494+
495+
// TODO: half = "e"
496+
[typeof(float)] = "f",
497+
[typeof(double)] = "d",
498+
};
499+
500+
static string GetFormat(Type elementType)
501+
=> ItemFormats.TryGetValue(elementType, out string result) ? result : null;
502+
503+
static readonly GetBufferProc getBufferProc = GetBuffer;
504+
static readonly ReleaseBufferProc releaseBufferProc = ReleaseBuffer;
505+
static readonly IntPtr BufferProcsAddress = AllocateBufferProcs();
506+
static IntPtr AllocateBufferProcs()
507+
{
508+
var procs = new PyBufferProcs
509+
{
510+
Get = Marshal.GetFunctionPointerForDelegate(getBufferProc),
511+
Release = Marshal.GetFunctionPointerForDelegate(releaseBufferProc),
512+
};
513+
IntPtr result = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PyBufferProcs)));
514+
Marshal.StructureToPtr(procs, result, fDeleteOld: false);
515+
return result;
516+
}
517+
#endregion
518+
519+
/// <summary>
520+
/// <see cref="TypeManager.InitializeSlots(IntPtr, Type, SlotsHolder)"/>
521+
/// </summary>
522+
public static void InitializeSlots(IntPtr type, ISet<string> initialized, SlotsHolder slotsHolder)
523+
{
524+
if (initialized.Add(nameof(TypeOffset.tp_as_buffer)))
525+
{
526+
// TODO: only for unmanaged arrays
527+
int offset = TypeOffset.GetSlotOffset(nameof(TypeOffset.tp_as_buffer));
528+
Marshal.WriteIntPtr(type, offset, BufferProcsAddress);
529+
}
530+
}
369531
}
370532
}

‎src/runtime/bufferinterface.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,4 +103,15 @@ public enum PyBUF
103103
/// </summary>
104104
FULL_RO = (INDIRECT | FORMATS),
105105
}
106+
107+
internal struct PyBufferProcs
108+
{
109+
public IntPtr Get;
110+
public IntPtr Release;
111+
}
112+
113+
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
114+
delegate int GetBufferProc(BorrowedReference obj, out Py_buffer buffer, PyBUF flags);
115+
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
116+
delegate void ReleaseBufferProc(BorrowedReference obj, ref Py_buffer buffer);
106117
}

‎src/runtime/exceptions.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,7 @@ public static variables on the Exceptions class filled in from
413413

414414
public static IntPtr AssertionError;
415415
public static IntPtr AttributeError;
416+
public static IntPtr BufferError;
416417
public static IntPtr EOFError;
417418
public static IntPtr FloatingPointError;
418419
public static IntPtr EnvironmentError;

‎src/runtime/native/TypeOffset.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties)
159159
"GetClrType",
160160
"getPreload",
161161
"Initialize",
162+
"InitializeSlots",
162163
"ListAssemblies",
163164
"_load_clr_module",
164165
"Release",

‎src/runtime/typemanager.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -745,6 +745,9 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo
745745
seen.Add(name);
746746
}
747747

748+
var initSlot = impl.GetMethod("InitializeSlots", BindingFlags.Static | BindingFlags.Public);
749+
initSlot?.Invoke(null, parameters: new object[] { type, seen, slotsHolder });
750+
748751
impl = impl.BaseType;
749752
}
750753

0 commit comments

Comments
 (0)
0