10000 implements buffer interface for .NET arrays of primitive types (#1511) · pythonnet/pythonnet@ee0ab7f · GitHub
[go: up one dir, main page]
Skip to content

Commit ee0ab7f

Browse files
lostmsulostmsu
authored
implements buffer interface for .NET arrays of primitive types (#1511)
fixes losttech/Gradient#27
1 parent 1e32d8c commit ee0ab7f

File tree

10 files changed

+291
-55
lines changed

10 files changed

+291
-55
lines changed

‎.github/workflows/main.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ jobs:
4141
- name: Install dependencies
4242
run: |
4343
pip install --upgrade -r requirements.txt
44+
pip install numpy # for tests
4445
4546
- name: Build and Install
4647
run: |

‎CHANGELOG.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
1515
- Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355])
1616
- `PyType` - a wrapper for Python type objects, that also permits creating new heap types from `TypeSpec`
1717
- Improved exception handling:
18-
- exceptions can now be converted with codecs
19-
- `InnerException` and `__cause__` are propagated properly
18+
- exceptions can now be converted with codecs
19+
- `InnerException` and `__cause__` are propagated properly
20+
- .NET arrays implement Python buffer protocol
21+
2022

2123
### Changed
2224
- Drop support for Python 2, 3.4, and 3.5

‎src/embed_tests/NumPyTests.cs

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using NUnit.Framework;
4+
using Python.Runtime;
5+
using Python.Runtime.Codecs;
6+
7+
namespace Python.EmbeddingTest
8+
{
9+
public class NumPyTests
10+
{
11+
[OneTimeSetUp]
12+
public void SetUp()
13+
{
14+
PythonEngine.Initialize();
15+
TupleCodec<ValueTuple>.Register();
16+
}
17+
18+
[OneTimeTearDown]
19+
public void Dispose()
20+
{
21+
PythonEngine.Shutdown();
22+
}
23+
24+
[Test]
25+
public void TestReadme()
26+
{
27+
dynamic np;
28+
try
29+
{
30+
np = Py.Import("numpy");
31+
}
32+
catch (PythonException)
33+
{
34+
Assert.Inconclusive("Numpy or dependency not installed");
35+
return;
36+
}
37+
38+
Assert.AreEqual("1.0", np.cos(np.pi * 2).ToString 10000 ());
39+
40+
dynamic sin = np.sin;
41+
StringAssert.StartsWith("-0.95892", sin(5).ToString());
42+
43+
double c = np.cos(5) + sin(5);
44+
Assert.AreEqual(-0.675262, c, 0.01);
45+
46+
dynamic a = np.array(new List<float> { 1, 2, 3 });
47+
Assert.AreEqual("float64", a.dtype.ToString());
48+
49+
dynamic b = np.array(new List<float> { 6, 5, 4 }, Py.kw("dtype", np.int32));
50+
Assert.AreEqual("int32", b.dtype.ToString());
51+
52+
Assert.AreEqual("[ 6. 10. 12.]", (a * b).ToString().Replace(" ", " "));
53+
}
54+
55+
[Test]
56+
public void MultidimensionalNumPyArray()
57+
{
58+
PyObject np;
59+
try {
60+
np = Py.Import("numpy");
61+
} catch (PythonException) {
62+
Assert.Inconclusive("Numpy or dependency not installed");
63+
return;
64+
}
65+
66+
var array = new[,] { { 1, 2 }, { 3, 4 } };
67+
var ndarray = np.InvokeMethod("asarray", array.ToPython());
68+
Assert.AreEqual((2,2), ndarray.GetAttr("shape").As<(int,int)>());
69+
Assert.AreEqual(1, ndarray[(0, 0).ToPython()].InvokeMethod("__int__").As<int>());
70+
Assert.AreEqual(array[1, 0], ndarray[(1, 0).ToPython()].InvokeMethod("__int__").As<int>());
71+
}
72+
73+
[Test]
74+
public void Int64Array()
75+
{
76+
PyObject np;
77+
try
78+
{
79+
np = Py.Import("numpy");
80+
}
81+
catch (PythonException)
82+
{
83+
Assert.Inconclusive("Numpy or dependency not installed");
84+
return;
85+
}
86+
87+
var array = new long[,] { { 1, 2 }, { 3, 4 } };
88+
var ndarray = np.InvokeMethod("asarray", array.ToPython());
89+
Assert.AreEqual((2, 2), ndarray.GetAttr("shape").As<(int, int)>());
90+
Assert.AreEqual(1, ndarray[(0, 0).ToPython()].InvokeMethod("__int__").As<long>());
91+
Assert.AreEqual(array[1, 0], ndarray[(1, 0).ToPython()].InvokeMethod("__int__").As<long>());
92+
}
93+
}
94+
}

‎src/embed_tests/TestExample.cs

Lines changed: 0 additions & 53 deletions
This file was deleted.

‎src/embed_tests/TestPyBuffer.cs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
using System;
12
using System.Text;
23
using NUnit.Framework;
34
using Python.Runtime;
5+
using Python.Runtime.Codecs;
46

57
namespace Python.EmbeddingTest {
68
class TestPyBuffer
@@ -9,6 +11,7 @@ class TestPyBuffer
911
public void SetUp()
1012
{
1113
PythonEngine.Initialize();
14+
TupleCodec<ValueTuple>.Register();
1215
}
1316

1417
[OneTimeTearDown]
@@ -64,5 +67,15 @@ public void TestBufferRead()
6467
}
6568
}
6669
}
70+
71+
[Test]
72+
public void ArrayHasBuffer()
73+
{
74+
var array = new[,] {{1, 2}, {3,4}};
75+
var memoryView = PythonEngine.Eval("memoryview");
76+
var mem = memoryView.Invoke(array.ToPython());
77+
Assert.AreEqual(1, mem[(0, 0).ToPython()].As<int>());
78+
Assert.AreEqual(array[1,0], mem[(1, 0).ToPython()].As<int>());
79+
}
6780
}
6881
}

‎src/runtime/arrayobject.cs

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
using System;
22
using System.Collections;
3+
using System.Collections.Generic;
4+
using System.Runtime.InteropServices;
35

46
namespace Python.Runtime
57
{
@@ -366,5 +368,166 @@ public static int sq_contains(IntPtr ob, IntPtr v)
366368

367369
return 0;
368370
}
371+
372+
#region Buffer protocol
373+
static int GetBuffer(BorrowedReference obj, out Py_buffer buffer, PyBUF flags)
374+
{
375+
buffer = default;
376+
377+
if (flags == PyBUF.SIMPLE)
378+
{
379+
Exceptions.SetError(Exceptions.BufferError, "SIMPLE not implemented");
380+
return -1;
381+
}
382+
if ((flags & PyBUF.F_CONTIGUOUS) == PyBUF.F_CONTIGUOUS)
383+
{
384+
Exceptions.SetError(Exceptions.BufferError, "only C-contiguous supported");
385+
return -1;
386+
}
387+
var self = (Array)((CLRObject)GetManagedObject(obj)).inst;
388+
Type itemType = self.GetType().GetElementType();
389+
390+
bool formatRequested = (flags & PyBUF.FORMATS) != 0;
391+
string format = GetFormat(itemType);
392+
if (formatRequested && format is null)
393+
{
394+
Exceptions.SetError(Exceptions.BufferError, "unsupported element type: " + itemType.Name);
395+
return -1;
396+
}
397+
GCHandle gcHandle;
398+
try
399+
{
400+
gcHandle = GCHandle.Alloc(self, GCHandleType.Pinned);
401+
} catch (ArgumentException ex)
402+
{
403+
Exceptions.SetError(Exceptions.BufferError, ex.Message);
404+
return -1;
405+
}
406+
407+
int itemSize = Marshal.SizeOf(itemType);
408+
IntPtr[] shape = GetShape(self);
409+
IntPtr[] strides = GetStrides(shape, itemSize);
410+
buffer = new Py_buffer
411+
{
412+
buf = gcHandle.AddrOfPinnedObject(),
413+
obj = Runtime.SelfIncRef(obj.DangerousGetAddress()),
414+
len = (IntPtr)(self.LongLength*itemSize),
415+
itemsize = (IntPtr)itemSize,
416+
_readonly = false,
417+
ndim = self.Rank,
418+
format = format,
419+
shape = ToUnmanaged(shape),
420+
strides = (flags & PyBUF.STRIDES) == PyBUF.STRIDES ? ToUnmanaged(strides) : IntPtr.Zero,
421+
suboffsets = IntPtr.Zero,
422+
_internal = (IntPtr)gcHandle,
423+
};
424+
425+
return 0;
426+
}
427+
static void ReleaseBuffer(BorrowedReference obj, ref Py_buffer buffer)
428+
{
429+
if (buffer._internal == IntPtr.Zero) return;
430+
431+
UnmanagedFree(ref buffer.shape);
432+
UnmanagedFree(ref buffer.strides);
433+
UnmanagedFree(ref buffer.suboffsets);
434+
435+
var gcHandle = (GCHandle)buffer._internal;
436+
gcHandle.Free();
437+
buffer._internal = IntPtr.Zero;
438+
}
439+
440+
static IntPtr[] GetStrides(IntPtr[] shape, long itemSize)
441+
{
442+
var result = new IntPtr[shape.Length];
443+
result[shape.Length - 1] = new IntPtr(itemSize);
444+
for (int dim = shape.Length - 2; dim >= 0; dim--)
445+
{
446+
itemSize *= shape[dim + 1].ToInt64();
447+
result[dim] = new IntPtr(itemSize);
448+
}
449+
return result;
450+
}
451+
static IntPtr[] GetShape(Array array)
452+
{
453+
var result = new IntPtr[array.Rank];
454+
for (int i = 0; i < result.Length; i++)
455+
result[i] = (IntPtr)array.GetLongLength(i);
456+
return result;
457+
}
458+
459+
static void UnmanagedFree(ref IntPtr address)
460+
{
461+
if (address == IntPtr.Zero) return;
462+
463+
Marshal.FreeHGlobal(address);
464+
address = IntPtr.Zero;
465+
}
466+
static unsafe IntPtr ToUnmanaged<T>(T[] array) where T : unmanaged
467+
{
468+
IntPtr result = Marshal.AllocHGlobal(checked(Marshal.SizeOf(typeof(T)) * array.Length));
469+
fixed (T* ptr = array)
470+
{
471+
var @out = (T*)result;
472+
for (int i = 0; i < array.Length; i++)
473+
@out[i] = ptr[i];
474+
}
475+
return result;
476+
}
477+
478+
static readonly Dictionary<Type, string> ItemFormats = new Dictionary<Type, string>
479+
{
480+
[typeof(byte)] = "B",
481+
[typeof(sbyte)] = "b",
482+
483+
[typeof(bool)] = "?",
484+
485+
[typeof(short)] = "h",
486+
[typeof(ushort)] = "H",
487+
// see https://github.com/pybind/pybind11/issues/1908#issuecomment-658358767
488+
[typeof(int)] = "i",
489+
[typeof(uint)] = "I",
490+
[typeof(long)] = "q",
491+
[typeof(ulong)] = "Q",
492+
493+
[typeof(IntPtr)] = "n",
494+
[typeof(UIntPtr)] = "N",
495+
496+
// TODO: half = "e"
497+
[typeof(float)] = "f",
498+
[typeof(double)] = "d",
499+
};
500+
501+
static string GetFormat(Type elementType)
502+
=> ItemFormats.TryGetValue(elementType, out string result) ? result : null;
503+
504+
static readonly GetBufferProc getBufferProc = GetBuffer;
505+
static readonly ReleaseBufferProc releaseBufferProc = ReleaseBuffer;
506+
static readonly IntPtr BufferProcsAddress = AllocateBufferProcs();
507+
static IntPtr AllocateBufferProcs()
508+
{
509+
var procs = new PyBufferProcs
510+
{
511+
Get = Marshal.GetFunctionPointerForDelegate(getBufferProc),
512+
Release = Marshal.GetFunctionPointerForDelegate(releaseBufferProc),
513+
};
514+
IntPtr result = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PyBufferProcs)));
515+
Marshal.StructureToPtr(procs, result, fDeleteOld: false);
516+
return result;
517+
}
518+
#endregion
519+
520+
/// <summary>
521+
/// <see cref="TypeManager.InitializeSlots(IntPtr, Type, SlotsHolder)"/>
522+
/// </summary>
523+
public static void InitializeSlots(IntPtr type, ISet<string> initialized, SlotsHolder slotsHolder)
524+
{
525+
if (initialized.Add(nameof(TypeOffset.tp_as_buffer)))
526+
{
527+
// TODO: only for unmanaged arrays
528+
int offset = TypeOffset.GetSlotOffset(nameof(TypeOffset.tp_as_buffer));
529+
Marshal.WriteIntPtr(type, offset, BufferProcsAddress);
530+
}
531+
}
369532
}
370533
}

‎src/runtime/bufferinterface.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,4 +103,15 @@ public enum PyBUF
103103
/// </summary>
104104
FULL_RO = (INDIRECT | FORMATS),
105105
}
106+
107+
internal struct PyBufferProcs
108+
{
109+
public IntPtr Get;
110+
public IntPtr Release;
111+
}
112+
113+
[UnmanagedFunctionPointer(CallingConvention.Cdecl)] 4B32
114+
delegate int GetBufferProc(BorrowedReference obj, out Py_buffer buffer, PyBUF flags);
115+
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
116+
delegate void ReleaseBufferProc(BorrowedReference obj, ref Py_buffer buffer);
106117
}

0 commit comments

Comments
 (0)
0