From ac997a962c632ac73ff8cbb4f21fd8e12285d54c Mon Sep 17 00:00:00 2001
From: SnGmng <38666407+SnGmng@users.noreply.github.com>
Date: Sun, 27 Oct 2019 20:35:42 +0100
Subject: [PATCH 1/8] Add Python buffer api support
---
AUTHORS.md | 1 +
CHANGELOG.md | 1 +
src/embed_tests/Python.EmbeddingTest.csproj | 1 +
src/embed_tests/TestPyBuffer.cs | 72 ++++++
src/runtime/Python.Runtime.csproj | 2 +
src/runtime/bufferinterface.cs | 106 +++++++++
src/runtime/pybuffer.cs | 244 ++++++++++++++++++++
src/runtime/pyobject.cs | 14 ++
src/runtime/runtime.cs | 33 +++
9 files changed, 474 insertions(+)
create mode 100644 src/embed_tests/TestPyBuffer.cs
create mode 100644 src/runtime/bufferinterface.cs
create mode 100644 src/runtime/pybuffer.cs
diff --git a/AUTHORS.md b/AUTHORS.md
index 85812a47d..3b9d5534b 100644
--- a/AUTHORS.md
+++ b/AUTHORS.md
@@ -71,6 +71,7 @@
- ([@OneBlue](https://github.com/OneBlue))
- ([@rico-chet](https://github.com/rico-chet))
- ([@rmadsen-ks](https://github.com/rmadsen-ks))
+- ([@SnGmng](https://github.com/SnGmng))
- ([@stonebig](https://github.com/stonebig))
- ([@testrunner123](https://github.com/testrunner123))
- ([@DanBarzilian](https://github.com/DanBarzilian))
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3e82f40e3..b9266b487 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,6 +35,7 @@ This version improves performance on benchmarks significantly compared to 2.3.
- Support for Python 3.8
- Codecs as the designated way to handle automatic conversions between
.NET and Python types
+- Added Python 3 buffer api support and PyBuffer interface for fast byte and numpy array read/write ([#980][p980])
### Changed
diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj
index 5dee66e64..eff226dd5 100644
--- a/src/embed_tests/Python.EmbeddingTest.csproj
+++ b/src/embed_tests/Python.EmbeddingTest.csproj
@@ -97,6 +97,7 @@
+
diff --git a/src/embed_tests/TestPyBuffer.cs b/src/embed_tests/TestPyBuffer.cs
new file mode 100644
index 000000000..3919c4dfb
--- /dev/null
+++ b/src/embed_tests/TestPyBuffer.cs
@@ -0,0 +1,72 @@
+using System.Text;
+using NUnit.Framework;
+using Python.Runtime;
+
+namespace Python.EmbeddingTest {
+ class TestPyBuffer
+ {
+ [OneTimeSetUp]
+ public void SetUp()
+ {
+ PythonEngine.Initialize();
+ }
+
+ [OneTimeTearDown]
+ public void Dispose()
+ {
+ PythonEngine.Shutdown();
+ }
+
+ [Test]
+ public void TestBufferWrite()
+ {
+ if (Runtime.Runtime.pyversionnumber < 35) return;
+
+ string bufferTestString = "hello world! !$%&/()=?";
+
+ using (Py.GIL())
+ {
+ using (var scope = Py.CreateScope())
+ {
+ scope.Exec($"arr = bytearray({bufferTestString.Length})");
+ PyObject pythonArray = scope.Get("arr");
+ byte[] managedArray = new UTF8Encoding().GetBytes(bufferTestString);
+
+ using (PyBuffer buf = pythonArray.GetBuffer())
+ {
+ buf.Write(managedArray, 0, managedArray.Length);
+ }
+
+ string result = scope.Eval("arr.decode('utf-8')").ToString();
+ Assert.IsTrue(result == bufferTestString);
+ }
+ }
+ }
+
+ [Test]
+ public void TestBufferRead()
+ {
+ if (Runtime.Runtime.pyversionnumber < 35) return;
+
+ string bufferTestString = "hello world! !$%&/()=?";
+
+ using (Py.GIL())
+ {
+ using (var scope = Py.CreateScope())
+ {
+ scope.Exec($"arr = b'{bufferTestString}'");
+ PyObject pythonArray = scope.Get("arr");
+ byte[] managedArray = new byte[bufferTestString.Length];
+
+ using (PyBuffer buf = pythonArray.GetBuffer())
+ {
+ buf.Read(managedArray, 0, managedArray.Length);
+ }
+
+ string result = new UTF8Encoding().GetString(managedArray);
+ Assert.IsTrue(result == bufferTestString);
+ }
+ }
+ }
+ }
+}
diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
index 1fd1d991e..87e5ac54c 100644
--- a/src/runtime/Python.Runtime.csproj
+++ b/src/runtime/Python.Runtime.csproj
@@ -89,6 +89,7 @@
+
@@ -130,6 +131,7 @@
+
diff --git a/src/runtime/bufferinterface.cs b/src/runtime/bufferinterface.cs
new file mode 100644
index 000000000..0c0ac2140
--- /dev/null
+++ b/src/runtime/bufferinterface.cs
@@ -0,0 +1,106 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace Python.Runtime
+{
+ /* buffer interface */
+ [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)]
+ internal struct Py_buffer {
+ public IntPtr buf;
+ public IntPtr obj; /* owned reference */
+ [MarshalAs(UnmanagedType.SysInt)]
+ public IntPtr len;
+ [MarshalAs(UnmanagedType.SysInt)]
+ public IntPtr itemsize; /* This is Py_ssize_t so it can be
+ pointed to by strides in simple case.*/
+ [MarshalAs(UnmanagedType.Bool)]
+ public bool _readonly;
+ public int ndim;
+ [MarshalAs(UnmanagedType.LPStr)]
+ public string format;
+ public IntPtr shape;
+ public IntPtr strides;
+ public IntPtr suboffsets;
+ public IntPtr _internal;
+ }
+
+ public enum BufferOrderStyle
+ {
+ C,
+ Fortran,
+ EitherOne,
+ }
+
+ /* Flags for getting buffers */
+ public enum PyBUF
+ {
+ ///
+ /// Simple buffer without shape strides and suboffsets
+ ///
+ SIMPLE = 0,
+ ///
+ /// Controls the field. If set, the exporter MUST provide a writable buffer or else report failure. Otherwise, the exporter MAY provide either a read-only or writable buffer, but the choice MUST be consistent for all consumers.
+ ///
+ WRITABLE = 0x0001,
+ ///
+ /// Controls the field. If set, this field MUST be filled in correctly. Otherwise, this field MUST be NULL.
+ ///
+ FORMATS = 0x0004,
+ ///
+ /// N-Dimensional buffer with shape
+ ///
+ ND = 0x0008,
+ ///
+ /// Buffer with strides and shape
+ ///
+ STRIDES = (0x0010 | ND),
+ ///
+ /// C-Contigous buffer with strides and shape
+ ///
+ C_CONTIGUOUS = (0x0020 | STRIDES),
+ ///
+ /// F-Contigous buffer with strides and shape
+ ///
+ F_CONTIGUOUS = (0x0040 | STRIDES),
+ ///
+ /// C or Fortran contigous buffer with strides and shape
+ ///
+ ANY_CONTIGUOUS = (0x0080 | STRIDES),
+ ///
+ /// Buffer with suboffsets (if needed)
+ ///
+ INDIRECT = (0x0100 | STRIDES),
+ ///
+ /// Writable C-Contigous buffer with shape
+ ///
+ CONTIG = (ND | WRITABLE),
+ ///
+ /// Readonly C-Contigous buffer with shape
+ ///
+ CONTIG_RO = (ND),
+ ///
+ /// Writable buffer with shape and strides
+ ///
+ STRIDED = (STRIDES | WRITABLE),
+ ///
+ /// Readonly buffer with shape and strides
+ ///
+ STRIDED_RO = (STRIDES),
+ ///
+ /// Writable buffer with shape, strides and format
+ ///
+ RECORDS = (STRIDES | WRITABLE | FORMATS),
+ ///
+ /// Readonly buffer with shape, strides and format
+ ///
+ RECORDS_RO = (STRIDES | FORMATS),
+ ///
+ /// Writable indirect buffer with shape, strides, format and suboffsets (if needed)
+ ///
+ FULL = (INDIRECT | WRITABLE | FORMATS),
+ ///
+ /// Readonly indirect buffer with shape, strides, format and suboffsets (if needed)
+ ///
+ FULL_RO = (INDIRECT | FORMATS),
+ }
+}
diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs
new file mode 100644
index 000000000..f1d853c75
--- /dev/null
+++ b/src/runtime/pybuffer.cs
@@ -0,0 +1,244 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace Python.Runtime
+{
+ public sealed class PyBuffer : IDisposable
+ {
+ private PyObject _exporter;
+ private Py_buffer _view;
+
+ unsafe internal PyBuffer(PyObject exporter, PyBUF flags)
+ {
+ _view = new Py_buffer();
+
+ if (Runtime.PyObject_GetBuffer(exporter.Handle, ref _view, (int)flags) < 0)
+ {
+ throw new PythonException();
+ }
+
+ _exporter = exporter;
+
+ var intPtrBuf = new IntPtr[_view.ndim];
+ if (_view.shape != IntPtr.Zero)
+ {
+ Marshal.Copy(_view.shape, intPtrBuf, 0, (int)_view.len * sizeof(IntPtr));
+ Shape = intPtrBuf.Select(x => (long)x).ToArray();
+ }
+
+ if (_view.strides != IntPtr.Zero) {
+ Marshal.Copy(_view.strides, intPtrBuf, 0, (int)_view.len * sizeof(IntPtr));
+ Strides = intPtrBuf.Select(x => (long)x).ToArray();
+ }
+
+ if (_view.suboffsets != IntPtr.Zero) {
+ Marshal.Copy(_view.suboffsets, intPtrBuf, 0, (int)_view.len * sizeof(IntPtr));
+ SubOffsets = intPtrBuf.Select(x => (long)x).ToArray();
+ }
+ }
+
+ public PyObject Object => _exporter;
+ public long Length => (long)_view.len;
+ public long ItemSize => (long)_view.itemsize;
+ public int Dimensions => _view.ndim;
+ public bool ReadOnly => _view._readonly;
+ public IntPtr Buffer => _view.buf;
+ public string Format => _view.format;
+
+ ///
+ /// An array of length indicating the shape of the memory as an n-dimensional array.
+ ///
+ public long[] Shape { get; private set; }
+
+ ///
+ /// An array of length giving the number of bytes to skip to get to a new element in each dimension.
+ /// Will be null except when PyBUF_STRIDES or PyBUF_INDIRECT flags in GetBuffer/>.
+ ///
+ public long[] Strides { get; private set; }
+
+ ///
+ /// An array of Py_ssize_t of length ndim. If suboffsets[n] >= 0,
+ /// the values stored along the nth dimension are pointers and the suboffset value dictates how many bytes to add to each pointer after de-referencing.
+ /// A suboffset value that is negative indicates that no de-referencing should occur (striding in a contiguous memory block).
+ ///
+ public long[] SubOffsets { get; private set; }
+
+ private static char OrderStyleToChar(BufferOrderStyle order, bool eitherOneValid)
+ {
+ char style = 'C';
+ if (order == BufferOrderStyle.C)
+ style = 'C';
+ else if (order == BufferOrderStyle.Fortran)
+ style = 'F';
+ else if (order == BufferOrderStyle.EitherOne)
+ {
+ if (eitherOneValid) style = 'A';
+ else throw new ArgumentException("BufferOrderStyle can not be EitherOne and has to be C or Fortran");
+ }
+ return style;
+ }
+
+ ///
+ /// Return the implied itemsize from format. On error, raise an exception and return -1.
+ /// New in version 3.9.
+ ///
+ public static long SizeFromFormat(string format)
+ {
+ if (Runtime.pyversionnumber < 39)
+ throw new NotSupportedException("SizeFromFormat requires at least Python 3.9");
+ return (long)Runtime.PyBuffer_SizeFromFormat(format);
+ }
+
+ ///
+ /// Returns true if the memory defined by the view is C-style (order is 'C') or Fortran-style (order is 'F') contiguous or either one (order is 'A'). Returns false otherwise.
+ ///
+ /// C-style (order is 'C') or Fortran-style (order is 'F') contiguous or either one (order is 'A')
+ public bool IsContiguous(BufferOrderStyle order)
+ {
+ if (disposedValue)
+ throw new ObjectDisposedException(nameof(PyBuffer));
+ return Convert.ToBoolean(Runtime.PyBuffer_IsContiguous(ref _view, OrderStyleToChar(order, true)));
+ }
+
+ ///
+ /// Get the memory area pointed to by the indices inside the given view. indices must point to an array of view->ndim indices.
+ ///
+ public IntPtr GetPointer(long[] indices)
+ {
+ if (disposedValue)
+ throw new ObjectDisposedException(nameof(PyBuffer));
+ if (Runtime.pyversionnumber < 37)
+ throw new NotSupportedException("GetPointer requires at least Python 3.7");
+ return Runtime.PyBuffer_GetPointer(ref _view, indices.Select(x => (IntPtr)x).ToArray());
+ }
+
+ ///
+ /// Copy contiguous len bytes from buf to view. fort can be 'C' or 'F' (for C-style or Fortran-style ordering).
+ ///
+ public void FromContiguous(IntPtr buf, long len, BufferOrderStyle fort)
+ {
+ if (disposedValue)
+ throw new ObjectDisposedException(nameof(PyBuffer));
+ if (Runtime.pyversionnumber < 37)
+ throw new NotSupportedException("FromContiguous requires at least Python 3.7");
+
+ if (Runtime.PyBuffer_FromContiguous(ref _view, buf, (IntPtr)len, OrderStyleToChar(fort, false)) < 0)
+ throw new PythonException();
+ }
+
+ ///
+ /// Copy len bytes from view to its contiguous representation in buf. order can be 'C' or 'F' or 'A' (for C-style or Fortran-style ordering or either one). 0 is returned on success, -1 on error.
+ ///
+ /// order can be 'C' or 'F' or 'A' (for C-style or Fortran-style ordering or either one).
+ /// Buffer to copy to
+ public void ToContiguous(IntPtr buf, BufferOrderStyle order)
+ {
+ if (disposedValue)
+ throw new ObjectDisposedException(nameof(PyBuffer));
+ if (Runtime.pyversionnumber < 36)
+ throw new NotSupportedException("ToContiguous requires at least Python 3.6");
+
+ if (Runtime.PyBuffer_ToContiguous(buf, ref _view, _view.len, OrderStyleToChar(order, true)) < 0)
+ throw new PythonException();
+ }
+
+ ///
+ /// Fill the strides array with byte-strides of a contiguous (C-style if order is 'C' or Fortran-style if order is 'F') array of the given shape with the given number of bytes per element.
+ ///
+ public static void FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, BufferOrderStyle order)
+ {
+ Runtime.PyBuffer_FillContiguousStrides(ndims, shape, strides, itemsize, OrderStyleToChar(order, false));
+ }
+
+ ///
+ /// FillInfo Method
+ ///
+ ///
+ /// Handle buffer requests for an exporter that wants to expose buf of size len with writability set according to readonly. buf is interpreted as a sequence of unsigned bytes.
+ /// The flags argument indicates the request type. This function always fills in view as specified by flags, unless buf has been designated as read-only and PyBUF_WRITABLE is set in flags.
+ /// On success, set view->obj to a new reference to exporter and return 0. Otherwise, raise PyExc_BufferError, set view->obj to NULL and return -1;
+ /// If this function is used as part of a getbufferproc, exporter MUST be set to the exporting object and flags must be passed unmodified.Otherwise, exporter MUST be NULL.
+ ///
+ /// On success, set view->obj to a new reference to exporter and return 0. Otherwise, raise PyExc_BufferError, set view->obj to NULL and return -1;
+ public void FillInfo(IntPtr exporter, IntPtr buf, long len, bool _readonly, int flags)
+ {
+ if (disposedValue)
+ throw new ObjectDisposedException(nameof(PyBuffer));
+ if (Runtime.PyBuffer_FillInfo(ref _view, exporter, buf, (IntPtr)len, Convert.ToInt32(_readonly), flags) < 0)
+ throw new PythonException();
+ }
+
+ ///
+ /// Writes a managed byte array into the buffer of a python object. This can be used to pass data like images from managed to python.
+ ///
+ public void Write(byte[] buffer, int offset, int count)
+ {
+ if (disposedValue)
+ throw new ObjectDisposedException(nameof(PyBuffer));
+ if (ReadOnly)
+ throw new InvalidOperationException("Buffer is read-only");
+ if ((long)_view.len > int.MaxValue)
+ throw new NotSupportedException("Python buffers bigger than int.MaxValue are not supported.");
+ if (count > buffer.Length)
+ throw new ArgumentOutOfRangeException("count", "Count is bigger than the buffer.");
+ if (count > (int)_view.len)
+ throw new ArgumentOutOfRangeException("count", "Count is bigger than the python buffer.");
+ if (_view.ndim != 1)
+ throw new NotSupportedException("Multidimensional arrays, scalars and objects without a buffer are not supported.");
+
+ Marshal.Copy(buffer, offset, _view.buf, count);
+ }
+
+ ///
+ /// Reads the buffer of a python object into a managed byte array. This can be used to pass data like images from python to managed.
+ ///
+ public int Read(byte[] buffer, int offset, int count) {
+ if (disposedValue)
+ throw new ObjectDisposedException(nameof(PyBuffer));
+ if (count > buffer.Length)
+ throw new ArgumentOutOfRangeException("count", "Count is bigger than the buffer.");
+ if (_view.ndim != 1)
+ throw new NotSupportedException("Multidimensional arrays, scalars and objects without a buffer are not supported.");
+ if (_view.len.ToInt64() > int.MaxValue)
+ throw new NotSupportedException("Python buffers bigger than int.MaxValue are not supported.");
+
+ int copylen = count < (int)_view.len ? count : (int)_view.len;
+ Marshal.Copy(_view.buf, buffer, offset, copylen);
+ return copylen;
+ }
+
+ private bool disposedValue = false; // To detect redundant calls
+
+ private void Dispose(bool disposing)
+ {
+ if (!disposedValue) {
+ Runtime.PyBuffer_Release(ref _view);
+
+ _exporter = null;
+ Shape = null;
+ Strides = null;
+ SubOffsets = null;
+
+ disposedValue = true;
+ }
+ }
+
+ ~PyBuffer()
+ {
+ Dispose(false);
+ }
+
+ ///
+ /// Release the buffer view and decrement the reference count for view->obj. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur.
+ /// It is an error to call this function on a buffer that was not obtained via .
+ ///
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+ }
+}
diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs
index 699ebf873..3abaf8d72 100644
--- a/src/runtime/pyobject.cs
+++ b/src/runtime/pyobject.cs
@@ -1092,6 +1092,20 @@ public override int GetHashCode()
return ((ulong)Runtime.PyObject_Hash(obj)).GetHashCode();
}
+ ///
+ /// GetBuffer Method. This Method only works for objects that have a buffer (like "bytes", "bytearray" or "array.array")
+ ///
+ ///
+ /// Send a request to the PyObject to fill in view as specified by flags. If the PyObject cannot provide a buffer of the exact type, it MUST raise PyExc_BufferError, set view->obj to NULL and return -1.
+ /// On success, fill in view, set view->obj to a new reference to exporter and return 0. In the case of chained buffer providers that redirect requests to a single object, view->obj MAY refer to this object instead of exporter(See Buffer Object Structures).
+ /// Successful calls to must be paired with calls to , similar to malloc() and free(). Thus, after the consumer is done with the buffer, must be called exactly once.
+ ///
+ public PyBuffer GetBuffer(PyBUF flags = PyBUF.SIMPLE)
+ {
+ if (Runtime.pyversionnumber < 35) throw new NotSupportedException("GetBuffer requires at least Python 3.5");
+ return new PyBuffer(this, flags);
+ }
+
public long Refcount
{
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index f63b1feae..c8458646b 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -998,6 +998,39 @@ internal static long PyObject_Size(IntPtr pointer)
[DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
internal static extern IntPtr PyObject_Dir(IntPtr pointer);
+ //====================================================================
+ // Python buffer API
+ //====================================================================
+
+ [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
+ internal static extern int PyObject_CheckBuffer(IntPtr obj);
+
+ [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
+ internal static extern int PyObject_GetBuffer(IntPtr exporter, ref Py_buffer view, int flags);
+
+ [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
+ internal static extern void PyBuffer_Release(ref Py_buffer view);
+
+ [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)]
+ internal static extern IntPtr PyBuffer_SizeFromFormat([MarshalAs(UnmanagedType.LPStr)] string format);
+
+ [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
+ internal static extern int PyBuffer_IsContiguous(ref Py_buffer view, char order);
+
+ [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
+ internal static extern IntPtr PyBuffer_GetPointer(ref Py_buffer view, IntPtr[] indices);
+
+ [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
+ internal static extern int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort);
+
+ [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
+ internal static extern int PyBuffer_ToContiguous(IntPtr buf, ref Py_buffer src, IntPtr len, char order);
+
+ [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
+ internal static extern void PyBuffer_FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, char order);
+
+ [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
+ internal static extern int PyBuffer_FillInfo(ref Py_buffer view, IntPtr exporter, IntPtr buf, IntPtr len, int _readonly, int flags);
//====================================================================
// Python number API
From def2cc2bfafcb5244fa4ad5a82b65567e21d0608 Mon Sep 17 00:00:00 2001
From: SnGmng <38666407+SnGmng@users.noreply.github.com>
Date: Sat, 23 May 2020 14:46:27 +0200
Subject: [PATCH 2/8] Update finalizer
---
src/runtime/pybuffer.cs | 42 +++++++++++++++++++++++++++++++++++++++--
1 file changed, 40 insertions(+), 2 deletions(-)
diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs
index f1d853c75..80c7aa6c1 100644
--- a/src/runtime/pybuffer.cs
+++ b/src/runtime/pybuffer.cs
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
+using System.Diagnostics;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
@@ -214,8 +215,45 @@ public int Read(byte[] buffer, int offset, int count) {
private void Dispose(bool disposing)
{
- if (!disposedValue) {
- Runtime.PyBuffer_Release(ref _view);
+ if (!disposedValue)
+ {
+ Debug.Assert(_view.obj != IntPtr.Zero, "Buffer object is invalid (no exporter object ref)");
+ //if (_view.obj == IntPtr.Zero)
+ //{
+ // return;
+ //}
+
+ if (Runtime.Py_IsInitialized() == 0)
+ throw new InvalidOperationException("Python runtime must be initialized");
+
+ if (!Runtime.IsFinalizing)
+ {
+ long refcount = Runtime.Refcount(_view.obj);
+ Debug.Assert(refcount > 0, "Object refcount is 0 or less");
+
+ if (refcount == 1)
+ {
+ Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback);
+
+ try
+ {
+ // this also decrements ref count for _view->obj
+ Runtime.PyBuffer_Release(ref _view);
+ Runtime.CheckExceptionOccurred();
+ }
+ finally
+ {
+ // Python requires finalizers to preserve exception:
+ // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation
+ Runtime.PyErr_Restore(errType, errVal, traceback);
+ }
+ }
+ else
+ {
+ // this also decrements ref count for _view->obj
+ Runtime.PyBuffer_Release(ref _view);
+ }
+ }
_exporter = null;
Shape = null;
From 8237e87b2c396d3cd31b2934fb7cb5947872a22f Mon Sep 17 00:00:00 2001
From: SnGmng <38666407+SnGmng@users.noreply.github.com>
Date: Fri, 29 May 2020 08:25:18 +0200
Subject: [PATCH 3/8] Update finalizer 2
---
src/runtime/pybuffer.cs | 49 +++++++++++------------------------------
1 file changed, 13 insertions(+), 36 deletions(-)
diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs
index 80c7aa6c1..7539e1ac6 100644
--- a/src/runtime/pybuffer.cs
+++ b/src/runtime/pybuffer.cs
@@ -7,7 +7,7 @@
namespace Python.Runtime
{
- public sealed class PyBuffer : IDisposable
+ public sealed class PyBuffer : IPyDisposable
{
private PyObject _exporter;
private Py_buffer _view;
@@ -217,43 +217,11 @@ private void Dispose(bool disposing)
{
if (!disposedValue)
{
- Debug.Assert(_view.obj != IntPtr.Zero, "Buffer object is invalid (no exporter object ref)");
- //if (_view.obj == IntPtr.Zero)
- //{
- // return;
- //}
-
if (Runtime.Py_IsInitialized() == 0)
throw new InvalidOperationException("Python runtime must be initialized");
- if (!Runtime.IsFinalizing)
- {
- long refcount = Runtime.Refcount(_view.obj);
- Debug.Assert(refcount > 0, "Object refcount is 0 or less");
-
- if (refcount == 1)
- {
- Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback);
-
- try
- {
- // this also decrements ref count for _view->obj
- Runtime.PyBuffer_Release(ref _view);
- Runtime.CheckExceptionOccurred();
- }
- finally
- {
- // Python requires finalizers to preserve exception:
- // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation
- Runtime.PyErr_Restore(errType, errVal, traceback);
- }
- }
- else
- {
- // this also decrements ref count for _view->obj
- Runtime.PyBuffer_Release(ref _view);
- }
- }
+ // this also decrements ref count for _view->obj
+ Runtime.PyBuffer_Release(ref _view);
_exporter = null;
Shape = null;
@@ -266,7 +234,11 @@ private void Dispose(bool disposing)
~PyBuffer()
{
- Dispose(false);
+ if (disposedValue)
+ {
+ return;
+ }
+ Finalizer.Instance.AddFinalizedObject(this);
}
///
@@ -278,5 +250,10 @@ public void Dispose()
Dispose(true);
GC.SuppressFinalize(this);
}
+
+ public IntPtr[] GetTrackedHandles()
+ {
+ return new IntPtr[] { _view.obj };
+ }
}
}
From edd849fab8a356e7015d67d50e5b092f06c59711 Mon Sep 17 00:00:00 2001
From: Mohamed Koubaa
Date: Sat, 1 Aug 2020 12:41:05 -0500
Subject: [PATCH 4/8] bring back pyversionnumber but leave it internal
---
src/runtime/runtime.cs | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index c8458646b..b78892bfa 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -147,6 +147,19 @@ public class Runtime
private static PyReferenceCollection _pyRefs = new PyReferenceCollection();
+ internal static int pyversionnumber
+ {
+ get
+ {
+ var versionString = PythonEngine.Version;
+ //"3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]"
+
+ //we only support python 3 so we just need to check the third character
+ return 30 + Convert.ToInt32(versionString[2]);
+ }
+ }
+
+
///
/// Initialize the runtime...
///
From 7b30cfa37178553876104b4169b9a3d582ebf717 Mon Sep 17 00:00:00 2001
From: Mohamed Koubaa
Date: Sat, 1 Aug 2020 18:36:15 -0500
Subject: [PATCH 5/8] remove 3.5 checks
---
src/embed_tests/TestPyBuffer.cs | 4 ----
src/runtime/pyobject.cs | 1 -
2 files changed, 5 deletions(-)
diff --git a/src/embed_tests/TestPyBuffer.cs b/src/embed_tests/TestPyBuffer.cs
index 3919c4dfb..0338a1480 100644
--- a/src/embed_tests/TestPyBuffer.cs
+++ b/src/embed_tests/TestPyBuffer.cs
@@ -20,8 +20,6 @@ public void Dispose()
[Test]
public void TestBufferWrite()
{
- if (Runtime.Runtime.pyversionnumber < 35) return;
-
string bufferTestString = "hello world! !$%&/()=?";
using (Py.GIL())
@@ -46,8 +44,6 @@ public void TestBufferWrite()
[Test]
public void TestBufferRead()
{
- if (Runtime.Runtime.pyversionnumber < 35) return;
-
string bufferTestString = "hello world! !$%&/()=?";
using (Py.GIL())
diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs
index 3abaf8d72..f456fd69d 100644
--- a/src/runtime/pyobject.cs
+++ b/src/runtime/pyobject.cs
@@ -1102,7 +1102,6 @@ public override int GetHashCode()
///
public PyBuffer GetBuffer(PyBUF flags = PyBUF.SIMPLE)
{
- if (Runtime.pyversionnumber < 35) throw new NotSupportedException("GetBuffer requires at least Python 3.5");
return new PyBuffer(this, flags);
}
From 1e82ee8ad59e2b4088f2bfd50f7ed5a14a7df9a1 Mon Sep 17 00:00:00 2001
From: Mohamed Koubaa
Date: Sat, 1 Aug 2020 18:49:56 -0500
Subject: [PATCH 6/8] use sys version info with C API
---
src/runtime/runtime.cs | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index b78892bfa..64740858b 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -151,11 +151,10 @@ internal static int pyversionnumber
{
get
{
- var versionString = PythonEngine.Version;
- //"3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]"
-
- //we only support python 3 so we just need to check the third character
- return 30 + Convert.ToInt32(versionString[2]);
+ var versionTuple = new PyTuple(PySys_GetObject("version_info"));
+ var major = versionTuple[0].As();
+ var minor = versionTuple[1].As();
+ return major * 10 + minor;
}
}
From d2011972a267cc10d3f42c61da9a83e81ce3799b Mon Sep 17 00:00:00 2001
From: Mohamed Koubaa
Date: Sun, 2 Aug 2020 12:22:47 -0500
Subject: [PATCH 7/8] Use System.Version
---
src/runtime/pybuffer.cs | 8 ++++----
src/runtime/runtime.cs | 5 +++--
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs
index 7539e1ac6..4c34bfeb1 100644
--- a/src/runtime/pybuffer.cs
+++ b/src/runtime/pybuffer.cs
@@ -88,7 +88,7 @@ private static char OrderStyleToChar(BufferOrderStyle order, bool eitherOneValid
///
public static long SizeFromFormat(string format)
{
- if (Runtime.pyversionnumber < 39)
+ if (Runtime.PyVersion.Minor < 9)
throw new NotSupportedException("SizeFromFormat requires at least Python 3.9");
return (long)Runtime.PyBuffer_SizeFromFormat(format);
}
@@ -111,7 +111,7 @@ public IntPtr GetPointer(long[] indices)
{
if (disposedValue)
throw new ObjectDisposedException(nameof(PyBuffer));
- if (Runtime.pyversionnumber < 37)
+ if (Runtime.PyVersion.Minor < 7)
throw new NotSupportedException("GetPointer requires at least Python 3.7");
return Runtime.PyBuffer_GetPointer(ref _view, indices.Select(x => (IntPtr)x).ToArray());
}
@@ -123,7 +123,7 @@ public void FromContiguous(IntPtr buf, long len, BufferOrderStyle fort)
{
if (disposedValue)
throw new ObjectDisposedException(nameof(PyBuffer));
- if (Runtime.pyversionnumber < 37)
+ if (Runtime.PyVersion.Minor < 7)
throw new NotSupportedException("FromContiguous requires at least Python 3.7");
if (Runtime.PyBuffer_FromContiguous(ref _view, buf, (IntPtr)len, OrderStyleToChar(fort, false)) < 0)
@@ -139,7 +139,7 @@ public void ToContiguous(IntPtr buf, BufferOrderStyle order)
{
if (disposedValue)
throw new ObjectDisposedException(nameof(PyBuffer));
- if (Runtime.pyversionnumber < 36)
+ if (Runtime.PyVersion.Minor < 6)
throw new NotSupportedException("ToContiguous requires at least Python 3.6");
if (Runtime.PyBuffer_ToContiguous(buf, ref _view, _view.len, OrderStyleToChar(order, true)) < 0)
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
index 64740858b..9e0d91b70 100644
--- a/src/runtime/runtime.cs
+++ b/src/runtime/runtime.cs
@@ -147,14 +147,15 @@ public class Runtime
private static PyReferenceCollection _pyRefs = new PyReferenceCollection();
- internal static int pyversionnumber
+ internal static Version PyVersion
{
get
{
var versionTuple = new PyTuple(PySys_GetObject("version_info"));
var major = versionTuple[0].As();
var minor = versionTuple[1].As();
- return major * 10 + minor;
+ var micro = versionTuple[2].As();
+ return new Version(major, minor, micro);
}
}
From 0ed417ca4606f5f235cb938863bab6c96068f686 Mon Sep 17 00:00:00 2001
From: Mohamed Koubaa
Date: Thu, 6 Aug 2020 21:00:36 -0500
Subject: [PATCH 8/8] compare full version
---
src/runtime/pybuffer.cs | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs
index 4c34bfeb1..978684c8d 100644
--- a/src/runtime/pybuffer.cs
+++ b/src/runtime/pybuffer.cs
@@ -88,7 +88,7 @@ private static char OrderStyleToChar(BufferOrderStyle order, bool eitherOneValid
///
public static long SizeFromFormat(string format)
{
- if (Runtime.PyVersion.Minor < 9)
+ if (Runtime.PyVersion < new Version(3,9))
throw new NotSupportedException("SizeFromFormat requires at least Python 3.9");
return (long)Runtime.PyBuffer_SizeFromFormat(format);
}
@@ -111,7 +111,7 @@ public IntPtr GetPointer(long[] indices)
{
if (disposedValue)
throw new ObjectDisposedException(nameof(PyBuffer));
- if (Runtime.PyVersion.Minor < 7)
+ if (Runtime.PyVersion < new Version(3, 7))
throw new NotSupportedException("GetPointer requires at least Python 3.7");
return Runtime.PyBuffer_GetPointer(ref _view, indices.Select(x => (IntPtr)x).ToArray());
}
@@ -123,7 +123,7 @@ public void FromContiguous(IntPtr buf, long len, BufferOrderStyle fort)
{
if (disposedValue)
throw new ObjectDisposedException(nameof(PyBuffer));
- if (Runtime.PyVersion.Minor < 7)
+ if (Runtime.PyVersion < new Version(3, 7))
throw new NotSupportedException("FromContiguous requires at least Python 3.7");
if (Runtime.PyBuffer_FromContiguous(ref _view, buf, (IntPtr)len, OrderStyleToChar(fort, false)) < 0)
@@ -139,7 +139,7 @@ public void ToContiguous(IntPtr buf, BufferOrderStyle order)
{
if (disposedValue)
throw new ObjectDisposedException(nameof(PyBuffer));
- if (Runtime.PyVersion.Minor < 6)
+ if (Runtime.PyVersion < new Version(3, 6))
throw new NotSupportedException("ToContiguous requires at least Python 3.6");
if (Runtime.PyBuffer_ToContiguous(buf, ref _view, _view.len, OrderStyleToChar(order, true)) < 0)