8000 Merge pull request #11277 from eric-wieser/fix-older-ctypes-compat · numpy/numpy@71650e3 · GitHub
[go: up one dir, main page]
Skip to content

Commit 71650e3

Browse files
charrischarris
authored
Merge pull request #11277 from eric-wieser/fix-older-ctypes-compat
BUG: Work around past and present PEP3118 issues in ctypes
2 parents af66e48 + 1e5df66 commit 71650e3

File tree

3 files changed

+144
-15
lines changed

3 files changed

+144
-15
lines changed

‎numpy/core/_internal.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -756,3 +756,16 @@ def _ufunc_doc_signature_formatter(ufunc):
756756
out_args=out_args,
757757
kwargs=kwargs
758758
)
759+
760+
761+
def _is_from_ctypes(obj):
762+
# determine if an object comes from ctypes, in order to work around
763+
# a bug in the buffer protocol for those objects, bpo-10746
764+
try:
765+
# ctypes class are new-style, so have an __mro__. This probably fails
766+
# for ctypes classes with multiple inheritance.
767+
ctype_base = type(obj).__mro__[-2]
768+
# right now, they're part of the _ctypes module
769+
return 'ctypes' in ctype_base.__module__
770+
except Exception:
771+
return False

‎numpy/core/src/multiarray/ctors.c

Lines changed: 108 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
#include "npy_config.h"
1313

14+
#include "npy_import.h"
1415
#include "npy_pycompat.h"
1516
#include "multiarraymodule.h"
1617

@@ -1302,7 +1303,51 @@ PyArray_New(PyTypeObject *subtype, int nd, npy_intp *dims, int type_num,
13021303
}
13031304

13041305

1305-
/* Steals a reference to the memory view */
1306+
NPY_NO_EXPORT PyArray_Descr *
1307+
_dtype_from_buffer_3118(PyObject *memoryview)
1308+
{
1309+
PyArray_Descr *descr;
1310+
Py_buffer *view = PyMemoryView_GET_BUFFER(memoryview);
1311+
if (view->format != NULL) {
1312+
descr = _descriptor_from_pep3118_format(view->format);
1313+
if (descr == NULL) {
1314+
return NULL;
1315+
}
1316+
}
1317+
else {
1318+
/* If no format is specified, just assume a byte array
1319+
* TODO: void would make more sense here, as it wouldn't null
1320+
* terminate.
1321+
*/
1322+
descr = PyArray_DescrNewFromType(NPY_STRING);
1323+
descr->elsize = view->itemsize;
1324+
}
1325+
return descr;
1326+
}
1327+
1328+
1329+
/*
1330+
* Call the python _is_from_ctypes
1331+
*/
1332+
NPY_NO_EXPORT int
1333+
_is_from_ctypes(PyObject *obj) {
1334+
PyObject *ret_obj;
1335+
static PyObject *py_func = NULL;
1336+
1337+
npy_cache_import("numpy.core._internal", "_is_from_ctypes", &py_func);
1338+
1339+
if (py_func == NULL) {
1340+
return -1;
1341+
}
1342+
ret_obj = PyObject_CallFunctionObjArgs(py_func, obj, NULL);
1343+
if (ret_obj == NULL) {
1344+
return -1;
1345+
}
1346+
1347+
return PyObject_IsTrue(ret_obj);
1348+
}
1349+
1350+
13061351
NPY_NO_EXPORT PyObject *
13071352
_array_from_buffer_3118(PyObject *memoryview)
13081353
{
@@ -1315,27 +1360,75 @@ _array_from_buffer_3118(PyObject *memoryview)
13151360
npy_intp shape[NPY_MAXDIMS], strides[NPY_MAXDIMS];
13161361

13171362
view = PyMemoryView_GET_BUFFER(memoryview);
1318-
if (view->format != NULL) {
1319-
descr = _descriptor_from_pep3118_format(view->format);
1320-
if (descr == NULL) {
1321-
goto fail;
1363+
nd = view->ndim;
1364+
descr = _dtype_from_buffer_3118(memoryview);
1365+
1366+
if (descr == NULL) {
1367+
return NULL;
1368+
}
1369+
1370+
/* Sanity check */
1371+
if (descr->elsize != view->itemsize) {
1372+
/* Ctypes has bugs in its PEP3118 implementation, which we need to
1373+
* work around.
1374+
*
1375+
* bpo-10746
1376+
* bpo-32780
1377+
* bpo-32782
1378+
*
1379+
* Note that even if the above are fixed in master, we have to drop the
1380+
* early patch versions of python to actually make use of the fixes.
1381+
*/
1382+
1383+
int is_ctypes = _is_from_ctypes(view->obj);
1384+
if (is_ctypes < 0) {
1385+
/* This error is not useful */
1386+
PyErr_WriteUnraisable(view->obj);
1387+
is_ctypes = 0;
13221388
}
13231389

1324-
/* Sanity check */
1325-
if (descr->elsize != view->itemsize) {
1390+
if (!is_ctypes) {
1391+
/* This object has no excuse for a broken PEP3118 buffer */
13261392
PyErr_SetString(
13271393
PyExc_RuntimeError,
13281394
"Item size computed from the PEP 3118 buffer format "
13291395
"string does not match the actual item size.");
1330-
goto fail;
1396+
Py_DECREF(descr);
1397+
return NULL;
1398+
}
1399+
1400+
if (PyErr_Warn(
1401+
PyExc_RuntimeWarning,
1402+
"A builtin ctypes object gave a PEP3118 format "
1403+
"string that does not match its itemsize, so a "
1404+
"best-guess will be made of the data type. "
1405+
"Newer versions of python may behave correctly.") < 0) {
1406+
Py_DECREF(descr);
1407+
return NULL;
1408+
}
1409+
1410+
/* Thankfully, np.dtype(ctypes_type) works in most cases.
1411+
* For an array input, this produces a dtype containing all the
1412+
* dimensions, so the array is now 0d.
1413+
*/
1414+
nd = 0;
1415+
descr = (PyArray_Descr *)PyObject_CallFunctionObjArgs(
1416+
(PyObject *)&PyArrayDescr_Type, Py_TYPE(view->obj), NULL);
1417+
if (descr == NULL) {
1418+
return NULL;
1419+
}
1420+
if (descr->elsize != view->len) {
1421+
PyErr_SetString(
1422+
PyExc_RuntimeError,
1423+
"For the given ctypes object, neither the item size "
1424+
"computed from the PEP 3118 buffer format nor from "
1425+
"converting the type to a np.dtype matched the actual "
1426+
"size. This is a bug both in python and numpy");
1427+
Py_DECREF(descr);
1428+
return NULL;
13311429
}
1332-
}
1333-
else {
1334-
descr = PyArray_DescrNewFromType(NPY_STRING);
1335-
descr->elsize = view->itemsize;
13361430
}
13371431

1338-
nd = view->ndim;
13391432
if (view->shape != NULL) {
13401433
int k;
13411434
if (nd > NPY_MAXDIMS || nd < 0) {
@@ -1379,13 +1472,12 @@ _array_from_buffer_3118(PyObject *memoryview)
13791472
&PyArray_Type, descr,
13801473
nd, shape, strides, view->buf,
13811474
flags, NULL, memoryview);
1382-
Py_DECREF(memoryview);
13831475
return r;
13841476

1477+
13851478
fail:
13861479
Py_XDECREF(r);
13871480
Py_XDECREF(descr);
1388-
Py_DECREF(memoryview);
13891481
return NULL;
13901482

13911483
}
@@ -1506,6 +1598,7 @@ PyArray_GetArrayParamsFromObject(PyObject *op,
15061598
}
15071599
else {
15081600
PyObject *arr = _array_from_buffer_3118(memoryview);
1601+
Py_DECREF(memoryview);
15091602
if (arr == NULL) {
15101603
return -1;
15111604
}

‎numpy/core/tests/test_multiarray.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6562,6 +6562,29 @@ def test_error_pointer_type(self):
65626562
ValueError, "format string",
65636563
np.array, m)
65646564

6565+
def test_ctypes_integer_via_memoryview(self):
6566+
# gh-11150, due to bpo-10746
6567+
for c_integer in {ctypes.c_int, ctypes.c_long, ctypes.c_longlong}:
6568+
value = c_integer(42)
6569+
with warnings.catch_warnings(record=True) as w:
6570+
warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning)
6571+
np.asarray(value)
6572+
6573+
def test_ctypes_struct_via_memoryview(self):
6574+
# gh-10528
6575+
class foo(ctypes.Structure):
6576+
_fields_ = [('a', ctypes.c_uint8), ('b', ctypes.c_uint32)]
6577+
f = foo(a=1, b=2)
6578+
6579+
with warnings.catch_warnings(record=True) as w:
6580+
warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning)
6581+
arr = np.asarray(f)
6582+
6583+
assert_equal(arr['a'], 1)
6584+
assert_equal(arr['b'], 2)
6585+
f.a = 3
6586+
assert_equal(arr['a'], 3)
6587+
65656588

65666589
class TestArrayAttributeDeletion(object):
65676590

0 commit comments

Comments
 (0)
0