10000 BUG: Work around bugs in pep3118 ctypes by falling back on np.dtype(t… · numpy/numpy@1e5df66 · GitHub
[go: up one dir, main page]
Skip to content

Commit 1e5df66

Browse files
eric-wiesereric-wieser
committed
BUG: Work around bugs in pep3118 ctypes by falling back on np.dtype(type(ctype_obj))
Fixes gh-10528 Fixes gh-10978 Fixes gh-11150 A warning is thrown when ctypes misbehaves, in order to not hide issues that need fixing upstream
1 parent 6a4ca1e commit 1e5df66

File tree

3 files changed

+122
-10
lines changed

3 files changed

+122
-10
lines changed

‎numpy/core/_internal.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -756,3 +756,16 @@ def _ufunc_doc_signature_formatter(ufunc):
756756
out_args=out_args,
757757
kwargs=kwargs
758758
)
759+
760+
761+
def _is_from_ctypes(obj):
762+
# determine if an object comes from ctypes, in order to work around
763+
# a bug in the buffer protocol for those objects, bpo-10746
764+
try:
765+
# ctypes class are new-style, so have an __mro__. This probably fails
766+
# for ctypes classes with multiple inheritance.
767+
ctype_base = type(obj).__mro__[-2]
768+
# right now, they're part of the _ctypes module
769+
return 'ctypes' in ctype_base.__module__
770+
except Exception:
771+
return False

‎numpy/core/src/multiarray/ctors.c

Lines changed: 86 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
#include "npy_config.h"
1313

14+
#include "npy_import.h"
1415
#include "npy_pycompat.h"
1516
#include "multiarraymodule.h"
1617

@@ -1312,16 +1313,6 @@ _dtype_from_buffer_3118(PyObject *memoryview)
13121313
if (descr == NULL) {
13131314
return NULL;
13141315
}
1315-
1316-
/* Sanity check */
1317-
if (descr->elsize != view->itemsize) {
1318-
PyErr_SetString(
1319-
PyExc_RuntimeError,
1320-
"Item size computed from the PEP 3118 buffer format "
1321-
"string does not match the actual item size.");
1322-
Py_DECREF(descr);
1323-
return NULL;
1324-
}
13251316
}
13261317
else {
13271318
/* If no format is specified, just assume a byte array
@@ -1335,6 +1326,28 @@ _dtype_from_buffer_3118(PyObject *memoryview)
13351326
}
13361327

13371328

1329+
/*
1330+
* Call the python _is_from_ctypes
1331+
*/
1332+
NPY_NO_EXPORT int
1333+
_is_from_ctypes(PyObject *obj) {
1334+
PyObject *ret_obj;
1335+
static PyObject *py_func = NULL;
1336+
1337+
npy_cache_import("numpy.core._internal", "_is_from_ctypes", &py_func);
1338+
1339+
if (py_func == NULL) {
1340+
return -1;
1341+
}
1342+
ret_obj = PyObject_CallFunctionObjArgs(py_func, obj, NULL);
1343+
if (ret_obj == NULL) {
1344+
return -1;
1345+
}
1346+
1347+
return PyObject_IsTrue(ret_obj);
1348+
}
1349+
1350+
13381351
NPY_NO_EXPORT PyObject *
13391352
_array_from_buffer_3118(PyObject *memoryview)
13401353
{
@@ -1354,6 +1367,68 @@ _array_from_buffer_3118(PyObject *memoryview)
13541367
return NULL;
13551368
}
13561369

1370+
/* Sanity check */
1371+
if (descr->elsize != view->itemsize) {
1372+
/* Ctypes has bugs in its PEP3118 implementation, which we need to
1373+
* work around.
1374+
*
1375+
* bpo-10746
1376+
* bpo-32780
1377+
* bpo-32782
1378+
*
1379+
* Note that even if the above are fixed in master, we have to drop the
1380+
* early patch versions of python to actually make use of the fixes.
1381+
*/
1382+
1383+
int is_ctypes = _is_from_ctypes(view->obj);
1384+
if (is_ctypes < 0) {
1385+
/* This error is not useful */
1386+
PyErr_WriteUnraisable(view->obj);
1387+
is_ctypes = 0;
1388+
}
1389+
1390+
if (!is_ctypes) {
1391+
/* This object has no excuse for a broken PEP3118 buffer */
1392+
PyErr_SetString(
1393+
PyExc_RuntimeError,
1394+
"Item size computed from the PEP 3118 buffer format "
1395+
"string does not match the actual item size.");
1396+
Py_DECREF(descr);
1397+
return NULL;
1398+
}
1399+
1400+
if (PyErr_Warn(
1401+
PyExc_RuntimeWarning,
1402+
"A builtin ctypes object gave a PEP3118 format "
1403+
"string that does not match its itemsize, so a "
1404+
"best-guess will be made of the data type. "
1405+
"Newer versions of python may behave correctly.") < 0) {
1406+
Py_DECREF(descr);
1407+
return NULL;
1408+
}
1409+
1410+
/* Thankfully, np.dtype(ctypes_type) works in most cases.
1411+
* For an array input, this produces a dtype containing all the
1412+
* dimensions, so the array is now 0d.
1413+
*/
1414+
nd = 0;
1415+
descr = (PyArray_Descr *)PyObject_CallFunctionObjArgs(
1416+
(PyObject *)&PyArrayDescr_Type, Py_TYPE(view->obj), NULL);
1417+
if (descr == NULL) {
1418+
return NULL;
1419+
}
1420+
if (descr->elsize != view->len) {
1421+
PyErr_SetString(
1422+
PyExc_RuntimeError,
1423+
"For the given ctypes object, neither the item size "
1424+
"computed from the PEP 3118 buffer format nor from "
1425+
"converting the type to a np.dtype matched the actual "
1426+
"size. This is a bug both in python and numpy");
1427+
Py_DECREF(descr);
1428+
return NULL;
1429+
}
1430+
}
1431+
13571432
if (view->shape != NULL) {
13581433
int k;
13591434
if (nd > NPY_MAXDIMS || nd < 0) {
@@ -1399,6 +1474,7 @@ _array_from_buffer_3118(PyObject *memoryview)
13991474
flags, NULL, memoryview);
14001475
return r;
14011476

1477+
14021478
fail:
14031479
Py_XDECREF(r);
14041480
Py_XDECREF(descr);

‎numpy/core/tests/test_multiarray.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6562,6 +6562,29 @@ def test_error_pointer_type(self):
65626562
ValueError, "format string",
65636563
np.array, m)
65646564

6565+
def test_ctypes_integer_via_memoryview(self):
6566+
# gh-11150, due to bpo-10746
6567+
for c_integer in {ctypes.c_int, ctypes.c_long, ctypes.c_longlong}:
6568+
value = c_integer(42)
6569+
with warnings.catch_warnings(record=True) as w:
6570+
warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning)
6571+
np.asarray(value)
6572+
6573+
def test_ctypes_struct_via_memoryview(self):
6574+
# gh-10528
6575+
class foo(ctypes.Structure):
6576+
_fields_ = [('a', ctypes.c_uint8), ('b', ctypes.c_uint32)]
6577+
f = foo(a=1, b=2)
6578+
6579+
with warnings.catch_warnings(record=True) as w:
6580+
warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning)
6581+
arr = np.asarray(f)
6582+
6583+
assert_equal(arr['a'], 1)
6584+
assert_equal(arr['b'], 2)
6585+
f.a = 3
6586+
assert_equal(arr['a'], 3)
6587+
65656588

65666589
class TestArrayAttributeDeletion(object):
65676590

0 commit comments

Comments
 (0)
0