8000 MAINT/BUG: Move ctype -> dtype conversion to python, fix issues · shoyer/numpy@44810f0 · GitHub
[go: up one dir, main page]
Skip to content

Commit 44810f0

Browse files
eric-wiesereric-wieser
committed
MAINT/BUG: Move ctype -> dtype conversion to python, fix issues
This comes with a number of changes: * Propagating the error if parsing a ctypes object goes wrong, rather than silencing it and trying more things. * Rejecting ctypes bitfields, which were incorrectly treated as subarrays. * Rejecting ctypes Unions, which were previously treated as if they were structs * Rejecting ctypes pointers, which were silently converted into their dereferenced types. This was particularly dangerous behavior when parsing structs with nested pointers, where swapping out the type changes the struct layout! * No longer support ctypes duck-types, which includes... * No longer supporting a `_fields_` attribute on `np.void` subclasses. This was a bug due to a poorly-shared code-path between void and ctypes types. The upshot is it should be easier to fix future problems in parsing ctypes types.
1 parent ea3a3d4 commit 44810f0

File tree

5 files changed

+157
-69
lines changed

5 files changed

+157
-69
lines changed

‎numpy/core/_dtype_ctypes.py

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
"""
2+
Conversion from ctypes to dtype.
3+
4+
In an ideal world, we could acheive this through the PEP3118 buffer protocol,
5+
something like::
6+
7+
def dtype_from_ctypes_type(t):
8+
# needed to ensure that the shape of `t` is within memoryview.format
9+
class DummyStruct(ctypes.Structure):
10+
_fields_ = [('a', t)]
11+
12+
# empty to avoid memory allocation
13+
ctype_0 = (DummyStruct * 0)()
14+
mv = memoryview(ctype_0)
15+
16+
# convert the struct, and slice back out the field
17+
return _dtype_from_pep3118(mv.format)['a']
18+
19+
Unfortunately, this fails because:
20+
21+
* ctypes cannot handle length-0 arrays with PEP3118 (bpo-32782)
22+
* PEP3118 cannot represent unions, but both numpy and ctypes can
23+
* ctypes cannot handle big-endian structs with PEP3118 (bpo-32780)
24+
"""
25+
import _ctypes
26+
import ctypes
27+
28+
import numpy as np
29+
30+
31+
def _from_ctypes_array(t):
32+
return np.dtype((dtype_from_ctypes_type(t._type_), (t._length_,)))
33+
34+
35+
def _from_ctypes_structure(t):
36+
# TODO: gh-10533, gh-10532
37+
fields = []
38+
for item in t._fields_:
39+
if len(item) > 2:
40+
raise TypeError(
41+
"ctypes bitfields have no dtype equivalent")
42+
fname, ftyp = item
43+
fields.append((fname, dtype_from_ctypes_type(ftyp)))
44+
45+
# by default, ctypes structs are aligned
46+
return np.dtype(fields, align=True)
47+
48+
49+
def dtype_from_ctypes_type(t):
50+
"""
51+
Construct a dtype object from a ctypes type
52+
"""
53+
if issubclass(t, _ctypes.Array):
54+
return _from_ctypes_array(t)
55+
elif issubclass(t, _ctypes._Pointer):
56+
raise TypeError("ctypes pointers have no dtype equivalent")
57+
elif issubclass(t, _ctypes.Structure):
58+
return _from_ctypes_structure(t)
59+
elif issubclass(t, _ctypes.Union):
60+
# TODO
61+
raise NotImplementedError(
62+
"conversion from ctypes.Union types like {} to dtype"
63+
.format(t.__name__))
64+
elif isinstance(t._type_, str):
65+
return np.dtype(t._type_)
66+
else:
67+
raise NotImplementedError(
68+
"Unknown ctypes type {}".format(t.__name__))

‎numpy/core/src/multiarray/descriptor.c

Lines changed: 57 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
#include "numpy/arrayscalars.h"
1111

1212
#include "npy_config.h"
13-
13+
#include "npy_ctypes.h"
1414
#include "npy_pycompat.h"
1515

1616
#include "_datetime.h"
@@ -54,79 +54,46 @@ Borrowed_PyMapping_GetItemString(PyObject *o, char *key)
5454
return ret;
5555
}
5656

57-
/*
58-
* Creates a dtype object from ctypes inputs.
59-
*
60-
* Returns a new reference to a dtype object, or NULL
61-
* if this is not possible. When it returns NULL, it does
62-
* not set a Python exception.
63-
*/
6457
static PyArray_Descr *
65-
_arraydescr_fromctypes(PyObject *obj)
58+
_arraydescr_from_ctypes_type(PyTypeObject *type)
6659
{
67-
PyObject *dtypedescr;
68-
PyArray_Descr *newdescr;
69-
int ret;
60+
PyObject *_numpy_dtype_ctypes;
61+
PyObject *res;
7062

71-
/* Understand basic ctypes */
72-
dtypedescr = PyObject_GetAttrString(obj, "_type_");
73-
PyErr_Clear();
74-
if (dtypedescr) {
75-
ret = PyArray_DescrConverter(dtypedescr, &newdescr);
76-
Py_DECREF(dtypedescr);
77-
if (ret == NPY_SUCCEED< F438 /span>) {
78-
PyObject *length;
79-
/* Check for ctypes arrays */
80-
length = PyObject_GetAttrString(obj, "_length_");
81-
PyErr_Clear();
82-
if (length) {
83-
/* derived type */
84-
PyObject *newtup;
85-
PyArray_Descr *derived;
86-
newtup = Py_BuildValue("N(N)", newdescr, length);
87-
ret = PyArray_DescrConverter(newtup, &derived);
88-
Py_DECREF(newtup);
89-
if (ret == NPY_SUCCEED) {
90-
return derived;
91-
}
92-
PyErr_Clear();
93-
return NULL;
94-
}
95-
return newdescr;
96-
}
97-
PyErr_Clear();
63+
/* Call the python function of the same name. */
64+
_numpy_dtype_ctypes = PyImport_ImportModule("numpy.core._dtype_ctypes");
65+
if (_numpy_dtype_ctypes == NULL) {
9866
return NULL;
9967
}
100-
/* Understand ctypes structures --
101-
bit-fields are not supported
102-
automatically aligns */
103-
dtypedescr = PyObject_GetAttrString(obj, "_fields_");
104-
PyErr_Clear();
105-
if (dtypedescr) {
106-
ret = PyArray_DescrAlignConverter(dtypedescr, &newdescr);
107-
Py_DECREF(dtypedescr);
108-
if (ret == NPY_SUCCEED) {
109-
return newdescr;
110-
}
111-
PyErr_Clear();
68+
res = PyObject_CallMethod(_numpy_dtype_ctypes, "dtype_from_ctypes_type", "O", (PyObject *)type);
69+
Py_DECREF(_numpy_dtype_ctypes);
70+
if (res == NULL) {
71+
return NULL;
11272
}
11373

114-
return NULL;
74+
/*
75+
* sanity check that dtype_from_ctypes_type returned the right type,
76+
* since getting it wrong would give segfaults.
77+
*/
78+
if (!PyObject_TypeCheck(res, &PyArrayDescr_Type)) {
79+
Py_DECREF(res);
80+
PyErr_BadInternalCall();
81+
return NULL;
82+
}
83+
84+
return (PyArray_Descr *)res;
11585
}
11686

11787
/*
118-
* This function creates a dtype object when:
119-
* - The object has a "dtype" attribute, and it can be converted
120-
* to a dtype object.
121-
* - The object is a ctypes type object, including array
122-
* and structure types.
88+
* This function creates a dtype object when the object has a "dtype" attribute,
89+
* and it can be converted to a dtype object.
12390
*
12491
* Returns a new reference to a dtype object, or NULL
12592
* if this is not possible. When it returns NULL, it does
12693
* not set a Python exception.
12794
*/
12895
NPY_NO_EXPORT PyArray_Descr *
129-
_arraydescr_fromobj(PyObject *obj)
96+
_arraydescr_from_dtype_attr(PyObject *obj)
13097
{
13198
PyObject *dtypedescr;
13299
PyArray_Descr *newdescr = NULL;
@@ -135,15 +102,18 @@ _arraydescr_fromobj(PyObject *obj)
135102
/* For arbitrary objects that have a "dtype" attribute */
136103
dtypedescr = PyObject_GetAttrString(obj, "dtype");
137104
PyErr_Clear();
138-
if (dtypedescr != NULL) {
139-
ret = PyArray_DescrConverter(dtypedescr, &newdescr);
140-
Py_DECREF(dtypedescr);
141-
if (ret == NPY_SUCCEED) {
142-
return newdescr;
143-
}
105+
if (dtypedescr == NULL) {
106+
return NULL;
107+
}
108+
109+
ret = PyArray_DescrConverter(dtypedescr, &newdescr);
110+
Py_DECREF(dtypedescr);
111+
if (ret != NPY_SUCCEED) {
144112
PyErr_Clear();
113+
return NULL;
145114
}
146-
return _arraydescr_fromctypes(obj);
115+
116+
return newdescr;
147117
}
148118

149119
/*
@@ -1423,10 +1393,20 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at)
14231393
check_num = NPY_VOID;
14241394
}
14251395
else {
1426-
*at = _arraydescr_fromobj(obj);
1396+
*at = _arraydescr_from_dtype_attr(obj);
14271397
if (*at) {
14281398
return NPY_SUCCEED;
14291399
}
1400+
1401+
/*
1402+
* Note: this comes after _arraydescr_from_dtype_attr because the ctypes
1403+
* type might override the dtype if numpy does not otherwise
1404+
* support it.
1405+
*/
1406+
if (npy_ctypes_check((PyTypeObject *)obj)) {
1407+
*at = _arraydescr_from_ctypes_type((PyTypeObject *)obj);
1408+
return *at ? NPY_SUCCEED : NPY_FAIL;
1409+
}
14301410
}
14311411
goto finish;
14321412
}
@@ -1596,13 +1576,23 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at)
15961576
goto fail;
15971577
}
15981578
else {
1599-
*at = _arraydescr_fromobj(obj);
1579+
*at = _arraydescr_from_dtype_attr(obj);
16001580
if (*at) {
16011581
return NPY_SUCCEED;
16021582
}
16031583
if (PyErr_Occurred()) {
16041584
return NPY_FAIL;
16051585
}
1586+
1587+
/*
1588+
* Note: this comes after _arraydescr_from_dtype_attr because the ctypes
1589+
* type might override the dtype if numpy does not otherwise
1590+
* support it.
1591+
*/
1592+
if (npy_ctypes_check(Py_TYPE(obj))) {
1593+
*at = _arraydescr_from_ctypes_type(Py_TYPE(obj));
1594+
return *at ? NPY_SUCCEED : NPY_FAIL;
1595+
}
16061596
goto fail;
16071597
}
16081598
if (PyErr_Occurred()) {

‎numpy/core/src/multiarray/descriptor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ NPY_NO_EXPORT PyObject *
88
array_set_typeDict(PyObject *NPY_UNUSED(ignored), PyObject *args);
99

1010
NPY_NO_EXPORT PyArray_Descr *
11-
_arraydescr_fromobj(PyObject *obj);
11+
_arraydescr_from_dtype_attr(PyObject *obj);
1212

1313

1414
NPY_NO_EXPORT int

‎numpy/core/src/multiarray/scalarapi.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,7 @@ PyArray_DescrFromTypeObject(PyObject *type)
471471
/* Do special thing for VOID sub-types */
472472
if (PyType_IsSubtype((PyTypeObject *)type, &PyVoidArrType_Type)) {
473473
new = PyArray_DescrNewFromType(NPY_VOID);
474-
conv = _arraydescr_fromobj(type);
474+
conv = _arraydescr_from_dtype_attr(type);
475475
if (conv) {
476476
new->fields = conv->fields;
477477
Py_INCREF(new->fields);

‎numpy/core/tests/test_dtype.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -794,6 +794,36 @@ class PaddedStruct(ctypes.Structure):
794794
], align=True)
795795
self.check(PaddedStruct, expected)
796796

797+
def test_bit_fields(self):
798+
class BitfieldStruct(ctypes.Structure):
799+
_fields_ = [
800+
('a', ctypes.c_uint8, 7),
801+
('b', ctypes.c_uint8, 1)
802+
]
803+
assert_raises(TypeError, np.dtype, BitfieldStruct)
804+
assert_raises(TypeError, np.dtype, BitfieldStruct())
805+
806+
def test_pointer(self):
807+
p_uint8 = ctypes.POINTER(ctypes.c_uint8)
808+
assert_raises(TypeError, np.dtype, p_uint8)
809+
810+
@pytest.mark.xfail(
811+
reason="Unions are not implemented",
812+
raises=NotImplementedError)
813+
def test_union(self):
814+
class Union(ctypes.Union):
815+
_fields_ = [
816+
('a', ctypes.c_uint8),
817+
('b', ctypes.c_uint16),
818+
]
819+
expected = np.dtype(dict(
820+
names=['a', 'b'],
821+
formats=[np.uint8, np.uint16],
822+
offsets=[0, 0],
823+
itemsize=2
824+
))
825+
self.check(Union, expected)
826+
797827
@pytest.mark.xfail(reason="_pack_ is ignored - see gh-11651")
798828
def test_packed_structure(self):
799829
class PackedStructure(ctypes.Structure):

0 commit comments

Comments
 (0)
0