8000 Merge pull request #12254 from eric-wieser/dtype-from-ctypes-type · numpy/numpy@763be37 · GitHub
[go: up one dir, main page]
Skip to content

Commit 763be37

Browse files
charrischarris
authored
Merge pull request #12254 from eric-wieser/dtype-from-ctypes-type
MAINT: Move ctype -> dtype conversion to python
2 parents 437534b + 0709d92 commit 763be37

File tree

9 files changed

+213
-82
lines changed

9 files changed

+213
-82
lines changed

‎numpy/core/_dtype_ctypes.py

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
"""
2+
Conversion from ctypes to dtype.
3+
4+
In an ideal world, we could acheive this through the PEP3118 buffer protocol,
5+
something like::
6+
7+
def dtype_from_ctypes_type(t):
8+
# needed to ensure that the shape of `t` is within memoryview.format
9+
class DummyStruct(ctypes.Structure):
10+
_fields_ = [('a', t)]
11+
12+
# empty to avoid memory allocation
13+
ctype_0 = (DummyStruct * 0)()
14+
mv = memoryview(ctype_0)
15+
16+
# convert the struct, and slice back out the field
17+
return _dtype_from_pep3118(mv.format)['a']
18+
19+
Unfortunately, this fails because:
20+
21+
* ctypes cannot handle length-0 arrays with PEP3118 (bpo-32782)
22+
* PEP3118 cannot represent unions, but both numpy and ctypes can
23+
* ctypes cannot handle big-endian structs with PEP3118 (bpo-32780)
24+
"""
25+
import _ctypes
26+
import ctypes
27+
28+
import numpy as np
29+
30+
31+
def _from_ctypes_array(t):
32+
return np.dtype((dtype_from_ctypes_type(t._type_), (t._length_,)))
33+
34+
35+
def _from_ctypes_structure(t):
36+
# TODO: gh-10533, gh-10532
37+
fields = []
38+
for item in t._fields_:
39+
if len(item) > 2:
40+
raise TypeError(
41+
"ctypes bitfields have no dtype equivalent")
42+
fname, ftyp = item
43+
fields.append((fname, dtype_from_ctypes_type(ftyp)))
44+
45+
# by default, ctypes structs are aligned
46+
return np.dtype(fields, align=True)
47+
48+
49+
def dtype_from_ctypes_type(t):
50+
"""
51+
Construct a dtype object from a ctypes type
52+
"""
53+
if issubclass(t, _ctypes.Array):
54+
return _from_ctypes_array(t)
55+
elif issubclass(t, _ctypes._Pointer):
56+
raise TypeError("ctypes pointers have no dtype equivalent")
57+
elif issubclass(t, _ctypes.Structure):
58+
return _from_ctypes_structure(t)
59+
elif issubclass(t, _ctypes.Union):
60+
# TODO
61+
raise NotImplementedError(
62+
"conversion from ctypes.Union types like {} to dtype"
63+
.format(t.__name__))
64+
elif isinstance(t._type_, str):
65+
return np.dtype(t._type_)
66+
else:
67+
raise NotImplementedError(
68+
"Unknown ctypes type {}".format(t.__name__))

‎numpy/core/_internal.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -796,13 +796,13 @@ def _ufunc_doc_signature_formatter(ufunc):
796796
)
797797

798798

799-
def _is_from_ctypes(obj):
800-
# determine if an object comes from ctypes, in order to work around
799+
def npy_ctypes_check(cls):
800+
# determine if a class comes from ctypes, in order to work around
801801
# a bug in the buffer protocol for those objects, bpo-10746
802802
try:
803803
# ctypes class are new-style, so have an __mro__. This probably fails
804804
# for ctypes classes with multiple inheritance.
805-
ctype_base = type(obj).__mro__[-2]
805+
ctype_base = cls.__mro__[-2]
806806
# right now, they're part of the _ctypes module
807807
return 'ctypes' in ctype_base.__module__
808808
except Exception:

‎numpy/core/setup.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -731,7 +731,9 @@ def get_mathlib_info(*args):
731731
join('src', 'common', 'lowlevel_strided_loops.h'),
732732
join('src', 'common', 'mem_overlap.h'),
733733
join('src', 'common', 'npy_config.h'),
734+
join('src', 'common', 'npy_ctypes.h'),
734735
join('src', 'common', 'npy_extint128.h'),
736+
join('src', 'common', 'npy_import.h'),
735737
join('src', 'common', 'npy_longdouble.h'),
736738
join('src', 'common', 'templ_common.h.src'),
737739
join('src', 'common', 'ucsnarrow.h'),

‎numpy/core/src/common/npy_ctypes.h

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#ifndef NPY_CTYPES_H
2+
#define NPY_CTYPES_H
3+
4+
#include <Python.h>
5+
6+
#include "npy_import.h"
7+
8+
/*
9+
* Check if a python type is a ctypes class.
10+
*
11+
* Works like the Py<type>_Check functions, returning true if the argument
12+
* looks like a ctypes object.
13+
*
14+
* This entire function is just a wrapper around the Python function of the
15+
* same name.
16+
*/
17+
NPY_INLINE static int
18+
npy_ctypes_check(PyTypeObject *obj)
19+
{
20+
static PyObject *py_func = NULL;
21+
PyObject *ret_obj;
22+
int ret;
23+
24+
npy_cache_import("numpy.core._internal", "npy_ctypes_check", &py_func);
25+
if (py_func == NULL) {
26+
goto fail;
27+
}
28+
29+
ret_obj = PyObject_CallFunctionObjArgs(py_func, (PyObject *)obj, NULL);
30+
if (ret_obj == NULL) {
31+
goto fail;
32+
}
33+
34+
ret = PyObject_IsTrue(ret_obj);
35+
if (ret == -1) {
36+
goto fail;
37+
}
38+
39+
return ret;
40+
41+
fail:
42+
/* If the above fails, then we should just assume that the type is not from
43+
* ctypes
44+
*/
45+
PyErr_Clear();
46+
return 0;
47+
}
48+
49+
#endif

‎numpy/core/src/multiarray/ctors.c

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
#include "npy_config.h"
1313

14-
#include "npy_import.h"
14+
#include "npy_ctypes.h"
1515
#include "npy_pycompat.h"
1616
#include "multiarraymodule.h"
1717

@@ -1381,15 +1381,7 @@ _array_from_buffer_3118(PyObject *memoryview)
13811381
* Note that even if the above are fixed in master, we have to drop the
13821382
* early patch versions of python to actually make use of the fixes.
13831383
*/
1384-
1385-
int is_ctypes = _is_from_ctypes(view->obj);
1386-
if (is_ctypes < 0) {
1387-
/* This error is not useful */
1388-
PyErr_WriteUnraisable(view->obj);
1389-
is_ctypes = 0;
1390-
}
1391-
1392-
if (!is_ctypes) {
1384+
if (!npy_ctypes_check(Py_TYPE(view->obj))) {
13931385
/* This object has no excuse for a broken PEP3118 buffer */
13941386
PyErr_Format(
13951387
PyExc_RuntimeError,

‎numpy/core/src/multiarray/descriptor.c

Lines changed: 57 additions & 67 deletions
140
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
#include "numpy/arrayscalars.h"
1111

1212
#include "npy_config.h"
13-
13+
#include "npy_ctypes.h"
1414
#include "npy_pycompat.h"
1515

1616
#include "_datetime.h"
@@ -54,79 +54,46 @@ Borrowed_PyMapping_GetItemString(PyObject *o, char *key)
5454
return ret;
5555
}
5656

57-
/*
58-
* Creates a dtype object from ctypes inputs.
59-
*
60-
* Returns a new reference to a dtype object, or NULL
61-
* if this is not possible. When it returns NULL, it does
62-
* not set a Python exception.
63-
*/
6457
static PyArray_Descr *
65-
_arraydescr_fromctypes(PyObject *obj)
58+
_arraydescr_from_ctypes_type(PyTypeObject *type)
6659
{
67-
PyObject *dtypedescr;
68-
PyArray_Descr *newdescr;
69-
int ret;
60+
PyObject *_numpy_dtype_ctypes;
61+
PyObject *res;
7062

71-
/* Understand basic ctypes */
72-
dtypedescr = PyObject_GetAttrString(obj, "_type_");
73-
PyErr_Clear();
74-
if (dtypedescr) {
75-
ret = PyArray_DescrConverter(dtypedescr, &newdescr);
76-
Py_DECREF(dtypedescr);
77-
if (ret == NPY_SUCCEED) {
78-
PyObject *length;
79-
/* Check for ctypes arrays */
80-
length = PyObject_GetAttrString(obj, "_length_");
81-
PyErr_Clear();
82-
if (length) {
83-
/* derived type */
84-
PyObject *newtup;
85-
PyArray_Descr *derived;
86-
newtup = Py_BuildValue("N(N)", newdescr, length);
87-
ret = PyArray_DescrConverter(newtup, &derived);
88-
Py_DECREF(newtup);
89-
if (ret == NPY_SUCCEED) {
90-
return derived;
91-
}
92-
PyErr_Clear();
93-
return NULL;
94-
}
95-
return newdescr;
96-
}
97-
PyErr_Clear();
63+
/* Call the python function of the same name. */
64+
_numpy_dtype_ctypes = PyImport_ImportModule("numpy.core._dtype_ctypes");
65+
if (_numpy_dtype_ctypes == NULL) {
9866
return NULL;
9967
}
100-
/* Understand ctypes structures --
101-
bit-fields are not supported
102-
automatically aligns */
103-
dtypedescr = PyObject_GetAttrString(obj, "_fields_");
104-
PyErr_Clear();
105-
if (dtypedescr) {
106-
ret = PyArray_DescrAlignConverter(dtypedescr, &newdescr);
107-
Py_DECREF(dtypedescr);
108-
if (ret == NPY_SUCCEED) {
109-
return newdescr;
110-
}
111-
PyErr_Clear();
68+
res = PyObject_CallMethod(_numpy_dtype_ctypes, "dtype_from_ctypes_type", "O", (PyObject *)type);
69+
Py_DECREF(_numpy_dtype_ctypes);
70+
if (res == NULL) {
71+
return NULL;
11272
}
11373

114-
return NULL;
74+
/*
75+
* sanity check that dtype_from_ctypes_type returned the right type,
76+
* since getting it wrong would give segfaults.
77+
*/
78+
if (!PyObject_TypeCheck(res, &PyArrayDescr_Type)) {
79+
Py_DECREF(res);
80+
PyErr_BadInternalCall();
81+
return NULL;
82+
}
83+
84+
return (PyArray_Descr *)res;
11585
}
11686

11787
/*
118-
* This function creates a dtype object when:
119-
* - The object has a "dtype" attribute, and it can be converted
120-
* to a dtype object.
121-
* - The object is a ctypes type object, including array
122-
* and structure types.
88+
* This function creates a dtype object when the object has a "dtype" attribute,
89+
* and it can be converted to a dtype object.
12390
*
12491
* Returns a new reference to a dtype object, or NULL
12592
* if this is not possible. When it returns NULL, it does
12693
* not set a Python exception.
12794
*/
12895
NPY_NO_EXPORT PyArray_Descr *
129-
_arraydescr_fromobj(PyObject *obj)
96+
_arraydescr_from_dtype_attr(PyObject *obj)
13097
{
13198
PyObject *dtypedescr;
13299
PyArray_Descr *newdescr = NULL;
@@ -135,15 +102,18 @@ _arraydescr_fromobj(PyObject *obj)
135102
/* For arbitrary objects that have a "dtype" attribute */
136103
dtypedescr = PyObject_GetAttrString(obj, "dtype");
137104
PyErr_Clear();
138-
if (dtypedescr != NULL) {
139-
ret = PyArray_DescrConverter(dtypedescr, &newdescr);
-
Py_DECREF(dtypedescr);
141-
if (ret == NPY_SUCCEED) {
142-
return newdescr;
143-
}
105+
if (dtypedescr == NULL) {
106+
return NULL;
107+
}
108+
109+
ret = PyArray_DescrConverter(dtypedescr, &newdescr);
110+
Py_DECREF(dtypedescr);
111+
if (ret != NPY_SUCCEED) {
144112
PyErr_Clear();
113+
return NULL;
145114
}
146-
return _arraydescr_fromctypes(obj);
115+
116+
return newdescr;
147117
}
148118

149119
/*
@@ -1423,10 +1393,20 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at)
14231393
check_num = NPY_VOID;
14241394
}
14251395
else {
1426-
*at = _arraydescr_fromobj(obj);
1396+
*at = _arraydescr_from_dtype_attr(obj);
14271397
if (*at) {
14281398
return NPY_SUCCEED;
14291399
}
1400+
1401+
/*
1402+
* Note: this comes after _arraydescr_from_dtype_attr because the ctypes
1403+
* type might override the dtype if numpy does not otherwise
1404+
* support it.
1405+
*/
1406+
if (npy_ctypes_check((PyTypeObject *)obj)) {
1407+
*at = _arraydescr_from_ctypes_type((PyTypeObject *)obj);
1408+
return *at ? NPY_SUCCEED : NPY_FAIL;
1409+
}
14301410
}
14311411
goto finish;
14321412
}
@@ -1596,13 +1576,23 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at)
15961576
goto fail;
15971577
}
15981578
else {
1599-
*at = _arraydescr_fromobj(obj);
1579+
*at = _arraydescr_from_dtype_attr(obj);
16001580
if (*at) {
16011581
return NPY_SUCCEED;
16021582
}
16031583
if (PyErr_Occurred()) {
16041584
return NPY_FAIL;
16051585
}
1586+
1587+
/*
1588+
* Note: this comes after _arraydescr_from_dtype_attr because the ctypes
1589+
* type might override the dtype if numpy does not otherwise
1590+
* support it.
1591+
*/
1592+
if (npy_ctypes_check(Py_TYPE(obj))) {
1593+
*at = _arraydescr_from_ctypes_type(Py_TYPE(obj));
1594+
return *at ? NPY_SUCCEED : NPY_FAIL;
1595+
}
16061596
goto fail;
16071597
}
16081598
if (PyErr_Occurred()) {

‎numpy/core/src/multiarray/descriptor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ NPY_NO_EXPORT PyObject *
88
array_set_typeDict(PyObject *NPY_UNUSED(ignored), PyObject *args);
99

1010
NPY_NO_EXPORT PyArray_Descr *
11-
_arraydescr_fromobj(PyObject *obj);
11+
_arraydescr_from_dtype_attr(PyObject *obj);
1212

1313

1414
NPY_NO_EXPORT int

‎numpy/core/src/multiarray/scalarapi.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,7 @@ PyArray_DescrFromTypeObject(PyObject *type)
471471
/* Do special thing for VOID sub-types */
472472
if (PyType_IsSubtype((PyTypeObject *)type, &PyVoidArrType_Type)) {
473473
new = PyArray_DescrNewFromType(NPY_VOID);
474-
conv = _arraydescr_fromobj(type);
474+
conv = _arraydescr_from_dtype_attr(type);
475475
if (conv) {
476476
new->fields = conv->fields;
477477
Py_INCREF(new->fields);

0 commit comments

Comments
 (0)
0