8000 Merge pull request #10970 from eric-wieser/cut-down-ctypeslib · numpy/numpy@bfaaf9d · GitHub
[go: up one dir, main page]
Skip to content

Commit bfaaf9d

Browse files
charrischarris
authored
Merge pull request #10970 from eric-wieser/cut-down-ctypeslib
WIP: Remove fragile use of __array_interface__ in ctypeslib.as_array
2 parents 57d17c3 + 8a8be50 commit bfaaf9d

File tree

3 files changed

+101
-114
lines changed

3 files changed

+101
-114
lines changed

‎doc/release/1.15.0-notes.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,11 @@ available, but will not be maintained. The standard testing utilities,
118118
the nose specific functions `import_nose` and `raises`. Those functions are
119119
not used in numpy, but are kept for downstream compatibility.
120120

121+
Numpy no longer monkey-patches ``ctypes`` with ``__array_interface__``
122+
----------------------------------------------------------------------
123+
Previously numpy added ``__array_interface__`` attributes to all the integer
124+
types from ``ctypes``.
125+
121126
``np.ma.notmasked_contiguous`` and ``np.ma.flatnotmasked_contiguous`` always return lists
122127
-----------------------------------------------------------------------------------------
123128
This was always the documented behavior, but in reality the result used to be

‎numpy/ctypeslib.py

Lines changed: 29 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -319,120 +319,47 @@ def ndpointer(dtype=None, ndim=None, shape=None, flags=None):
319319
_pointer_type_cache[(dtype, shape, ndim, num)] = klass
320320
return klass
321321

322-
if ctypes is not None:
323-
ct = ctypes
324-
################################################################
325-
# simple types
326-
327-
# maps the numpy typecodes like '<f8' to simple ctypes types like
328-
# c_double. Filled in by prep_simple.
329-
_typecodes = {}
330-
331-
def prep_simple(simple_type, dtype):
332-
"""Given a ctypes simple type, construct and attach an
333-
__array_interface__ property to it if it does not yet have one.
334-
"""
335-
try: simple_type.__array_interface__
336-
except AttributeError: pass
337-
else: return
338-
339-
typestr = _dtype(dtype).str
340-
_typecodes[typestr] = simple_type
341-
342-
def __array_interface__(self):
343-
return {'descr': [('', typestr)],
344-
'__ref': self,
345-
'strides': None,
346-
'shape': (),
347-
'version': 3,
348-
'typestr': typestr,
349-
'data': (ct.addressof(self), False),
350-
}
351-
352-
simple_type.__array_interface__ = property(__array_interface__)
353322

323+
def _get_typecodes():
324+
""" Return a dictionary mapping __array_interface__ formats to ctypes types """
325+
ct = ctypes
354326
simple_types = [
355-
((ct.c_byte, ct.c_short, ct.c_int, ct.c_long, ct.c_longlong), "i"),
356-
((ct.c_ubyte, ct.c_ushort, ct.c_uint, ct.c_ulong, ct.c_ulonglong), "u"),
357-
((ct.c_float, ct.c_double), "f"),
327+
ct.c_byte, ct.c_short, ct.c_int, ct.c_long, ct.c_longlong,
328+
ct.c_ubyte, ct.c_ushort, ct.c_uint, ct.c_ulong, ct.c_ulonglong,
329+
ct.c_float, ct.c_double,
358330
]
359331

360-
# Prep that numerical ctypes types:
361-
for types, code in simple_types:
362-
for tp in types:
363-
prep_simple(tp, "%c%d" % (code, ct.sizeof(tp)))
332+
return {_dtype(ctype).str: ctype for ctype in simple_types}
364333

365-
################################################################
366-
# array types
367334

368-
_ARRAY_TYPE = type(ct.c_int * 1)
335+
def _ctype_ndarray(element_type, shape):
336+
""" Create an ndarray of the given element type and shape """
337+
for dim in shape[::-1]:
338+
element_type = element_type * dim
339+
return element_type
369340

370-
def prep_array(array_type):
371-
"""Given a ctypes array type, construct and attach an
372-
__array_interface__ property to it if it does not yet have one.
373-
"""
374-
try: array_type.__array_interface__
375-
except AttributeError: pass
376-
else: return
377-
378-
shape = []
379-
ob = array_type
380-
while type(ob) is _ARRAY_TYPE:
381-
shape.append(ob._length_)
382-
ob = ob._type_
383-
shape = tuple(shape)
384-
ai = ob().__array_interface__
385-
descr = ai['descr']
386-
typestr = ai['typestr']
387-
388-
def __array_interface__(self):
389-
return {'descr': descr,
390-
'__ref': self,
391-
'strides': None,
392-
'shape': shape,
393-
'version': 3,
394-
'typestr': typestr,
395-
'data': (ct.addressof(self), False),
396-
}
397-
398-
array_type.__array_interface__ = property(__array_interface__)
399-
400-
def prep_pointer(pointer_obj, shape):
401-
"""Given a ctypes pointer object, construct and
402-
attach an __array_interface__ property to it if it does not
403-
yet have one.
404-
"""
405-
try: pointer_obj.__array_interface__
406-
except AttributeError: pass
407-
else: return
408-
409-
contents = pointer_obj.contents
410-
dtype = _dtype(type(contents))
411-
412-
inter = {'version': 3,
413-
'typestr': dtype.str,
414-
'data': (ct.addressof(contents), False),
415-
'shape': shape}
416-
417-
pointer_obj.__array_interface__ = inter
418341

419-
################################################################
420-
# public functions
342+
if ctypes is not None:
343+
_typecodes = _get_typecodes()
421344

422345
def as_array(obj, shape=None):
423-
"""Create a numpy array from a ctypes array or a ctypes POINTER.
346+
"""
347+
Create a numpy array from a ctypes array or POINTER.
348+
424349
The numpy array shares the memory with the ctypes object.
425350
426-
The size parameter must be given if converting from a ctypes POINTER.
427-
The size parameter is ignored if converting from a ctypes array
351+
The shape parameter must be given if converting from a ctypes POINTER.
352+
The shape parameter is ignored if converting from a ctypes array
428353
"""
429-
tp = type(obj)
430-
try: tp.__array_interface__
431-
except AttributeError:
432-
if hasattr(obj, 'contents'):
433-
prep_pointer(obj, shape)
434-
else:
435-
prep_array(tp)
354+
if isinstance(obj, ctypes._Pointer):
355+
# convert pointers to an array of the desired shape
356+
if shape is None:
357+
raise TypeError(
358+
'as_array() requires a shape argument when called on a '
359+
'pointer')
360+
p_arr_type = ctypes.POINTER(_ctype_ndarray(obj._type_, shape))
361+
obj = ctypes.cast(obj, p_arr_type).contents
362+
436363
return array(obj, copy=False)
437364

438365
def as_ctypes(obj):
@@ -446,9 +373,7 @@ def as_ctypes(obj):
446373
addr, readonly = ai["data"]
447374
if readonly:
448375
raise TypeError("readonly arrays unsupported")
449-
tp = _typecodes[ai["typestr"]]
450-
for dim in ai["shape"][::-1]:
451-
tp = tp * dim
376+
tp = _ctype_ndarray(_typecodes[ai["typestr"]], ai["shape"])
452377
result = tp.from_address(addr)
453378
result.__keep = ai
454379
return result

‎numpy/tests/test_ctypeslib.py

Lines changed: 67 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@
44
import pytest
55

66
import numpy as np
7-
from numpy.ctypeslib import ndpointer, load_library
7+
from numpy.ctypeslib import ndpointer, load_library, as_array
88
from numpy.distutils.misc_util import get_shared_lib_extension
9-
from numpy.testing import assert_, assert_raises
9+
from numpy.testing import assert_, assert_array_equal, assert_raises, assert_equal
1010

1111
try:
1212
cdll = None
@@ -21,11 +21,12 @@
2121
except ImportError:
2222
_HAS_CTYPE = False
2323

24+
25+
@pytest.mark.skipif(not _HAS_CTYPE,
26+
reason="ctypes not available in this python")
27+
@pytest.mark.skipif(sys.platform == 'cygwin',
28+
reason="Known to fail on cygwin")
2429
class TestLoadLibrary(object):
25-
@pytest.mark.skipif(not _HAS_CTYPE,
26-
reason="ctypes not available in this python")
27-
@pytest.mark.skipif(sys.platform == 'cygwin',
28-
reason="Known to fail on cygwin")
2930
def test_basic(self):
3031
try:
3132
# Should succeed
@@ -35,10 +36,6 @@ def test_basic(self):
3536
" (import error was: %s)" % str(e))
3637
print(msg)
3738

38-
@pytest.mark.skipif(not _HAS_CTYPE,
39-
reason="ctypes not available in this python")
40-
@pytest.mark.skipif(sys.platform == 'cygwin',
41-
reason="Known to fail on cygwin")
4239
def test_basic2(self):
4340
# Regression for #801: load_library with a full library name
4441
# (including extension) does not work.
@@ -54,6 +51,7 @@ def test_basic2(self):
5451
" (import error was: %s)" % str(e))
5552
print(msg)
5653

54+
5755
class TestNdpointer(object):
5856
def test_dtype(self):
5957
dt = np.intc
@@ -113,3 +111,62 @@ def test_cache(self):
113111
a1 = ndpointer(dtype=np.float64)
114112
a2 = ndpointer(dtype=np.float64)
115113
assert_(a1 == a2)
114+
115+
116+
@pytest.mark.skipif(not _HAS_CTYPE,
117+
reason="ctypes not available on this python installation")
118+
class TestAsArray(object):
119+
def test_array(self):
120+
from ctypes import c_int
121+
122+
pair_t = c_int * 2
123+
a = as_array(pair_t(1, 2))
124+
assert_equal(a.shape, (2,))
125+
assert_array_equal(a, np.array([1, 2]))
126+
a = as_array((pair_t * 3)(pair_t(1, 2), pair_t(3, 4), pair_t(5, 6)))
127+
assert_equal(a.shape, (3, 2))
128+
assert_array_equal(a, np.array([[1, 2], [3, 4], [5, 6]]))
129+
130+
def test_pointer(self):
131+
from ctypes import c_int, cast, POINTER
132+
133+
p = cast((c_int * 10)(*range(10)), POINTER(c_int))
134+
135+
a = as_array(p, shape=(10,))
136+
assert_equal(a.shape, (10,))
137+
assert_array_equal(a, np.arange(10))
138+
139+
a = as_array(p, shape=(2, 5))
140+
assert_equal(a.shape, (2, 5))
141+
assert_array_equal(a, np.arange(10).reshape((2, 5)))
142+
143+
# shape argument is required
144+
assert_raises(TypeError, as_array, p)
145+
146+
def test_struct_array_pointer(self):
147+
from ctypes import c_int16, Structure, pointer
148+
149+
class Struct(Structure):
150+
_fields_ = [('a', c_int16)]
151+
152+
Struct3 = 3 * Struct
153+
154+
c_array = (2 * Struct3)(
155+
Struct3(Struct(a=1), Struct(a=2), Struct(a=3)),
156+
Struct3(Struct(a=4), Struct(a=5), Struct(a=6))
157+
)
158+
159+
expected = np.array([
160+
[(1,), (2,), (3,)],
161+
[(4,), (5,), (6,)],
162+
], dtype=[('a', np.int16)])
163+
164+
def check(x):
165+
assert_equal(x.dtype, expected.dtype)
166+
assert_equal(x, expected)
167+
168+
# all of these should be equivalent
169+
check(as_array(c_array))
170+
check(as_array(pointer(c_array), shape=()))
171+
check(as_array(pointer(c_array[0]), shape=(2,)))
172+
check(as_array(pointer(c_array[0][0]), shape=(2, 3)))

0 commit comments

Comments
 (0)
0