10000 API: Enforce one copy for __array__ when copy=True · numpy/numpy@d654d93 · GitHub
[go: up one dir, main page]
Skip to content

Commit d654d93

Browse files
mtsokolmtsokol
committed
API: Enforce one copy for __array__ when copy=True
1 parent e59c074 commit d654d93

File tree

13 files changed

+111
-33
lines changed

13 files changed

+111
-33
lines changed

‎environment.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,4 @@ dependencies:
4545
# Used in some tests
4646
- cffi
4747
- pytz
48+
- psutil

‎numpy/_core/src/multiarray/array_coercion.c

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -961,6 +961,8 @@ PyArray_AdaptDescriptorToArray(
961961
* @param flags Discovery flags (reporting and behaviour flags, see def.)
962962
* @param copy Specifies the copy behavior. -1 is corresponds to copy=None,
963963
* 0 to copy=False, and 1 to copy=True in the Python API.
964+
* @param copy_indicator Set to 1 if it can be assumed that a copy was made
965+
* by implementor.
964966
* @return The updated number of maximum dimensions (i.e. scalars will set
965967
* this to the current dimensions).
966968
*/
@@ -970,7 +972,7 @@ PyArray_DiscoverDTypeAndShape_Recursive(
970972
npy_intp out_shape[NPY_MAXDIMS],
971973
coercion_cache_obj ***coercion_cache_tail_ptr,
972974
PyArray_DTypeMeta *fixed_DType, enum _dtype_discovery_flags *flags,
973-
int copy)
975+
int copy, int *copy_indicator)
974976
{
975977
PyArrayObject *arr = NULL;
976978
PyObject *seq;
@@ -1028,7 +1030,7 @@ PyArray_DiscoverDTypeAndShape_Recursive(
10281030
requested_descr = *out_descr;
10291031
}
10301032
arr = (PyArrayObject *)_array_from_array_like(obj,
1031-
requested_descr, 0, NULL, copy);
1033+
requested_descr, 0, NULL, copy, copy_indicator);
10321034
if (arr == NULL) {
10331035
return -1;
10341036
}
@@ -1175,7 +1177,7 @@ PyArray_DiscoverDTypeAndShape_Recursive(
11751177
max_dims = PyArray_DiscoverDTypeAndShape_Recursive(
11761178
objects[i], curr_dims + 1, max_dims,
11771179
out_descr, out_shape, coercion_cache_tail_ptr, fixed_DType,
1178-
flags, copy);
1180+
flags, copy, copy_indicator);
11791181

11801182
if (max_dims < 0) {
11811183
return -1;
@@ -1217,6 +1219,8 @@ PyArray_DiscoverDTypeAndShape_Recursive(
12171219
* to choose a default.
12181220
* @param copy Specifies the copy behavior. -1 is corresponds to copy=None,
12191221
* 0 to copy=False, and 1 to copy=True in the Python API.
1222+
* @param copy_indicator Set to 1 if it can be assumed that a copy was made
1223+
* by implementor.
12201224
* @return dimensions of the discovered object or -1 on error.
12211225
* WARNING: If (and only if) the output is a single array, the ndim
12221226
* returned _can_ exceed the maximum allowed number of dimensions.
@@ -1229,7 +1233,7 @@ PyArray_DiscoverDTypeAndShape(
12291233
npy_intp out_shape[NPY_MAXDIMS],
12301234
coercion_cache_obj **coercion_cache,
12311235
PyArray_DTypeMeta *fixed_DType, PyArray_Descr *requested_descr,
1232-
PyArray_Descr **out_descr, int copy)
1236+
PyArray_Descr **out_descr, int copy, int *copy_indicator)
12331237
{
12341238
coercion_cache_obj **coercion_cache_head = coercion_cache;
12351239
*coercion_cache = NULL;
@@ -1277,7 +1281,7 @@ PyArray_DiscoverDTypeAndShape(
12771281

12781282
int ndim = PyArray_DiscoverDTypeAndShape_Recursive(
12791283
obj, 0, max_dims, out_descr, out_shape, &coercion_cache,
1280-
fixed_DType, &flags, copy);
1284+
fixed_DType, &flags, copy, copy_indicator);
12811285
if (ndim < 0) {
12821286
goto fail;
12831287
}
@@ -1396,7 +1400,7 @@ _discover_array_parameters(PyObject *NPY_UNUSED(self),
13961400
int ndim = PyArray_DiscoverDTypeAndShape(
13971401
obj, NPY_MAXDIMS, shape,
13981402
&coercion_cache,
1399-
dt_info.dtype, dt_info.descr, (PyArray_Descr **)&out_dtype, 0);
1403+
dt_info.dtype, dt_info.descr, (PyArray_Descr **)&out_dtype, 0, NULL);
14001404
Py_XDECREF(dt_info.dtype);
14011405
Py_XDECREF(dt_info.descr);
14021406
if (ndim < 0) {

‎numpy/_core/src/multiarray/array_coercion.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ PyArray_DiscoverDTypeAndShape(
4040
npy_intp out_shape[NPY_MAXDIMS],
4141
coercion_cache_obj **coercion_cache,
4242
PyArray_DTypeMeta *fixed_DType, PyArray_Descr *requested_descr,
43-
PyArray_Descr **out_descr, int copy);
43+
PyArray_Descr **out_descr, int copy, int *copy_indicator);
4444

4545
NPY_NO_EXPORT PyObject *
4646
_discover_array_parameters(PyObject *NPY_UNUSED(self),

‎numpy/_core/src/multiarray/arrayobject.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ PyArray_CopyObject(PyArrayObject *dest, PyObject *src_object)
251251
*/
252252
ndim = PyArray_DiscoverDTypeAndShape(src_object,
253253
PyArray_NDIM(dest), dims, &cache,
254-
NPY_DTYPE(PyArray_DESCR(dest)), PyArray_DESCR(dest), &dtype, 1);
254+
NPY_DTYPE(PyArray_DESCR(dest)), PyArray_DESCR(dest), &dtype, 1, NULL);
255255
if (ndim < 0) {
256256
return -1;
257257
}

‎numpy/_core/src/multiarray/common.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ PyArray_DTypeFromObject(PyObject *obj, int maxdims, PyArray_Descr **out_dtype)
119119
int ndim;
120120

121121
ndim = PyArray_DiscoverDTypeAndShape(
122-
obj, maxdims, shape, &cache, NULL, NULL, out_dtype, 1);
122+
obj, maxdims, shape, &cache, NULL, NULL, out_dtype, 1, NULL);
123123
if (ndim < 0) {
124124
return -1;
125125
}

‎numpy/_core/src/multiarray/ctors.c

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1429,6 +1429,8 @@ _array_from_buffer_3118(PyObject *memoryview)
14291429
* @param writeable whether the result must be writeable.
14301430
* @param context Unused parameter, must be NULL (should be removed later).
14311431
* @param copy Specifies the copy behavior.
1432+
* @param copy_indicator Set to 1 if it can be assumed that a copy was made
1433+
* by implementor.
14321434
*
14331435
* @returns The array object, Py_NotImplemented if op is not array-like,
14341436
* or NULL with an error set. (A new reference to Py_NotImplemented
@@ -1437,7 +1439,7 @@ _array_from_buffer_3118(PyObject *memoryview)
14371439
NPY_NO_EXPORT PyObject *
14381440
_array_from_array_like(PyObject *op,
14391441
PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context,
1440-
int copy) {
1442+
int copy, int *copy_indicator) {
14411443
PyObject* tmp;
14421444

14431445
/*
@@ -1485,7 +1487,7 @@ _array_from_array_like(PyObject *op,
14851487
}
14861488

14871489
if (tmp == Py_NotImplemented) {
1488-
tmp = PyArray_FromArrayAttr_int(op, requested_dtype, copy);
1490+
tmp = PyArray_FromArrayAttr_int(op, requested_dtype, copy, copy_indicator);
14891491
if (tmp == NULL) {
14901492
return NULL;
14911493
}
@@ -1572,13 +1574,16 @@ PyArray_FromAny_int(PyObject *op, PyArray_Descr *in_descr,
15721574

15731575
// Default is copy = None
15741576
int copy = -1;
1577+
int copy_indicator = 0;
15751578

15761579
if (flags & NPY_ARRAY_ENSURENOCOPY) {
15771580
copy = 0;
1581+
} else if (flags & NPY_ARRAY_ENSURECOPY) {
1582+
copy = 1;
15781583
}
15791584

15801585
ndim = PyArray_DiscoverDTypeAndShape(
1581-
op, NPY_MAXDIMS, dims, &cache, in_DType, in_descr, &dtype, copy);
1586+
op, NPY_MAXDIMS, dims, &cache, in_DType, in_descr, &dtype, copy, &copy_indicator);
15821587

15831588
if (ndim < 0) {
15841589
return NULL;
@@ -1615,6 +1620,10 @@ PyArray_FromAny_int(PyObject *op, PyArray_Descr *in_descr,
16151620
assert(cache->converted_obj == op);
16161621
arr = (PyArrayObject *)(cache->arr_or_sequence);
16171622
/* we may need to cast or assert flags (e.g. copy) */
1623+
if (copy_indicator == 1 && flags & NPY_ARRAY_ENSURECOPY) {
1624+
flags = flags & ~NPY_ARRAY_ENSURECOPY;
1625+
flags = flags | NPY_ARRAY_ENSURENOCOPY;
1626+
}
16181627
PyObject *res = PyArray_FromArray(arr, dtype, flags);
16191628
npy_unlink_coercion_cache(cache);
16201629
return res;
@@ -1937,7 +1946,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags)
19371946
}
19381947

19391948
if (copy) {
1940-
if (flags & NPY_ARRAY_ENSURENOCOPY ) {
1949+
if (flags & NPY_ARRAY_ENSURENOCOPY) {
19411950
PyErr_SetString(PyExc_ValueError, npy_no_copy_err_msg);
19421951
Py_DECREF(newtype);
19431952
return NULL;
@@ -2485,12 +2494,14 @@ check_or_clear_and_warn_error_if_due_to_copy_kwarg(PyObject *kwnames)
24852494
* NOTE: For copy == -1 it passes `op.__array__(copy=None)`,
24862495
* for copy == 0, `op.__array__(copy=False)`, and
24872496
* for copy == 1, `op.__array__(copy=True).
2497+
* @param copy_indicator Set to 1 if it can be assumed that a copy was made
2498+
* by implementor.
24882499
* @returns NotImplemented if `__array__` is not defined or a NumPy array
24892500
* (or subclass). On error, return NULL.
24902501
*/
24912502
NPY_NO_EXPORT PyObject *
24922503
PyArray_FromArrayAttr_int(
2493-
PyObject *op, PyArray_Descr *descr, int copy)
2504+
PyObject *op, PyArray_Descr *descr, int copy, int *copy_indicator)
24942505
{
24952506
PyObject *new;
24962507
PyObject *array_meth;
@@ -2577,10 +2588,11 @@ PyArray_FromArrayAttr_int(
25772588
Py_DECREF(new);
25782589
return NULL;
25792590
}
2580-
if (must_copy_but_copy_kwarg_unimplemented) {
2581-
/* TODO: As of NumPy 2.0 this path is only reachable by C-API. */
2582-
Py_SETREF(new, PyArray_NewCopy((PyArrayObject *)new, NPY_KEEPORDER));
2591+
if (copy_indicator != NULL && copy == 1 && must_copy_but_copy_kwarg_unimplemented == 0) {
2592+
/* We can assume that a copy was made */
2593+
*copy_indicator = 1;
25832594
}
2595+
25842596
return new;
25852597
}
25862598

@@ -2595,7 +2607,7 @@ PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode, PyObject *context)
25952607
return NULL;
25962608
}
25972609

2598-
return PyArray_FromArrayAttr_int(op, typecode, 0);
2610+
return PyArray_FromArrayAttr_int(op, typecode, 0, NULL);
25992611
}
26002612

26012613

‎numpy/_core/src/multiarray/ctors.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ PyArray_New(
5454
NPY_NO_EXPORT PyObject *
5555
_array_from_array_like(PyObject *op,
5656
PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context,
57-
int copy);
57+
int copy, int *copy_indicator);
5858

5959
NPY_NO_EXPORT PyObject *
6060
PyArray_FromAny_int(PyObject *op, PyArray_Descr *in_descr,
@@ -85,7 +85,7 @@ PyArray_FromInterface(PyObject *input);
8585

8686
NPY_NO_EXPORT PyObject *
8787
PyArray_FromArrayAttr_int(
88-
PyObject *op, PyArray_Descr *descr, int copy);
88+
PyObject *op, PyArray_Descr *descr, int copy, int *copy_indicator);
8989

9090
NPY_NO_EXPORT PyObject *
9191
PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode,

‎numpy/_core/tests/test_array_coercion.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,9 @@ def __init__(self, a):
5454
self.a = a
5555

5656
def __array__(self, dtype=None, copy=None):
57-
return self.a
57+
if dtype is None:
58+
return self.a
59+
return self.a.astype(dtype)
5860

5961
yield param(ArrayDunder, id="__array__")
6062

‎numpy/_core/tests/test_multiarray.py

Lines changed: 57 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
from numpy.testing import (
2929
assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal,
3030
assert_array_equal, assert_raises_regex, assert_array_almost_equal,
31-
assert_allclose, IS_PYPY, IS_WASM, IS_PYSTON, HAS_REFCOUNT,
31+
assert_allclose, IS_PYPY, IS_WASM, IS_PYSTON, HAS_REFCOUNT, HAS_PSUTIL,
3232
assert_array_less, runstring, temppath, suppress_warnings, break_cycles,
3333
_SUPPORTS_SVE, assert_array_compare,
3434
)
@@ -8452,10 +8452,9 @@ def __array__(self, dtype=None, copy=None):
84528452
for copy in self.true_vals:
84538453
res = np.array(arr, copy=copy)
84548454
assert_array_equal(res, base_arr)
8455-
# An additional copy is currently forced by numpy in this case,
8456-
# you could argue, numpy does not trust the ArrayLike. This
8457-
# may be open for change:
8458-
assert res is not base_arr
8455+
# An additional copy is no longer forced by NumPy in this case.
8456+
# NumPy trusts the ArrayLike made a copy:
8457+
assert res is base_arr
84598458

84608459
for copy in self.if_needed_vals + self.false_vals:
84618460
res = np.array(arr, copy=copy)
@@ -8488,9 +8487,11 @@ def __array__(self, dtype=None):
84888487
assert_array_equal(arr, base_arr)
84898488
assert arr is base_arr
84908489

8491-
# As of NumPy 2, explicitly passing copy=True does not trigger passing
8492-
# it to __array__ (deprecation warning is not triggered).
8493-
arr = np.array(a, copy=True)
8490+
# As of NumPy 2.1, explicitly passing copy=True does trigger passing
8491+
# it to __array__ (deprecation warning is triggered).
8492+
with pytest.warns(DeprecationWarning,
8493+
match="__array__.*should implement.*'copy'"):
8494+
arr = np.array(a, copy=True)
84948495
assert_array_equal(arr, base_arr)
84958496
assert arr is not base_arr
84968497

@@ -8501,10 +8502,57 @@ def __array__(self, dtype=None):
85018502
match=r"Unable to avoid copy(.|\n)*numpy_2_0_migration_guide.html"):
85028503
np.array(a, copy=False)
85038504

8505+
@pytest.mark.skipif(IS_PYPY, reason="PyPy copies differently")
8506+
@pytest.mark.skipif(not HAS_PSUTIL, reason="psutil isn't installed")
8507+
def test___array__copy_once(self):
8508+
import psutil
8509+
8510+
size = 1000
8511+
base_arr = np.zeros((size, size))
8512+
8513+
class ArrayRandom:
8514+
def __array__(self, values=None, dtype=None, copy=None):
8515+
if copy:
8516+
return np.random.randn(size, size)
8517+
else:
8518+
return base_arr
8519+
8520+
process = psutil.Process()
8521+
8522+
arr_random = ArrayRandom()
8523+
8524+
_ = np.random.randn(size, size)
8525+
8526+
m1 = process.memory_info().rss
8527+
8528+
base_usage = np.random.randn(1000, 1000)
8529+
8530+
m2 = process.memory_info().rss
8531+
a = m2 - m1
8532+
8533+
first_copy = np.array(arr_random, copy=True)
8534+
8535+
m3 = process.memory_info().rss
8536+
b = m3 - m2
8537+
8538+
second_copy = np.array(arr_random, copy=True)
8539+
8540+
m4 = process.memory_info().rss
8541+
c = m4 - m3
8542+
8543+
no_copy = np.array(arr_random, copy=False)
8544+
8545+
m5 = process.memory_info().rss
8546+
d = m5 - m4
8547+
8548+
assert_allclose(a, b, rtol=0.05)
8549+
assert_allclose(b, c, rtol=0.05)
8550+
assert d == 0
8551+
85048552
@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts")
85058553
def test__array__reference_leak(self):
85068554
class NotAnArray:
8507-
def __array__(self):
8555+
def __array__(self, dtype=None, copy=None):
85088556
raise NotImplementedError()
85098557

85108558
x = NotAnArray()

‎numpy/_core/tests/test_protocols.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ def test_array_called():
3535
class Wrapper:
3636
val = '0' * 100
3737

38-
def __array__(self, result=None, copy=None):
39-
return np.array([self.val], dtype=object)
38+
def __array__(self, dtype=None, copy=None):
39+
return np.array([self.val], dtype=dtype, copy=copy)
4040

4141

4242
wrapped = Wrapper()

0 commit comments

Comments
 (0)
0