8000 API: Enforce one copy for __array__ when copy=True · numpy/numpy@553c473 · GitHub
[go: up one dir, main page]
Skip to content

Commit 553c473

Browse files
mtsokolmtsokol
committed
API: Enforce one copy for __array__ when copy=True
1 parent 4d65246 commit 553c473

File tree

9 files changed

+94
-28
lines changed

9 files changed

+94
-28
lines changed

‎numpy/_core/src/multiarray/array_coercion.c

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ enum _dtype_discovery_flags {
9999
DISCOVER_TUPLES_AS_ELEMENTS = 1 << 4,
100100
MAX_DIMS_WAS_REACHED = 1 << 5,
101101
DESCRIPTOR_WAS_SET = 1 << 6,
102+
COPY_WAS_CREATED = 1 << 7,
102103
};
103104

104105

@@ -1027,15 +1028,19 @@ PyArray_DiscoverDTypeAndShape_Recursive(
10271028
/* __array__ may be passed the requested descriptor if provided */
10281029
requested_descr = *out_descr;
10291030
}
1031+
int was_copied = 0;
10301032
arr = (PyArrayObject *)_array_from_array_like(obj,
1031-
requested_descr, 0, NULL, copy);
1033+
requested_descr, 0, NULL, copy, &was_copied);
10321034
if (arr == NULL) {
10331035
return -1;
10341036
}
10351037
else if (arr == (PyArrayObject *)Py_NotImplemented) {
10361038
Py_DECREF(arr);
10371039
arr = NULL;
10381040
}
1041+
if (was_copied == 1) {
1042+
*flags |= COPY_WAS_CREATED;
1043+
}
10391044
}
10401045
if (arr != NULL) {
10411046
/*
@@ -1170,6 +1175,15 @@ PyArray_DiscoverDTypeAndShape_Recursive(
11701175
return -1;
11711176
}
11721177

1178+
/*
1179+
* For a sequence we need to make a copy of the final aggreate anyway.
1180+
* There's no need to pass explicit `copy=True`, so we switch
1181+
* to `copy=None` (copy if needed).
1182+
*/
1183+
if (copy == 1) {
1184+
copy = -1;
1185+
}
1186+
11731187
/* Recursive call for each sequence item */
11741188
for (Py_ssize_t i = 0; i < size; i++) {
11751189
max_dims = PyArray_DiscoverDTypeAndShape_Recursive(
@@ -1217,6 +1231,8 @@ PyArray_DiscoverDTypeAndShape_Recursive(
12171231
* to choose a default.
12181232
* @param copy Specifies the copy behavior. -1 is corresponds to copy=None,
12191233
* 0 to copy=False, and 1 to copy=True in the Python API.
1234+
* @param was_copied Set to 1 if it can be assumed that a copy was made
1235+
* by implementor.
12201236
* @return dimensions of the discovered object or -1 on error.
12211237
* WARNING: If (and only if) the output is a single array, the ndim
12221238
* returned _can_ exceed the maximum allowed number of dimensions.
@@ -1229,7 +1245,7 @@ PyArray_DiscoverDTypeAndShape(
12291245
npy_intp out_shape[NPY_MAXDIMS],
12301246
coercion_cache_obj **coercion_cache,
12311247
PyArray_DTypeMeta *fixed_DType, PyArray_Descr *requested_descr,
1232-
PyArray_Descr **out_descr, int copy)
1248+
PyArray_Descr **out_descr, int copy, int *was_copied)
12331249
{
12341250
coercion_cache_obj **coercion_cache_head = coercion_cache;
12351251
*coercion_cache = NULL;
@@ -1282,6 +1298,10 @@ PyArray_DiscoverDTypeAndShape(
12821298
goto fail;
12831299
}
12841300

1301+
if (was_copied != NULL && flags & COPY_WAS_CREATED) {
1302+
*was_copied = 1;
1303+
}
1304+
12851305
if (NPY_UNLIKELY(flags & FOUND_RAGGED_ARRAY)) {
12861306
/*
12871307
* If max-dims was reached and the dimensions reduced, this is ragged.
@@ -1396,7 +1416,7 @@ _discover_array_parameters(PyObject *NPY_UNUSED(self),
13961416
int ndim = PyArray_DiscoverDTypeAndShape(
13971417
obj, NPY_MAXDIMS, shape,
13981418
&coercion_cache,
1399-
dt_info.dtype, dt_info.descr, (PyArray_Descr **)&out_dtype, 0);
1419+
dt_info.dtype, dt_info.descr, (PyArray_Descr **)&out_dtype, 0, NULL);
14001420
Py_XDECREF(dt_info.dtype);
14011421
Py_XDECREF(dt_info.descr);
14021422
if (ndim < 0) {

‎numpy/_core/src/multiarray/array_coercion.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ PyArray_DiscoverDTypeAndShape(
4040
npy_intp out_shape[NPY_MAXDIMS],
4141
coercion_cache_obj **coercion_cache,
4242
PyArray_DTypeMeta *fixed_DType, PyArray_Descr *requested_descr,
43-
PyArray_Descr **out_descr, int copy);
43+
PyArray_Descr **out_descr, int copy, int *was_copied);
4444

4545
NPY_NO_EXPORT PyObject *
4646
_discover_array_parameters(PyObject *NPY_UNUSED(self),

‎numpy/_core/src/multiarray/arrayobject.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ PyArray_CopyObject(PyArrayObject *dest, PyObject *src_object)
251251
*/
252252
ndim = PyArray_DiscoverDTypeAndShape(src_object,
253253
PyArray_NDIM(dest), dims, &cache,
254-
NPY_DTYPE(PyArray_DESCR(dest)), PyArray_DESCR(dest), &dtype, 1);
254+
NPY_DTYPE(PyArray_DESCR(dest)), PyArray_DESCR(dest), &dtype, 1, NULL);
255255
if (ndim < 0) {
256256
return -1;
257257
}

‎numpy/_core/src/multiarray/common.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ PyArray_DTypeFromObject(PyObject *obj, int maxdims, PyArray_Descr **out_dtype)
119119
int ndim;
120120

121121
ndim = PyArray_DiscoverDTypeAndShape(
122-
obj, maxdims, shape, &cache, NULL, NULL, out_dtype, 1);
122+
obj, maxdims, shape, &cache, NULL, NULL, out_dtype, 1, NULL);
123123
if (ndim < 0) {
124124
return -1;
125125
}

‎numpy/_core/src/multiarray/ctors.c

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1429,6 +1429,8 @@ _array_from_buffer_3118(PyObject *memoryview)
14291429
* @param writeable whether the result must be writeable.
14301430
* @param context Unused parameter, must be NULL (should be removed later).
14311431
* @param copy Specifies the copy behavior.
1432+
* @param was_copied Set to 1 if it can be assumed that a copy was made
1433+
* by implementor.
14321434
*
14331435
* @returns The array object, Py_NotImplemented if op is not array-like,
14341436
* or NULL with an error set. (A new reference to Py_NotImplemented
@@ -1437,7 +1439,7 @@ _array_from_buffer_3118(PyObject *memoryview)
14371439
NPY_NO_EXPORT PyObject *
14381440
_array_from_array_like(PyObject *op,
14391441
PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context,
1440-
int copy) {
1442+
int copy, int *was_copied) {
14411443
PyObject* tmp;
14421444

14431445
/*
@@ -1485,7 +1487,7 @@ _array_from_array_like(PyObject *op,
14851487
}
14861488

14871489
if (tmp == Py_NotImplemented) {
1488-
tmp = PyArray_FromArrayAttr_int(op, requested_dtype, copy);
1490+
tmp = PyArray_FromArrayAttr_int(op, requested_dtype, copy, was_copied);
14891491
if (tmp == NULL) {
14901492
return NULL;
14911493
}
@@ -1572,13 +1574,16 @@ PyArray_FromAny_int(PyObject *op, PyArray_Descr *in_descr,
15721574

15731575
// Default is copy = None
15741576
int copy = -1;
1577+
int was_copied = 0;
15751578

15761579
if (flags & NPY_ARRAY_ENSURENOCOPY) {
15771580
copy = 0;
1581+
} else if (flags & NPY_ARRAY_ENSURECOPY) {
1582+
copy = 1;
15781583
}
15791584

15801585
ndim = PyArray_DiscoverDTypeAndShape(
1581-
op, NPY_MAXDIMS, dims, &cache, in_DType, in_descr, &dtype, copy);
1586+
op, NPY_MAXDIMS, dims, &cache, in_DType, in_descr, &dtype, copy, &was_copied);
15821587

15831588
if (ndim < 0) {
15841589
return NULL;
@@ -1615,6 +1620,10 @@ PyArray_FromAny_int(PyObject *op, PyArray_Descr *in_descr,
16151620
assert(cache->converted_obj == op);
16161621
arr = (PyArrayObject *)(cache->arr_or_sequence);
16171622
/* we may need to cast or assert flags (e.g. copy) */
1623+
if (was_copied == 1 && flags & NPY_ARRAY_ENSURECOPY) {
1624+
flags = flags & ~NPY_ARRAY_ENSURECOPY;
1625+
flags = flags | NPY_ARRAY_ENSURENOCOPY;
1626+
}
16181627
PyObject *res = PyArray_FromArray(arr, dtype, flags);
16191628
npy_unlink_coercion_cache(cache);
16201629
return res;
@@ -1937,7 +1946,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags)
19371946
}
19381947

19391948
if (copy) {
1940-
if (flags & NPY_ARRAY_ENSURENOCOPY ) {
1949+
if (flags & NPY_ARRAY_ENSURENOCOPY) {
19411950
PyErr_SetString(PyExc_ValueError, npy_no_copy_err_msg);
19421951
Py_DECREF(newtype);
19431952
return NULL;
@@ -2485,12 +2494,14 @@ check_or_clear_and_warn_error_if_due_to_copy_kwarg(PyObject *kwnames)
24852494
* NOTE: For copy == -1 it passes `op.__array__(copy=None)`,
24862495
* for copy == 0, `op.__array__(copy=False)`, and
24872496
* for copy == 1, `op.__array__(copy=True).
2497+
* @param was_copied Set to 1 if it can be assumed that a copy was made
2498+
* by implementor.
24882499
* @returns NotImplemented if `__array__` is not defined or a NumPy array
24892500
* (or subclass). On error, return NULL.
24902501
*/
24912502
NPY_NO_EXPORT PyObject *
24922503
PyArray_FromArrayAttr_int(
2493-
PyObject *op, PyArray_Descr *descr, int copy)
2504+
PyObject *op, PyArray_Descr *descr, int copy, int *was_copied)
24942505
{
24952506
PyObject *new;
24962507
PyObject *array_meth;
@@ -2577,10 +2588,11 @@ PyArray_FromArrayAttr_int(
25772588
Py_DECREF(new);
25782589
return NULL;
25792590
}
2580-
if (must_copy_but_copy_kwarg_unimplemented) {
2581-
/* TODO: As of NumPy 2.0 this path is only reachable by C-API. */
2582-
Py_SETREF(new, PyArray_NewCopy((PyArrayObject *)new, NPY_KEEPORDER));
2591+
if (was_copied != NULL && copy == 1 && must_copy_but_copy_kwarg_unimplemented == 0) {
2592+
/* We can assume that a copy was made */
2593+
*was_copied = 1;
25832594
}
2595+
25842596
return new;
25852597
}
25862598

@@ -2595,7 +2607,7 @@ PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode, PyObject *context)
25952607
return NULL;
25962608
}
25972609

2598-
return PyArray_FromArrayAttr_int(op, typecode, 0);
2610+
return PyArray_FromArrayAttr_int(op, typecode, 0, NULL);
25992611
}
26002612

26012613

‎numpy/_core/src/multiarray/ctors.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ PyArray_New(
5454
NPY_NO_EXPORT PyObject *
5555
_array_from_array_like(PyObject *op,
5656
PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context,
57-
int copy);
57+
int copy, int *was_copied);
5858

5959
NPY_NO_EXPORT PyObject *
6060
PyArray_FromAny_int(PyObject *op, PyArray_Descr *in_descr,
@@ -85,7 +85,7 @@ PyArray_FromInterface(PyObject *input);
8585

8686
NPY_NO_EXPORT PyObject *
8787
PyArray_FromArrayAttr_int(
88-
PyObject *op, PyArray_Descr *descr, int copy);
88+
PyObject *op, PyArray_Descr *descr, int copy, int *was_copied);
8989

9090
NPY_NO_EXPORT PyObject *
9191
PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode,

‎numpy/_core/tests/test_array_coercion.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,9 @@ def __init__(self, a):
5454
self.a = a
5555

5656
def __array__(self, dtype=None, copy=None):
57-
return self.a
57+
if dtype is None:
58+
return self.a
59+
return self.a.astype(dtype)
5860

5961
yield param(ArrayDunder, id="__array__")
6062

‎numpy/_core/tests/test_multiarray.py

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8452,10 +8452,9 @@ def __array__(self, dtype=None, copy=None):
84528452
for copy in self.true_vals:
84538453
res = np.array(arr, copy=copy)
84548454
assert_array_equal(res, base_arr)
8455-
# An additional copy is currently forced by numpy in this case,
8456-
# you could argue, numpy does not trust the ArrayLike. This
8457-
# may be open for change:
8458-
assert res is not base_arr
8455+
# An additional copy is no longer forced by NumPy in this case.
8456+
# NumPy trusts the ArrayLike made a copy:
8457+
assert res is base_arr
84598458

84608459
for copy in self.if_needed_vals + self.false_vals:
84618460
res = np.array(arr, copy=copy)
@@ -8488,9 +8487,11 @@ def __array__(self, dtype=None):
84888487
assert_array_equal(arr, base_arr)
84898488
assert arr is base_arr
84908489

8491-
# As of NumPy 2, explicitly passing copy=True does not trigger passing
8492-
# it to __array__ (deprecation warning is not triggered).
8493-
arr = np.array(a, copy=True)
8490+
# As of NumPy 2.1, explicitly passing copy=True does trigger passing
8491+
# it to __array__ (deprecation warning is triggered).
8492+
with pytest.warns(DeprecationWarning,
8493+
match="__array__.*should implement.*'copy'"):
8494+
arr = np.array(a, copy=True)
84948495
assert_array_equal(arr, base_arr)
84958496
assert arr is not base_arr
84968497

@@ -8501,10 +8502,41 @@ def __array__(self, dtype=None):
85018502
match=r"Unable to avoid copy(.|\n)*numpy_2_0_migration_guide.html"):
85028503
np.array(a, copy=False)
85038504

8505+
@pytest.mark.skipif(IS_PYPY, reason="PyPy copies differently")
8506+
def test___array__copy_once(self):
8507+
size = 100
8508+
base_arr = np.zeros((size, size))
8509+
copy_arr = np.zeros((size, size))
8510+
8511+
class ArrayRandom:
8512+
def __init__(self):
8513+
self.true_passed = False
8514+
8515+
def __array__(self, dtype=None, copy=None):
8516+
if copy:
8517+
self.true_passed = True
8518+
return copy_arr
8519+
else:
8520+
return base_arr
8521+
8522+
arr_random = ArrayRandom()
8523+
first_copy = np.array(arr_random, copy=True)
8524+
assert arr_random.true_passed
8525+
assert first_copy is copy_arr
8526+
8527+
arr_random = ArrayRandom()
8528+
no_copy = np.array(arr_random, copy=False)
8529+
assert not arr_random.true_passed
8530+
assert no_copy is base_arr
8531+
8532+
arr_random = ArrayRandom()
8533+
_ = np.array([arr_random], copy=True)
8534+
assert not arr_random.true_passed
8535+
85048536
@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts")
85058537
def test__array__reference_leak(self):
85068538
class NotAnArray:
8507-
def __array__(self):
8539+
def __array__(self, dtype=None, copy=None):
85088540
raise NotImplementedError()
85098541

85108542
x = NotAnArray()

‎numpy/_core/tests/test_protocols.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ def test_array_called():
3535
class Wrapper:
3636
val = '0' * 100
3737

38-
def __array__(self, result=None, copy=None):
39-
return np.array([self.val], dtype=object)
38+
def __array__(self, dtype=None, copy=None):
39+
return np.array([self.val], dtype=dtype, copy=copy)
4040

4141

4242
wrapped = Wrapper()

0 commit comments

Comments
 (0)
0