8000 DEP: Futurewarn on requiring __len__ on array-likes · charris/numpy@d3104b9 · GitHub
[go: up one dir, main page]
Skip to content

Commit d3104b9

Browse files
sebergcharris
seberg
authored and
charris
committed
DEP: Futurewarn on requiring __len__ on array-likes
This fixes issue numpygh-17965. The slightly annoying thing is that there is no simple way to opt-in to the new behaviour and the old behaviour is a bit quirky to begin with (honoring the dtype, but not the shape).
1 parent 03cb000 commit d3104b9

File tree

4 files changed

+192
-6
lines changed

4 files changed

+192
-6
lines changed

‎doc/source/release/1.20.0-notes.rst

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,43 @@ Use ``next(it)`` instead of ``it.ndincr()``.
184184

185185
(`gh-17233 <https://github.com/numpy/numpy/pull/17233>`__)
186186

187+
ArrayLike objects which do not define ``__len__`` and ``__getitem__``
188+
---------------------------------------------------------------------
189+
Objects which define one of the protocols ``__array__``,
190+
``__array_interface__``, or ``__array_struct__`` but are not sequences
191+
(usually defined by having a ``__len__`` and ``__getitem__``) will behave
192+
differently during array-coercion in the future.
193+
194+
When nested inside sequences, such as ``np.array([array_like])``, these
195+
were handled as a single Python object rather than an array.
196+
In the future they will behave identically to::
197+
198+
np.array([np.array(array_like)])
199+
200+
This change should only have an effect if ``np.array(array_like)`` is not 0-D.
201+
The solution to this warning may depend on the object:
202+
203+
* Some array-likes may expect the new behaviour, and users can ignore the
204+
warning. The object can choose to expose the sequence protocol to opt-in
205+
to the new behaviour.
206+
* For example, ``shapely`` will allow conversion to an array-like using
207+
``line.coords`` rather than ``np.asarray(line)``. Users may work around
208+
the warning, or use the new convention when it becomes available.
209+
210+
Unfortunately, using the new behaviour can only be achieved by
211+
calling ``np.array(array_like)``.
212+
213+
If you wish to ensure that the old behaviour remains unchanged, please create
214+
an object array and then fill it explicitly, for example::
215+
216+
arr = np.empty(3, dtype=object)
217+
arr[:] = [array_like1, array_like2, array_like3]
218+
219+
This will ensure NumPy knows to not enter the array-like and use it as
220+
a object instead.
221+
222+
(`gh-17973 <https://github.com/numpy/numpy/pull/17973>`__)
223+
187224

188225
Future Changes
189226
==============
@@ -349,9 +386,15 @@ Things will now be more consistent with::
349386

350387
np.array([np.array(array_like1)])
351388

352-
This could potentially subtly change output for badly defined array-likes.
353-
We are not aware of any such case where the results were not clearly
354-
incorrect previously.
389+
This can subtly change output for some badly defined array-likes.
390+
One example for this are array-like objects which are not also sequences
391+
of matching shape.
392+
In NumPy 1.20, a warning will be given when an array-like is not also a
393+
sequence (but behaviour remains identical, see deprecations).
394+
If an array like is also a sequence (defines ``__getitem__`` and ``__len__``)
395+
NumPy will now only use the result given by ``__array__``,
396+
``__array_interface__``, or ``__array_struct__``. This will result in
397+
differences when the (nested) sequence describes a different shape.
355398

356399
(`gh-16200 <https://github.com/numpy/numpy/pull/16200>`__)
357400

‎numpy/core/src/multiarray/array_coercion.c

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -922,6 +922,53 @@ PyArray_DiscoverDTypeAndShape_Recursive(
922922
Py_DECREF(arr);
923923
arr = NULL;
924924
}
925+
else if (curr_dims > 0 && curr_dims != max_dims) {
926+
/*
927+
* Deprecated 2020-12-09, NumPy 1.20
928+
*
929+
* See https://github.com/numpy/numpy/issues/17965
930+
* Shapely had objects which are not sequences but did export
931+
* the array-interface (and so are arguably array-like).
932+
* Previously numpy would not use array-like information during
933+
* shape discovery, so that it ended up acting as if this was
934+
* an (unknown) scalar but with the specified dtype.
935+
* Thus we ignore "scalars" here, as the value stored in the
936+
* array should be acceptable.
937+
*/
938+
if (PyArray_NDIM(arr) > 0 && NPY_UNLIKELY(!PySequence_Check(obj))) {
939+
if (PyErr_WarnFormat(PyExc_FutureWarning, 1,
940+
"The input object of type '%s' is an array-like "
941+
"implementing one of the corresponding protocols "
942+
"(`__array__`, `__array_interface__` or "
943+
"`__array_struct__`); but not a sequence (or 0-D). "
944+
"In the future, this object will be coerced as if it "
945+
"was first converted using `np.array(obj)`. "
946+
"To retain the old behaviour, you have to either "
947+
"modify the type '%s', or assign to an empty array "
948+
"created with `np.empty(correct_shape, dtype=object)`.",
949+
Py_TYPE(obj)->tp_name, Py_TYPE(obj)->tp_name) < 0) {
950+
Py_DECREF(arr);
951+
return -1;
952+
}
953+
/*
954+
* Strangely enough, even though we threw away the result here,
955+
* we did use it during descriptor discovery, so promote it:
956+
*/
957+
if (update_shape(curr_dims, &max_dims, out_shape,
958+
0, NULL, NPY_FALSE, flags) < 0) {
959+
*flags |= FOUND_RAGGED_ARRAY;
960+
Py_DECREF(arr);
961+
return max_dims;
962+
}
963+
if (!(*flags & DESCRIPTOR_WAS_SET) && handle_promotion(
964+
out_descr, PyArray_DESCR(arr), fixed_DType, flags) < 0) {
965+
Py_DECREF(arr);
966+
return -1;
967+
}
968+
Py_DECREF(arr);
969+
return max_dims;
970+
}
971+
}
925972
}
926973
if (arr != NULL) {
927974
/*

‎numpy/core/tests/test_array_coercion.py

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,18 @@ def subclass(a):
3838

3939
yield subclass
4040

41+
class _SequenceLike():
42+
# We are giving a warning that array-like's were also expected to be
43+
# sequence-like in `np.array([array_like])`, this can be removed
44+
# when the deprecation exired (started NumPy 1.20)
45+
def __len__(self):
46+
raise TypeError
47+
48+
def __getitem__(self):
49+
raise TypeError
50+
4151
# Array-interface
42-
class ArrayDunder:
52+
class ArrayDunder(_SequenceLike):
4353
def __init__(self, a):
4454
self.a = a
4555

@@ -52,15 +62,15 @@ def __array__(self, dtype=None):
5262
yield param(memoryview, id="memoryview")
5363

5464
# Array-interface
55-
class ArrayInterface:
65+
class ArrayInterface(_SequenceLike):
5666
def __init__(self, a):
5767
self.a = a # need to hold on to keep interface valid
5868
self.__array_interface__ = a.__array_interface__
5969

6070
yield param(ArrayInterface, id="__array_interface__")
6171

6272
# Array-Struct
63-
class ArrayStruct:
73+
class ArrayStruct(_SequenceLike):
6474
def __init__(self, a):
6575
self.a = a # need to hold on to keep struct valid
6676
self.__array_struct__ = a.__array_struct__

‎numpy/core/tests/test_deprecations.py

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -773,6 +773,92 @@ def check():
773773
self.assert_deprecated(check)
774774

775775

776+
class TestFutureWarningArrayLikeNotIterable(_DeprecationTestCase):
777+
# Deprecated 2020-12-09, NumPy 1.20
778+
warning_cls = FutureWarning
779+
message = "The input object of type.*but not a sequence"
780+
781+
@pytest.mark.parametrize("protocol",
782+
["__array__", "__array_interface__", "__array_struct__"])
783+
def test_deprecated(self, protocol):
784+
"""Test that these objects give a warning since they are not 0-D,
785+
not coerced at the top level `np.array(obj)`, but nested, and do
786+
*not* define the sequence protocol.
787+
788+
NOTE: Tests for the versions including __len__ and __getitem__ exist
789+
in `test_array_coercion.py` and they can be modified or ammended
790+
when this deprecation expired.
791+
"""
792+
blueprint = np.arange(10)
793+
MyArr = type("MyArr", (), {protocol: getattr(blueprint, protocol)})
794+
self.assert_deprecated(lambda: np.array([MyArr()], dtype=object))
795+
796+
@pytest.mark.parametrize("protocol",
797+
["__array__", "__array_interface__", "__array_struct__"])
798+
def test_0d_not_deprecated(self, protocol):
799+
# 0-D always worked (albeit it would use __float__ or similar for the
800+
# conversion, which may not happen anymore)
801+
blueprint = np.array(1.)
802+
MyArr = type("MyArr", (), {protocol: getattr(blueprint, protocol)})
803+
myarr = MyArr()
804+
805+
self.assert_not_deprecated(lambda: np.array([myarr], dtype=object))
806+
res = np.array([myarr], dtype=object)
807+
expected = np.empty(1, dtype=object)
808+
expected[0] = myarr
809+
assert_array_equal(res, expected)
810+
811+
@pytest.mark.parametrize("protocol",
812+
["__array__", "__array_interface__", "__array_struct__"])
813+
def test_unnested_not_deprecated(self, protocol):
814+
blueprint = np.arange(10)
815+
MyArr = type("MyArr", (), {protocol: getattr(blueprint, protocol)})
816+
myarr = MyArr()
817+
818+
self.assert_not_deprecated(lambda: np.array(myarr))
819+
res = np.array(myarr)
820+
assert_array_equal(res, blueprint)
821+
822+
@pytest.mark.parametrize("protocol",
823+
["__array__", "__array_interface__", "__array_struct__"])
824+
def test_strange_dtype_handling(self, protocol):
825+
"""The old code would actually use the dtype from the array, but
826+
then end up not using the array (for dimension discovery)
827+
"""
828+
blueprint = np.arange(10).astype("f4")
829+
MyArr = type("MyArr", (), {protocol: getattr(blueprint, protocol),
830+
"__float__": lambda _: 0.5})
831+
myarr = MyArr()
832+
833+
# Make sure we warn (and capture the FutureWarning)
834+
with pytest.warns(FutureWarning, match=self.message):
835+
res = np.array([[myarr]])
836+
837+
assert res.shape == (1, 1)
838+
assert res.dtype == "f4"
839+
assert res[0, 0] == 0.5
840+
841+
@pytest.mark.parametrize("protocol",
842+
["__array__", "__array_interface__", "__array_struct__"])
843+
def test_assignment_not_deprecated(self, protocol):
844+
# If the result is dtype=object we do not unpack a nested array or
845+
# array-like, if it is nested at exactly the right depth.
846+
# NOTE: We actually do still call __array__, etc. but ignore the result
847+
# in the end. For `dtype=object` we could optimize that away.
848+
blueprint = np.arange(10).astype("f4")
849+
MyArr = type("MyArr", (), {protocol: getattr(blueprint, protocol),
850+
"__float__": lambda _: 0.5})
851+
myarr = MyArr()
852+
853+
res = np.empty(3, dtype=object)
854+
def set():
855+
res[:] = [myarr, myarr, myarr]
856+
self.assert_not_deprecated(set)
857+
assert res[0] is myarr
858+
assert res[1] is myarr
859+
assert res[2] is myarr
860+
861+
776862
class TestDeprecatedUnpickleObjectScalar(_DeprecationTestCase):
777863
# Deprecated 2020-11-24, NumPy 1.20
778864
"""

0 commit comments

Comments
 (0)
0