From de84ba263abc8472f37aa8e08a26da84cdbd0d2a Mon Sep 17 00:00:00 2001 From: Allan Haldane Date: Mon, 27 Jul 2015 21:22:03 -0400 Subject: [PATCH] MAINT: Add deprecation warning to multi-field views/assignment Behavior of multi-field indexes will change in 1.13: Multi-field indexes will return a view (not a copy) and assignment between structures with non-identical fieldnames occurs "by position" (not "by fieldname"): >>> a = zeros(10, dtype=[('x', 'i8'), ('y', 'i8'), ('z', 'i8')]) >>> a[['x', 'z']].view('i4') # Deprecation warning for multifield view >>> b = ones(10, dtype=[('y', 'i4'), ('x', 'f4')]) >>> a[:] = b # Deprecation warning for multifield assignment --- doc/release/1.12.0-notes.rst | 27 +++++++++++++ numpy/core/src/multiarray/convert.c | 14 +++++++ numpy/core/src/multiarray/dtype_transfer.c | 25 ++++++++++++ numpy/core/src/multiarray/mapping.c | 4 ++ numpy/core/src/multiarray/methods.c | 3 ++ numpy/core/tests/test_multiarray.py | 46 ++++++++++++++++------ numpy/core/tests/test_nditer.py | 43 ++++++++++++-------- numpy/core/tests/test_records.py | 20 +++++++--- numpy/ma/tests/test_core.py | 6 ++- 9 files changed, 152 insertions(+), 36 deletions(-) diff --git a/doc/release/1.12.0-notes.rst b/doc/release/1.12.0-notes.rst index f7f3b4e555ec..5ae84a160c52 100644 --- a/doc/release/1.12.0-notes.rst +++ b/doc/release/1.12.0-notes.rst @@ -29,6 +29,33 @@ Future Changes other numpy functions such as np.mean. In particular, this means calls which returned a scalar may return a 0-d subclass object instead. +Multiple-field manipulation of structured arrays +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +In 1.13 the behavior of structured arrays involving multiple fields will change +in two ways: + +First, indexing a structured array with multiple fields (eg, +``arr[['f1', 'f3']]``) will return a view into the original array in 1.13, +instead of a copy. Note the returned view will have extra padding bytes +corresponding to intervening fields in the original array, unlike the copy in +1.12, which will affect code such as ``arr[['f1', 'f3']].view(newdtype)``. + +Second, for numpy versions 1.6 to 1.12 assignment between structured arrays +occurs "by field name": Fields in the dst array are set to the +identically-named field in the src or to 0 if the src does not have a field: + + >>> a = np.array([(1,2),(3,4)], dtype=[('x', 'i4'), ('y', 'i4')]) + >>> b = np.ones(2, dtype=[('z', 'i4'), ('y', 'i4'), ('x', 'i4')]) + >>> b[:] = a + >>> b + array([(0, 2, 1), (0, 4, 3)], + dtype=[('z', 'names, + dst_dtype->names, Py_EQ); + if (PyErr_Occurred()) { + return NPY_FAIL; + } + if (cmpval != 1) { + if (DEPRECATE_FUTUREWARNING(msg) < 0) { + return NPY_FAIL; + } + } names = dst_dtype->names; names_size = PyTuple_GET_SIZE(dst_dtype->names); diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index 3d33f8a85089..28e69b94e48e 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -1428,6 +1428,7 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view) return 0; } + PyArray_CLEARFLAGS(*view, NPY_ARRAY_WARN_ON_WRITE); viewcopy = PyObject_CallFunction(copyfunc, "O", *view); if (viewcopy == NULL) { Py_DECREF(*view); @@ -1436,6 +1437,9 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view) } Py_DECREF(*view); *view = (PyArrayObject*)viewcopy; + + /* warn when writing to the copy */ + PyArray_ENABLEFLAGS(*view, NPY_ARRAY_WARN_ON_WRITE); return 0; } return -1; diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index cb8fea2134ef..5165a074b8b0 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1417,6 +1417,9 @@ array_deepcopy(PyArrayObject *self, PyObject *args) return NULL; } ret = (PyArrayObject *)PyArray_NewCopy(self, NPY_KEEPORDER); + if (ret == NULL) { + return NULL; + } if (PyDataType_REFCHK(PyArray_DESCR(self))) { copy = PyImport_ImportModule("copy"); if (copy == NULL) { diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 0f35812250f6..cdfe87579cb5 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -27,7 +27,7 @@ test_inplace_increment, get_buffer_info, test_as_c_array, ) from numpy.testing import ( - TestCase, run_module_suite, assert_, assert_raises, + TestCase, run_module_suite, assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal, assert_array_equal, assert_array_almost_equal, assert_allclose, IS_PYPY, HAS_REFCOUNT, assert_array_less, runstring, dec, SkipTest, temppath, suppress_warnings @@ -864,8 +864,10 @@ def test_casting(self): # Check that 'equiv' casting can reorder fields and change byte # order + # New in 1.12: This behavior changes in 1.13, test for dep warning assert_(np.can_cast(a.dtype, b.dtype, casting='equiv')) - c = a.astype(b.dtype, casting='equiv') + with assert_warns(FutureWarning): + c = a.astype(b.dtype, casting='equiv') assert_equal(a == c, [True, True]) # Check that 'safe' casting can change byte order and up-cast @@ -4241,14 +4243,23 @@ def test_field_names(self): # multiple subfields fn2 = func('f2') b[fn2] = 3 - assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3)) - assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2)) - assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,))) - # view of subfield view/copy - assert_equal(b[['f1', 'f2']][0].view(('i4', 2)).tolist(), (2, 3)) - assert_equal(b[['f2', 'f1']][0].view(('i4', 2)).tolist(), (3, 2)) - view_dtype = [('f1', 'i4'), ('f3', [('', 'i4')])] - assert_equal(b[['f1', 'f3']][0].view(view_dtype).tolist(), (2, (1,))) + with suppress_warnings() as sup: + sup.filter(FutureWarning, + "Assignment between structured arrays.*") + sup.filter(FutureWarning, + "Numpy has detected that you .*") + + assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3)) + assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2)) + assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,))) + # view of subfield view/copy + assert_equal(b[['f1', 'f2']][0].view(('i4', 2)).tolist(), + (2, 3)) + assert_equal(b[['f2', 'f1']][0].view(('i4', 2)).tolist(), + (3, 2)) + view_dtype = [('f1', 'i4'), ('f3', [('', 'i4')])] + assert_equal(b[['f1', 'f3']][0].view(view_dtype).tolist(), + (2, (1,))) # non-ascii unicode field indexing is well behaved if not is_py3: raise SkipTest('non ascii unicode field indexing skipped; ' @@ -4278,11 +4289,12 @@ def collect_warnings(f, *args, **kwargs): b['f2'][0] = 2 b['f3'][0] = (3,) - # All the different functions raise a warning, but not an error, and - # 'a' is not modified: + # All the different functions raise a warning, but not an error assert_equal(collect_warnings(a[['f1', 'f2']].__setitem__, 0, (10, 20)), [FutureWarning]) + # For <=1.12 a is not modified, but it will be in 1.13 assert_equal(a, b) + # Views also warn subset = a[['f1', 'f2']] subset_view = subset.view() @@ -4294,6 +4306,16 @@ def collect_warnings(f, *args, **kwargs): # are multiple views involved): assert_equal(collect_warnings(subset['f1'].__setitem__, 0, 10), []) + # make sure views of a multi-field index warn too + c = np.zeros(3, dtype='i8,i8,i8') + assert_equal(collect_warnings(c[['f0', 'f2']].view, 'i8,i8'), + [FutureWarning]) + + # make sure assignment using a different dtype warns + a = np.zeros(2, dtype=[('a', 'i4'), ('b', 'i4')]) + b = np.zeros(2, dtype=[('b', 'i4'), ('a', 'i4')]) + assert_equal(collect_warnings(a.__setitem__, (), b), [FutureWarning]) + def test_record_hash(self): a = np.array([(1, 2), (1, 2)], dtype='i1,i2') a.flags.writeable = False diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index 51c3d62864f7..f5096e023f13 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -1,6 +1,7 @@ from __future__ import division, absolute_import, print_function import sys +import warnings import numpy as np from numpy import array, arange, nditer, all @@ -8,7 +9,7 @@ from numpy.core.multiarray_tests import test_nditer_too_large from numpy.testing import ( run_module_suite, assert_, assert_equal, assert_array_equal, - assert_raises, dec, HAS_REFCOUNT, suppress_warnings + assert_raises, assert_warns, dec, HAS_REFCOUNT, suppress_warnings ) @@ -1740,9 +1741,11 @@ def test_iter_buffered_cast_structured_type(): sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', 'O')] sdt2 = [('d', 'u2'), ('a', 'O'), ('b', 'f8')] a = np.array([(1, 2, 3), (4, 5, 6)], dtype=sdt1) - i = nditer(a, ['buffered', 'refs_ok'], ['readonly'], - casting='unsafe', - op_dtypes=sdt2) + # New in 1.12: This behavior changes in 1.13, test for dep warning + with assert_warns(FutureWarning): + i = nditer(a, ['buffered', 'refs_ok'], ['readonly'], + casting='unsafe', + op_dtypes=sdt2) assert_equal(i[0].dtype, np.dtype(sdt2)) assert_equal([np.array(x_) for x_ in i], [np.array((3, 1, 2), dtype=sdt2), @@ -1752,9 +1755,11 @@ def test_iter_buffered_cast_structured_type(): sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', 'O')] sdt2 = [('b', 'O'), ('a', 'f8')] a = np.array([(1, 2, 3), (4, 5, 6)], dtype=sdt1) - i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'], - casting='unsafe', - op_dtypes=sdt2) + # New in 1.12: This behavior changes in 1.13, test for dep warning + with assert_warns(FutureWarning): + i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'], + casting='unsafe', + op_dtypes=sdt2) assert_equal(i[0].dtype, np.dtype(sdt2)) vals = [] for x in i: @@ -1768,9 +1773,11 @@ def test_iter_buffered_cast_structured_type(): sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', [('a', 'i2'), ('b', 'i4')])] sdt2 = [('b', 'O'), ('a', 'f8')] a = np.array([(1, 2, (0, 9)), (4, 5, (20, 21))], dtype=sdt1) - i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'], - casting='unsafe', - op_dtypes=sdt2) + # New in 1.12: This behavior changes in 1.13, test for dep warning + with assert_warns(FutureWarning): + i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'], + casting='unsafe', + op_dtypes=sdt2) assert_equal(i[0].dtype, np.dtype(sdt2)) vals = [] for x in i: @@ -1784,9 +1791,11 @@ def test_iter_buffered_cast_structured_type(): sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', [('a', 'i2'), ('b', 'O')])] sdt2 = [('b', 'O'), ('a', 'f8')] a = np.array([(1, 2, (0, 9)), (4, 5, (20, 21))], dtype=sdt1) - i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'], - casting='unsafe', - op_dtypes=sdt2) + # New in 1.12: This behavior changes in 1.13, test for dep warning + with assert_warns(FutureWarning): + i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'], + casting='unsafe', + op_dtypes=sdt2) assert_equal(i[0].dtype, np.dtype(sdt2)) vals = [] for x in i: @@ -1800,9 +1809,11 @@ def test_iter_buffered_cast_structured_type(): sdt1 = [('b', 'O'), ('a', 'f8')] sdt2 = [('a', 'f4'), ('b', 'i8'), ('d', [('a', 'i2'), ('b', 'O')])] a = np.array([(1, 2), (4, 5)], dtype=sdt1) - i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'], - casting='unsafe', - op_dtypes=sdt2) + # New in 1.12: This behavior changes in 1.13, test for dep warning + with assert_warns(FutureWarning): + i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'], + casting='unsafe', + op_dtypes=sdt2) assert_equal(i[0].dtype, np.dtype(sdt2)) vals = [] for x in i: diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py index 6ef6badda89c..c4360bcf2e36 100644 --- a/numpy/core/tests/test_records.py +++ b/numpy/core/tests/test_records.py @@ -3,13 +3,14 @@ import sys import collections import pickle +import warnings from os import path import numpy as np from numpy.compat import asbytes from numpy.testing import ( TestCase, run_module_suite, assert_, assert_equal, assert_array_equal, - assert_array_almost_equal, assert_raises + assert_array_almost_equal, assert_raises, assert_warns ) @@ -126,8 +127,11 @@ def test_recarray_views(self): ('c', 'i4,i4')])) assert_equal(r['c'].dtype.type, np.record) assert_equal(type(r['c']), np.recarray) - assert_equal(r[['a', 'b']].dtype.type, np.record) - assert_equal(type(r[['a', 'b']]), np.recarray) + + # suppress deprecation warning in 1.12 (remove in 1.13) + with assert_warns(FutureWarning): + assert_equal(r[['a', 'b']].dtype.type, np.record) + assert_equal(type(r[['a', 'b']]), np.recarray) #and that it preserves subclasses (gh-6949) class C(np.recarray): @@ -298,8 +302,11 @@ def assign_invalid_column(x): def test_out_of_order_fields(self): """Ticket #1431.""" - x = self.data[['col1', 'col2']] - y = self.data[['col2', 'col1']] + # this test will be invalid in 1.13 + # suppress deprecation warning in 1.12 (remove in 1.13) + with assert_warns(FutureWarning): + x = self.data[['col1', 'col2']] + y = self.data[['col2', 'col1']] assert_equal(x[0][0], y[0][1]) def test_pickle_1(self): @@ -330,7 +337,8 @@ def test_objview_record(self): # https://github.com/numpy/numpy/issues/3256 ra = np.recarray((2,), dtype=[('x', object), ('y', float), ('z', int)]) - ra[['x','y']] # TypeError? + with assert_warns(FutureWarning): + ra[['x','y']] # TypeError? def test_record_scalar_setitem(self): # https://github.com/numpy/numpy/issues/3561 diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 338a6d0dc9d6..42676978ff25 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -20,7 +20,7 @@ import numpy.core.fromnumeric as fromnumeric import numpy.core.umath as umath from numpy.testing import ( - TestCase, run_module_suite, assert_raises, suppress_warnings) + TestCase, run_module_suite, assert_raises, assert_warns, suppress_warnings) from numpy import ndarray from numpy.compat import asbytes, asbytes_nested from numpy.ma.testutils import ( @@ -1629,7 +1629,9 @@ def test_check_on_fields(self): # BEHAVIOR in 1.6 and later: match structured types by name fill_val = np.array(("???", -999, -12345678.9), dtype=[("c", "|S3"), ("a", int), ("b", float), ]) - fval = _check_fill_value(fill_val, ndtype) + # suppress deprecation warning in 1.12 (remove in 1.13) + with assert_warns(FutureWarning): + fval = _check_fill_value(fill_val, ndtype) self.assertTrue(isinstance(fval, ndarray)) assert_equal(fval.item(), [-999, -12345678.9, asbytes("???")])