numpy · charris · Sep 24, 2016 · Jul 28, 2015
diff --git a/doc/release/1.12.0-notes.rst b/doc/release/1.12.0-notes.rst
@@ -29,6 +29,33 @@ Future Changes
   other numpy functions such as np.mean. In particular, this means calls which
   returned a scalar may return a 0-d subclass object instead.
 
+Multiple-field manipulation of structured arrays
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+In 1.13 the behavior of structured arrays involving multiple fields will change
+in two ways:
+
+First, indexing a structured array with multiple fields (eg,
+``arr[['f1', 'f3']]``) will return a view into the original array in 1.13,
+instead of a copy. Note the returned view will have extra padding bytes
+corresponding to intervening fields in the original array, unlike the copy in
+1.12, which will affect code such as ``arr[['f1', 'f3']].view(newdtype)``.
+
+Second, for numpy versions 1.6 to 1.12 assignment between structured arrays
+occurs "by field name": Fields in the dst array are set to the
+identically-named field in the src or to 0 if the src does not have a field:
+
+    >>> a = np.array([(1,2),(3,4)], dtype=[('x', 'i4'), ('y', 'i4')])
+    >>> b = np.ones(2, dtype=[('z', 'i4'), ('y', 'i4'), ('x', 'i4')])
+    >>> b[:] = a
+    >>> b
+    array([(0, 2, 1), (0, 4, 3)],
+          dtype=[('z', '<i4'), ('y', '<i4'), ('x', '<i4')])
+
+In 1.13 assignment will instead occur "by position": The Nth field of the dst
+will be set to the Nth field of the src, regardless of field name. The old
+behavior can be obtained by using indexing to reorder the fields before
+assignment, eg ``b[['x', 'y']] = a[['y', 'x']]``.
+
 
 Compatibility notes
 ===================

diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c
@@ -601,6 +601,20 @@ PyArray_View(PyArrayObject *self, PyArray_Descr *type, PyTypeObject *pytype)
         subtype = Py_TYPE(self);
     }
 
+    if (type != NULL && (PyArray_FLAGS(self) & NPY_ARRAY_WARN_ON_WRITE)) {
+        const char *msg =
+            "Numpy has detected that you may be viewing or writing to an array "
+            "returned by selecting multiple fields in a structured array. \n\n"
+            "This code may break in numpy 1.13 because this will return a view "
+            "instead of a copy -- see release notes for details.";
+        /* 2016-09-19, 1.12 */
+        if (DEPRECATE_FUTUREWARNING(msg) < 0) {
+            return NULL;
+        }
+        /* Only warn once per array */
+        PyArray_CLEARFLAGS(self, NPY_ARRAY_WARN_ON_WRITE);
+    }
+
     flags = PyArray_FLAGS(self);
 
     dtype = PyArray_DESCR(self);

diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c
@@ -2746,6 +2746,31 @@ get_fields_transfer_function(int aligned,
     else {
         /* Keeps track of the names we already used */
         PyObject *used_names_dict = NULL;
+        int cmpval;
+
+        const char *msg =
+            "Assignment between structured arrays with different field names "
+            "will change in numpy 1.13.\n\n"
+            "Previously fields in the dst would be set to the value of the "
+            "identically-named field in the src. In numpy 1.13 fields will "
+            "instead be assigned 'by position': The Nth field of the dst "
+            "will be set to the Nth field of the src array.\n\n"
+            "See the release notes for details";
+        /*
+         * 2016-09-19, 1.12
+         * Warn if the field names of the dst and src are not
+         * identical, since then behavior will change in 1.13.
+         */
+        cmpval = PyObject_RichCompareBool(src_dtype->names,
+                                          dst_dtype->names, Py_EQ);
+        if (PyErr_Occurred()) {
+            return NPY_FAIL;
+        }
+        if (cmpval != 1) {
+            if (DEPRECATE_FUTUREWARNING(msg) < 0) {
+                return NPY_FAIL;
+            }
+        }
 
         names = dst_dtype->names;
         names_size = PyTuple_GET_SIZE(dst_dtype->names);

diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c
@@ -1428,6 +1428,7 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view)
             return 0;
         }
 
+        PyArray_CLEARFLAGS(*view, NPY_ARRAY_WARN_ON_WRITE);
         viewcopy = PyObject_CallFunction(copyfunc, "O", *view);
         if (viewcopy == NULL) {
             Py_DECREF(*view);
@@ -1436,6 +1437,9 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view)
         }
         Py_DECREF(*view);
         *view = (PyArrayObject*)viewcopy;
+
+        /* warn when writing to the copy */
+        PyArray_ENABLEFLAGS(*view, NPY_ARRAY_WARN_ON_WRITE);
         return 0;
     }
     return -1;

diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c
@@ -1417,6 +1417,9 @@ array_deepcopy(PyArrayObject *self, PyObject *args)
         return NULL;
     }
     ret = (PyArrayObject *)PyArray_NewCopy(self, NPY_KEEPORDER);
+    if (ret == NULL) {
+        return NULL;
+    }
     if (PyDataType_REFCHK(PyArray_DESCR(self))) {
         copy = PyImport_ImportModule("copy");
         if (copy == NULL) {

diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
@@ -27,7 +27,7 @@
     test_inplace_increment, get_buffer_info, test_as_c_array,
     )
 from numpy.testing import (
-    TestCase, run_module_suite, assert_, assert_raises,
+    TestCase, run_module_suite, assert_, assert_raises, assert_warns,
     assert_equal, assert_almost_equal, assert_array_equal,
     assert_array_almost_equal, assert_allclose, IS_PYPY, HAS_REFCOUNT,
     assert_array_less, runstring, dec, SkipTest, temppath, suppress_warnings
@@ -864,8 +864,10 @@ def test_casting(self):
 
         # Check that 'equiv' casting can reorder fields and change byte
         # order
+        # New in 1.12: This behavior changes in 1.13, test for dep warning
         assert_(np.can_cast(a.dtype, b.dtype, casting='equiv'))
-        c = a.astype(b.dtype, casting='equiv')
+        with assert_warns(FutureWarning):
+            c = a.astype(b.dtype, casting='equiv')
         assert_equal(a == c, [True, True])
 
         # Check that 'safe' casting can change byte order and up-cast
@@ -4241,14 +4243,23 @@ def test_field_names(self):
             # multiple subfields
             fn2 = func('f2')
             b[fn2] = 3
-            assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3))
-            assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2))
-            assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,)))
-            # view of subfield view/copy
-            assert_equal(b[['f1', 'f2']][0].view(('i4', 2)).tolist(), (2, 3))
-            assert_equal(b[['f2', 'f1']][0].view(('i4', 2)).tolist(), (3, 2))
-            view_dtype = [('f1', 'i4'), ('f3', [('', 'i4')])]
-            assert_equal(b[['f1', 'f3']][0].view(view_dtype).tolist(), (2, (1,)))
+            with suppress_warnings() as sup:
+                sup.filter(FutureWarning,
+                           "Assignment between structured arrays.*")
+                sup.filter(FutureWarning,
+                           "Numpy has detected that you .*")
+
+                assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3))
+                assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2))
+                assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,)))
+                # view of subfield view/copy
+                assert_equal(b[['f1', 'f2']][0].view(('i4', 2)).tolist(),
+                             (2, 3))
+                assert_equal(b[['f2', 'f1']][0].view(('i4', 2)).tolist(),
+                             (3, 2))
+                view_dtype = [('f1', 'i4'), ('f3', [('', 'i4')])]
+                assert_equal(b[['f1', 'f3']][0].view(view_dtype).tolist(),
+                             (2, (1,)))
         # non-ascii unicode field indexing is well behaved
         if not is_py3:
             raise SkipTest('non ascii unicode field indexing skipped; '
@@ -4278,11 +4289,12 @@ def collect_warnings(f, *args, **kwargs):
         b['f2'][0] = 2
         b['f3'][0] = (3,)
 
-        # All the different functions raise a warning, but not an error, and
-        # 'a' is not modified:
+        # All the different functions raise a warning, but not an error
         assert_equal(collect_warnings(a[['f1', 'f2']].__setitem__, 0, (10, 20)),
                      [FutureWarning])
+        # For <=1.12 a is not modified, but it will be in 1.13
         assert_equal(a, b)
+
         # Views also warn
         subset = a[['f1', 'f2']]
         subset_view = subset.view()
@@ -4294,6 +4306,16 @@ def collect_warnings(f, *args, **kwargs):
         # are multiple views involved):
         assert_equal(collect_warnings(subset['f1'].__setitem__, 0, 10), [])
 
+        # make sure views of a multi-field index warn too
+        c = np.zeros(3, dtype='i8,i8,i8')
+        assert_equal(collect_warnings(c[['f0', 'f2']].view, 'i8,i8'),
+                     [FutureWarning])
+
+        # make sure assignment using a different dtype warns
+        a = np.zeros(2, dtype=[('a', 'i4'), ('b', 'i4')])
+        b = np.zeros(2, dtype=[('b', 'i4'), ('a', 'i4')])
+        assert_equal(collect_warnings(a.__setitem__, (), b), [FutureWarning])
+
     def test_record_hash(self):
         a = np.array([(1, 2), (1, 2)], dtype='i1,i2')
         a.flags.writeable = False

diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py
@@ -1,14 +1,15 @@
 from __future__ import division, absolute_import, print_function
 
 import sys
+import warnings
 
 import numpy as np
 from numpy import array, arange, nditer, all
 from numpy.compat import asbytes, sixu
 from numpy.core.multiarray_tests import test_nditer_too_large
 from numpy.testing import (
     run_module_suite, assert_, assert_equal, assert_array_equal,
-    assert_raises, dec, HAS_REFCOUNT, suppress_warnings
+    assert_raises, assert_warns, dec, HAS_REFCOUNT, suppress_warnings
     )
 
 
@@ -1740,9 +1741,11 @@ def test_iter_buffered_cast_structured_type():
     sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', 'O')]
     sdt2 = [('d', 'u2'), ('a', 'O'), ('b', 'f8')]
     a = np.array([(1, 2, 3), (4, 5, 6)], dtype=sdt1)
-    i = nditer(a, ['buffered', 'refs_ok'], ['readonly'],
-                    casting='unsafe',
-                    op_dtypes=sdt2)
+    # New in 1.12: This behavior changes in 1.13, test for dep warning
+    with assert_warns(FutureWarning):
+        i = nditer(a, ['buffered', 'refs_ok'], ['readonly'],
+                        casting='unsafe',
+                        op_dtypes=sdt2)
     assert_equal(i[0].dtype, np.dtype(sdt2))
     assert_equal([np.array(x_) for x_ in i],
                  [np.array((3, 1, 2), dtype=sdt2),
@@ -1752,9 +1755,11 @@ def test_iter_buffered_cast_structured_type():
     sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', 'O')]
     sdt2 = [('b', 'O'), ('a', 'f8')]
     a = np.array([(1, 2, 3), (4, 5, 6)], dtype=sdt1)
-    i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
-                    casting='unsafe',
-                    op_dtypes=sdt2)
+    # New in 1.12: This behavior changes in 1.13, test for dep warning
+    with assert_warns(FutureWarning):
+        i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
+                        casting='unsafe',
+                        op_dtypes=sdt2)
     assert_equal(i[0].dtype, np.dtype(sdt2))
     vals = []
     for x in i:
@@ -1768,9 +1773,11 @@ def test_iter_buffered_cast_structured_type():
     sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', [('a', 'i2'), ('b', 'i4')])]
     sdt2 = [('b', 'O'), ('a', 'f8')]
     a = np.array([(1, 2, (0, 9)), (4, 5, (20, 21))], dtype=sdt1)
-    i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
-                    casting='unsafe',
-                    op_dtypes=sdt2)
+    # New in 1.12: This behavior changes in 1.13, test for dep warning
+    with assert_warns(FutureWarning):
+        i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
+                        casting='unsafe',
+                        op_dtypes=sdt2)
     assert_equal(i[0].dtype, np.dtype(sdt2))
     vals = []
     for x in i:
@@ -1784,9 +1791,11 @@ def test_iter_buffered_cast_structured_type():
     sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', [('a', 'i2'), ('b', 'O')])]
     sdt2 = [('b', 'O'), ('a', 'f8')]
     a = np.array([(1, 2, (0, 9)), (4, 5, (20, 21))], dtype=sdt1)
-    i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
-                    casting='unsafe',
-                    op_dtypes=sdt2)
+    # New in 1.12: This behavior changes in 1.13, test for dep warning
+    with assert_warns(FutureWarning):
+        i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
+                        casting='unsafe',
+                        op_dtypes=sdt2)
     assert_equal(i[0].dtype, np.dtype(sdt2))
     vals = []
     for x in i:
@@ -1800,9 +1809,11 @@ def test_iter_buffered_cast_structured_type():
     sdt1 = [('b', 'O'), ('a', 'f8')]
     sdt2 = [('a', 'f4'), ('b', 'i8'), ('d', [('a', 'i2'), ('b', 'O')])]
     a = np.array([(1, 2), (4, 5)], dtype=sdt1)
-    i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
-                    casting='unsafe',
-                    op_dtypes=sdt2)
+    # New in 1.12: This behavior changes in 1.13, test for dep warning
+    with assert_warns(FutureWarning):
+        i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
+                        casting='unsafe',
+                        op_dtypes=sdt2)
     assert_equal(i[0].dtype, np.dtype(sdt2))
     vals = []
     for x in i:

diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py
@@ -3,13 +3,14 @@
 import sys
 import collections
 import pickle
+import warnings
 from os import path
 
 import numpy as np
 from numpy.compat import asbytes
 from numpy.testing import (
     TestCase, run_module_suite, assert_, assert_equal, assert_array_equal,
-    assert_array_almost_equal, assert_raises
+    assert_array_almost_equal, assert_raises, assert_warns
     )
 
 
@@ -126,8 +127,11 @@ def test_recarray_views(self):
                                            ('c', 'i4,i4')]))
         assert_equal(r['c'].dtype.type, np.record)
         assert_equal(type(r['c']), np.recarray)
-        assert_equal(r[['a', 'b']].dtype.type, np.record)
-        assert_equal(type(r[['a', 'b']]), np.recarray)
+
+        # suppress deprecation warning in 1.12 (remove in 1.13)
+        with assert_warns(FutureWarning):
+            assert_equal(r[['a', 'b']].dtype.type, np.record)
+            assert_equal(type(r[['a', 'b']]), np.recarray)
 
         #and that it preserves subclasses (gh-6949)
         class C(np.recarray):
@@ -298,8 +302,11 @@ def assign_invalid_column(x):
 
     def test_out_of_order_fields(self):
         """Ticket #1431."""
-        x = self.data[['col1', 'col2']]
-        y = self.data[['col2', 'col1']]
+        # this test will be invalid in 1.13
+        # suppress deprecation warning in 1.12 (remove in 1.13)
+        with assert_warns(FutureWarning):
+            x = self.data[['col1', 'col2']]
+            y = self.data[['col2', 'col1']]
         assert_equal(x[0][0], y[0][1])
 
     def test_pickle_1(self):
@@ -330,7 +337,8 @@ def test_objview_record(self):
 
         # https://github.com/numpy/numpy/issues/3256
         ra = np.recarray((2,), dtype=[('x', object), ('y', float), ('z', int)])
-        ra[['x','y']]  # TypeError?
+        with assert_warns(FutureWarning):
+            ra[['x','y']]  # TypeError?
 
     def test_record_scalar_setitem(self):
         # https://github.com/numpy/numpy/issues/3561

diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py
@@ -20,7 +20,7 @@
 import numpy.core.fromnumeric as fromnumeric
 import numpy.core.umath as umath
 from numpy.testing import (
-    TestCase, run_module_suite, assert_raises, suppress_warnings)
+    TestCase, run_module_suite, assert_raises, assert_warns, suppress_warnings)
 from numpy import ndarray
 from numpy.compat import asbytes, asbytes_nested
 from numpy.ma.testutils import (
@@ -1629,7 +1629,9 @@ def test_check_on_fields(self):
         # BEHAVIOR in 1.6 and later: match structured types by name
         fill_val = np.array(("???", -999, -12345678.9),
                             dtype=[("c", "|S3"), ("a", int), ("b", float), ])
-        fval = _check_fill_value(fill_val, ndtype)
+        # suppress deprecation warning in 1.12 (remove in 1.13)
+        with assert_warns(FutureWarning):
+            fval = _check_fill_value(fill_val, ndtype)
         self.assertTrue(isinstance(fval, ndarray))
         assert_equal(fval.item(), [-999, -12345678.9, asbytes("???")])