From 496bd1a84a6a62adf5f8cb48cf7abc7795484531 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Sat, 5 Jun 2021 14:23:13 +0530 Subject: [PATCH 01/54] Added np.CopyMode --- numpy/__init__.py | 3 ++- numpy/_globals.py | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/numpy/__init__.py b/numpy/__init__.py index baff5e1417e6..690a8135c494 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -110,7 +110,8 @@ import warnings from ._globals import ( - ModuleDeprecationWarning, VisibleDeprecationWarning, _NoValue + ModuleDeprecationWarning, VisibleDeprecationWarning, + _NoValue, CopyMode ) # We first need to detect if we're being called as part of the numpy setup diff --git a/numpy/_globals.py b/numpy/_globals.py index 0b715c870870..a8f57f3b609f 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -15,8 +15,11 @@ def foo(arg=np._NoValue): motivated this module. """ +import enum + __ALL__ = [ - 'ModuleDeprecationWarning', 'VisibleDeprecationWarning', '_NoValue' + 'ModuleDeprecationWarning', 'VisibleDeprecationWarning', + '_NoValue', 'CopyMode' ] @@ -89,3 +92,19 @@ def __repr__(self): _NoValue = _NoValueType() + +class CopyMode(enum.Enum): + + ALWAYS = 1 + IF_NEEDED = 0 + NEVER = 2 + + def __bool__(self): + # For backwards compatiblity + if self == CopyMode.ALWAYS: + return True + + if self == CopyMode.IF_NEEDED: + return False + + raise TypeError(f"{self} is neither True nor False.") \ No newline at end of file From 3a3d31b338258c099fa4f2d4e4290b19b2a409b0 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Sat, 5 Jun 2021 15:08:40 +0530 Subject: [PATCH 02/54] initial work for supporting never_copy --- numpy/__init__.pyi | 3 ++- numpy/core/include/numpy/ndarraytypes.h | 2 ++ numpy/core/src/multiarray/conversion_utils.c | 5 +++++ numpy/core/src/multiarray/conversion_utils.h | 3 +++ numpy/core/src/multiarray/methods.c | 3 +-- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index ac37eb8ad409..e710aee76c9a 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -202,6 +202,7 @@ from numpy import ( rec as rec, testing as testing, version as version, + CopyMode as CopyMode ) from numpy.core.function_base import ( @@ -1232,7 +1233,7 @@ class _ArrayOrScalarCommon: order: _OrderKACF = ..., casting: _Casting = ..., subok: bool = ..., - copy: bool = ..., + copy: Union[bool, CopyMode[Any]] = ..., ) -> _ArraySelf: ... def copy(self: _ArraySelf, order: _OrderKACF = ...) -> _ArraySelf: ... def dump(self, file: str) -> None: ... diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index d1acfdf26235..569e6bb34c42 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -451,6 +451,8 @@ typedef struct { int len; } PyArray_Dims; +typedef enum {IF_NEEDED, ALWAYS, NEVER} PyNpCopyMode_Enum; + typedef struct { /* * Functions to cast to most other standard types diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 3c4c21dedd23..50ebea0e5ce8 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -162,6 +162,11 @@ PyArray_OptionalIntpConverter(PyObject *obj, PyArray_Dims *seq) return PyArray_IntpConverter(obj, seq); } +NPY_NO_EXPORT int +PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copymode) { + return 0; +} + /*NUMPY_API * Get buffer chunk from object * diff --git a/numpy/core/src/multiarray/conversion_utils.h b/numpy/core/src/multiarray/conversion_utils.h index 7d1871c43ddb..d2874aafa843 100644 --- a/numpy/core/src/multiarray/conversion_utils.h +++ b/numpy/core/src/multiarray/conversion_utils.h @@ -9,6 +9,9 @@ PyArray_IntpConverter(PyObject *obj, PyArray_Dims *seq); NPY_NO_EXPORT int PyArray_OptionalIntpConverter(PyObject *obj, PyArray_Dims *seq); +NPY_NO_EXPORT int +PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copyflag); + NPY_NO_EXPORT int PyArray_BufferConverter(PyObject *obj, PyArray_Chunk *buf); diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 251e527a6b96..892039508aa3 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -830,13 +830,12 @@ array_astype(PyArrayObject *self, NPY_ORDER order = NPY_KEEPORDER; int forcecopy = 1, subok = 1; NPY_PREPARE_ARGPARSER; - if (npy_parse_arguments("astype", args, len_args, kwnames, "dtype", &PyArray_DescrConverter, &dtype, "|order", &PyArray_OrderConverter, &order, "|casting", &PyArray_CastingConverter, &casting, "|subok", &PyArray_PythonPyIntFromInt, &subok, - "|copy", &PyArray_PythonPyIntFromInt, &forcecopy, + "|copy", &PyArray_CopyConverter, &forcecopy, NULL, NULL, NULL) < 0) { Py_XDECREF(dtype); return NULL; From be5823f603a37a5de2192f9772dca1b31cffb9be Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Mon, 7 Jun 2021 09:15:57 +0530 Subject: [PATCH 03/54] Addressed reviews and PyArray_CopyConverter defined --- numpy/__init__.py | 3 ++- numpy/__init__.pyi | 11 +++++++-- numpy/_globals.py | 6 +++-- numpy/core/include/numpy/ndarraytypes.h | 2 +- numpy/core/src/multiarray/conversion_utils.c | 25 +++++++++++++++++++- numpy/core/src/multiarray/methods.c | 2 +- 6 files changed, 41 insertions(+), 8 deletions(-) diff --git a/numpy/__init__.py b/numpy/__init__.py index 690a8135c494..c5cf6d753863 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -133,7 +133,8 @@ raise ImportError(msg) from e __all__ = ['ModuleDeprecationWarning', - 'VisibleDeprecationWarning'] + 'VisibleDeprecationWarning', + 'CopyMode'] # get the version using versioneer from ._version import get_versions diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index e710aee76c9a..a2d6ce435bed 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -2,6 +2,7 @@ import builtins import os import sys import datetime as dt +import enum from abc import abstractmethod from types import TracebackType from contextlib import ContextDecorator @@ -1233,7 +1234,7 @@ class _ArrayOrScalarCommon: order: _OrderKACF = ..., casting: _Casting = ..., subok: bool = ..., - copy: Union[bool, CopyMode[Any]] = ..., + copy: bool | CopyMode = ..., ) -> _ArraySelf: ... def copy(self: _ArraySelf, order: _OrderKACF = ...) -> _ArraySelf: ... def dump(self, file: str) -> None: ... @@ -3547,6 +3548,12 @@ trunc: _UFunc_Nin1_Nout1[L['trunc'], L[7], None] abs = absolute +class CopyMode(enum.IntEnum): + + ALWAYS: L[1] + IF_NEEDED: L[0] + NEVER: L[2] + # Warnings class ModuleDeprecationWarning(DeprecationWarning): ... class VisibleDeprecationWarning(UserWarning): ... @@ -3654,4 +3661,4 @@ class broadcast: def size(self) -> int: ... def __next__(self) -> Tuple[Any, ...]: ... def __iter__(self: _T) -> _T: ... - def reset(self) -> None: ... + def reset(self) -> None: ... \ No newline at end of file diff --git a/numpy/_globals.py b/numpy/_globals.py index a8f57f3b609f..a6fbec34067e 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -93,7 +93,7 @@ def __repr__(self): _NoValue = _NoValueType() -class CopyMode(enum.Enum): +class CopyMode(enum.IntEnum): ALWAYS = 1 IF_NEEDED = 0 @@ -107,4 +107,6 @@ def __bool__(self): if self == CopyMode.IF_NEEDED: return False - raise TypeError(f"{self} is neither True nor False.") \ No newline at end of file + raise TypeError(f"{self} is neither True nor False.") + +CopyMode.__module__ = 'numpy' \ No newline at end of file diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 569e6bb34c42..86dc19c64c98 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -451,7 +451,7 @@ typedef struct { int len; } PyArray_Dims; -typedef enum {IF_NEEDED, ALWAYS, NEVER} PyNpCopyMode_Enum; +typedef enum PyNpCopyMode {IF_NEEDED, ALWAYS, NEVER} PyNpCopyMode_Enum; typedef struct { /* diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 50ebea0e5ce8..984b0b94895d 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -17,6 +17,8 @@ #include "alloc.h" #include "npy_buffer.h" +#include "npy_argparse.h" + static int PyArray_PyIntAsInt_ErrMsg(PyObject *o, const char * msg) NPY_GCC_NONNULL(2); static npy_intp @@ -164,7 +166,28 @@ PyArray_OptionalIntpConverter(PyObject *obj, PyArray_Dims *seq) NPY_NO_EXPORT int PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copymode) { - return 0; + if (obj == Py_None) { + PyErr_SetString(PyExc_ValueError, + "NoneType copy mode not allowed. Please choose one of" + "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER"); + return NPY_FAIL; + } + + int int_copymode = -1; + PyArray_PythonPyIntFromInt(obj, &int_copymode); + + if( int_copymode != ALWAYS && + int_copymode != IF_NEEDED && + int_copymode != NEVER ) { + PyErr_Format(PyExc_ValueError, + "Unrecognized copy mode %d. Please choose one of" + "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER", + int_copymode); + return NPY_FAIL; + } + + *copymode = (PyNpCopyMode_Enum) int_copymode; + return NPY_SUCCEED; } /*NUMPY_API diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 892039508aa3..d4494d6e9584 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -828,7 +828,7 @@ array_astype(PyArrayObject *self, */ NPY_CASTING casting = NPY_UNSAFE_CASTING; NPY_ORDER order = NPY_KEEPORDER; - int forcecopy = 1, subok = 1; + PyNpCopyMode_Enum forcecopy = 1, subok = 1; NPY_PREPARE_ARGPARSER; if (npy_parse_arguments("astype", args, len_args, kwnames, "dtype", &PyArray_DescrConverter, &dtype, From e16f3ff524206723cff3517845a9e2d77277ac6d Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Mon, 7 Jun 2021 09:52:00 +0530 Subject: [PATCH 04/54] np.CopyMode.NEVER gives error if copy cannot be avoided in np.array.astype --- numpy/core/src/multiarray/methods.c | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index d4494d6e9584..cc161c4fcd95 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -852,20 +852,29 @@ array_astype(PyArrayObject *self, * and it's not a subtype if subok is False, then we * can skip the copy. */ - if (!forcecopy && (order == NPY_KEEPORDER || - (order == NPY_ANYORDER && - (PyArray_IS_C_CONTIGUOUS(self) || - PyArray_IS_F_CONTIGUOUS(self))) || - (order == NPY_CORDER && - PyArray_IS_C_CONTIGUOUS(self)) || - (order == NPY_FORTRANORDER && - PyArray_IS_F_CONTIGUOUS(self))) && - (subok || PyArray_CheckExact(self)) && - PyArray_EquivTypes(dtype, PyArray_DESCR(self))) { + if ( (forcecopy == IF_NEEDED || forcecopy == NEVER) && + (order == NPY_KEEPORDER || + (order == NPY_ANYORDER && + (PyArray_IS_C_CONTIGUOUS(self) || + PyArray_IS_F_CONTIGUOUS(self))) || + (order == NPY_CORDER && + PyArray_IS_C_CONTIGUOUS(self)) || + (order == NPY_FORTRANORDER && + PyArray_IS_F_CONTIGUOUS(self))) && + (subok || PyArray_CheckExact(self)) && + PyArray_EquivTypes(dtype, PyArray_DESCR(self))) { Py_DECREF(dtype); Py_INCREF(self); return (PyObject *)self; } + + if( forcecopy == NEVER ) { + PyErr_SetString(PyExc_ValueError, + "Unable to avoid copy while casting in np.CopyMode.NEVER"); + Py_DECREF(dtype); + return NULL; + } + if (!PyArray_CanCastArrayTo(self, dtype, casting)) { PyErr_Clear(); npy_set_invalid_cast_error( From 645fadbe3203da0846a2525f7a0d2080a8355ae9 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Mon, 7 Jun 2021 11:01:38 +0530 Subject: [PATCH 05/54] Updated API ready for formal testing --- numpy/__init__.pyi | 3 +-- numpy/core/include/numpy/ndarraytypes.h | 8 +++++- numpy/core/src/multiarray/conversion_utils.c | 6 ++--- numpy/core/src/multiarray/methods.c | 6 ++--- numpy/core/src/multiarray/multiarraymodule.c | 26 +++++++++++++++----- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index a2d6ce435bed..0555b0151fa8 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -203,7 +203,6 @@ from numpy import ( rec as rec, testing as testing, version as version, - CopyMode as CopyMode ) from numpy.core.function_base import ( @@ -3336,7 +3335,7 @@ def array( object: object, dtype: DTypeLike = ..., *, - copy: bool = ..., + copy: bool | CopyMode = ..., order: _OrderKACF = ..., subok: bool = ..., ndmin: int = ..., diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 86dc19c64c98..58a5aba47c48 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -451,7 +451,11 @@ typedef struct { int len; } PyArray_Dims; -typedef enum PyNpCopyMode {IF_NEEDED, ALWAYS, NEVER} PyNpCopyMode_Enum; +typedef enum PyNpCopyMode { + NPY_IF_NEEDED, + NPY_ALWAYS, + NPY_NEVER +} PyNpCopyMode_Enum; typedef struct { /* @@ -914,6 +918,8 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); #define NPY_ARRAY_UPDATEIFCOPY 0x1000 /* Deprecated in 1.14 */ #define NPY_ARRAY_WRITEBACKIFCOPY 0x2000 +#define NPY_ARRAY_ENSURENOCOPY 0x4000 + /* * NOTE: there are also internal flags defined in multiarray/arrayobject.h, * which start at bit 31 and work down. diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 984b0b94895d..663b9009a32c 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -176,9 +176,9 @@ PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copymode) { int int_copymode = -1; PyArray_PythonPyIntFromInt(obj, &int_copymode); - if( int_copymode != ALWAYS && - int_copymode != IF_NEEDED && - int_copymode != NEVER ) { + if( int_copymode != NPY_ALWAYS && + int_copymode != NPY_IF_NEEDED && + int_copymode != NPY_NEVER ) { PyErr_Format(PyExc_ValueError, "Unrecognized copy mode %d. Please choose one of" "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER", diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index cc161c4fcd95..d431c6f971ed 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -852,7 +852,7 @@ array_astype(PyArrayObject *self, * and it's not a subtype if subok is False, then we * can skip the copy. */ - if ( (forcecopy == IF_NEEDED || forcecopy == NEVER) && + if ( (forcecopy == NPY_IF_NEEDED || forcecopy == NPY_NEVER) && (order == NPY_KEEPORDER || (order == NPY_ANYORDER && (PyArray_IS_C_CONTIGUOUS(self) || @@ -868,8 +868,8 @@ array_astype(PyArrayObject *self, return (PyObject *)self; } - if( forcecopy == NEVER ) { - PyErr_SetString(PyExc_ValueError, + if( forcecopy == NPY_NEVER ) { + PyErr_SetString(PyExc_RuntimeError, "Unable to avoid copy while casting in np.CopyMode.NEVER"); Py_DECREF(dtype); return NULL; diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index f7c3ea093a29..edb2007002b4 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1557,7 +1557,7 @@ _prepend_ones(PyArrayObject *arr, int nd, int ndmin, NPY_ORDER order) static NPY_INLINE PyObject * _array_fromobject_generic( - PyObject *op, PyArray_Descr *type, npy_bool copy, NPY_ORDER order, + PyObject *op, PyArray_Descr *type, PyNpCopyMode_Enum copy, NPY_ORDER order, npy_bool subok, int ndmin) { PyArrayObject *oparr = NULL, *ret = NULL; @@ -1574,12 +1574,18 @@ _array_fromobject_generic( if (PyArray_CheckExact(op) || (subok && PyArray_Check(op))) { oparr = (PyArrayObject *)op; if (type == NULL) { - if (!copy && STRIDING_OK(oparr, order)) { + if ((copy == NPY_IF_NEEDED || copy == NPY_NEVER) && + STRIDING_OK(oparr, order)) { ret = oparr; Py_INCREF(ret); goto finish; } else { + if( copy == NPY_NEVER ) { + PyErr_SetString(PyExc_RuntimeError, + "Unable to avoid copy while creating a new array."); + return NULL; + } ret = (PyArrayObject *)PyArray_NewCopy(oparr, order); goto finish; } @@ -1587,12 +1593,18 @@ _array_fromobject_generic( /* One more chance */ oldtype = PyArray_DESCR(oparr); if (PyArray_EquivTypes(oldtype, type)) { - if (!copy && STRIDING_OK(oparr, order)) { + if ((copy == NPY_IF_NEEDED || copy == NPY_NEVER) && + STRIDING_OK(oparr, order)) { Py_INCREF(op); ret = oparr; goto finish; } else { + if( copy == NPY_NEVER ) { + PyErr_SetString(PyExc_RuntimeError, + "Unable to avoid copy while creating a new array."); + return NULL; + } ret = (PyArrayObject *)PyArray_NewCopy(oparr, order); if (oldtype == type || ret == NULL) { goto finish; @@ -1605,8 +1617,10 @@ _array_fromobject_generic( } } - if (copy) { + if (copy == NPY_ALWAYS) { flags = NPY_ARRAY_ENSURECOPY; + } else if( copy == NPY_NEVER ) { + flags = NPY_ARRAY_ENSURENOCOPY; } if (order == NPY_CORDER) { flags |= NPY_ARRAY_C_CONTIGUOUS; @@ -1651,7 +1665,7 @@ array_array(PyObject *NPY_UNUSED(ignored), { PyObject *op; npy_bool subok = NPY_FALSE; - npy_bool copy = NPY_TRUE; + PyNpCopyMode_Enum copy = NPY_ALWAYS; int ndmin = 0; PyArray_Descr *type = NULL; NPY_ORDER order = NPY_KEEPORDER; @@ -1662,7 +1676,7 @@ array_array(PyObject *NPY_UNUSED(ignored), if (npy_parse_arguments("array", args, len_args, kwnames, "object", NULL, &op, "|dtype", &PyArray_DescrConverter2, &type, - "$copy", &PyArray_BoolConverter, ©, + "$copy", &PyArray_CopyConverter, ©, "$order", &PyArray_OrderConverter, &order, "$subok", &PyArray_BoolConverter, &subok, "$ndmin", &PyArray_PythonPyIntFromInt, &ndmin, From 8538720ffcbf0ff69e4380e3ec4bfa985197ec7e Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Mon, 7 Jun 2021 11:09:11 +0530 Subject: [PATCH 06/54] tests for astype added --- numpy/core/tests/test_api.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py index 9e99e0bc3a06..38261a8c30af 100644 --- a/numpy/core/tests/test_api.py +++ b/numpy/core/tests/test_api.py @@ -585,3 +585,30 @@ def test_broadcast_arrays(): def test_full_from_list(shape, fill_value, expected_output): output = np.full(shape, fill_value) assert_equal(output, expected_output) + +def test_astype_copyflag(): + # test the various copyflag options + arr = np.arange(10, dtype=np.intp) + + res_true = arr.astype(np.intp, copy=True) + assert not np.may_share_memory(arr, res_true) + res_always = arr.astype(np.intp, copy=np.CopyMode.ALWAYS) + assert not np.may_share_memory(arr, res_always) + + res_false = arr.astype(np.intp, copy=False) + # `res_false is arr` currently, but check `may_share_memory`. + assert np.may_share_memory(arr, res_false) + res_if_needed = arr.astype(np.intp, copy=np.CopyMode.IF_NEEDED) + # `res_if_needed is arr` currently, but check `may_share_memory`. + assert np.may_share_memory(arr, res_if_needed) + + res_never = arr.astype(np.intp, copy=np.CopyMode.NEVER) + assert np.may_share_memory(arr, res_never) + + + # Simple tests for when a copy is necessary: + res_false = arr.astype(np.float64, copy=False) + assert_array_equal(res_false, arr) + res_if_needed = arr.astype(np.float64, copy=np.CopyMode.IF_NEEDED) + assert_array_equal(res_if_needed, arr) + assert_raises(RuntimeError, arr.astype, np.float64, copy=np.CopyMode.NEVER) From 8c00223ab944d18a7077c59f4ae0ed010f61a17e Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Mon, 7 Jun 2021 12:03:43 +0530 Subject: [PATCH 07/54] Added tests for np.array --- numpy/core/src/multiarray/conversion_utils.c | 8 +- numpy/core/tests/test_multiarray.py | 180 +++++++++++++++++++ 2 files changed, 184 insertions(+), 4 deletions(-) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 663b9009a32c..fc4904729d36 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -168,8 +168,8 @@ NPY_NO_EXPORT int PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copymode) { if (obj == Py_None) { PyErr_SetString(PyExc_ValueError, - "NoneType copy mode not allowed. Please choose one of" - "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER"); + "NoneType copy mode not allowed. Please choose one of " + "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER."); return NPY_FAIL; } @@ -180,8 +180,8 @@ PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copymode) { int_copymode != NPY_IF_NEEDED && int_copymode != NPY_NEVER ) { PyErr_Format(PyExc_ValueError, - "Unrecognized copy mode %d. Please choose one of" - "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER", + "Unrecognized copy mode %d. Please choose one of " + "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER.", int_copymode); return NPY_FAIL; } diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index d567653f5a4a..981e59e20916 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7745,6 +7745,186 @@ def test_error_if_stored_buffer_info_is_corrupted(self, obj): _multiarray_tests.corrupt_or_fix_bufferinfo(obj) +class TestArrayCreationCopyArgument(object): + + true_vals = [True, np.CopyMode.ALWAYS, np.True_] + false_vals = [False, np.CopyMode.IF_NEEDED, np.False_] + + def test_scalars(self): + + # Test both numpy and python scalars + for dtype in np.typecodes["All"]: + arr = np.zeros((), dtype=dtype) + scalar = arr[()] + pyscalar = arr.item(0) + + # Test never-copy raises error: + assert_raises(ValueError, np.array, scalar, copy=np.CopyMode.NEVER) + assert_raises(ValueError, np.array, pyscalar, copy=np.CopyMode.NEVER) + + + def test_compatible_cast(self): + + # Some types are compatible even though they are different, no + # copy is necessary for them. This is mostly true for some integers + def int_types(byteswap=False): + int_types = (np.typecodes["Integer"] + + np.typecodes["UnsignedInteger"]) + for int_type in int_types: + yield np.dtype(int_type) + if byteswap: + yield np.dtype(int_type).newbyteorder() + + for int1 in int_types(): + for int2 in int_types(True): + arr = np.arange(10, dtype=int1) + + for copy in self.true_vals: + res = np.array(arr, copy=copy, dtype=int2) + assert res is not arr and res.flags.owndata + assert_array_equal(res, arr) + + + + if int1 == int2: + # Casting is not necessary, base check is sufficient here + for copy in self.false_vals: + res = np.array(arr, copy=copy, dtype=int2) + assert res is arr or res.base is arr + + + res = np.array(arr, copy=np.CopyMode.NEVER, dtype=int2) + assert res is arr or res.base is arr + + + else: + # Casting is necessary, assert copy works: + for copy in self.false_vals: + res = np.array(arr, copy=copy, dtype=int2) + assert res is not arr and res.flags.owndata + assert_array_equal(res, arr) + + + assert_raises(ValueError, np.array, + arr, copy=np.CopyMode.NEVER, dtype=int2) + + + def test_buffer_interface(self): + + # Buffer interface gives direct memory access (no copy) + arr = np.arange(10) + view = memoryview(arr) + + # Checking bases is a bit tricky since numpy creates another + # memoryview, so use may_share_memory. + for copy in self.true_vals: + res = np.array(view, copy=copy) + assert not np.may_share_memory(arr, res) + for copy in self.false_vals: + res = np.array(view, copy=copy) + assert np.may_share_memory(arr, res) + res = np.array(view, copy=np.CopyMode.NEVER) + assert np.may_share_memory(arr, res) + + def test_array_interfaces(self): + # Array interface gives direct memory access (much like a memoryview) + base_arr = np.arange(10) + + + class ArrayLike: + __array_interface__ = base_arr.__array_interface__ + + arr = ArrayLike() + + for copy, val in [(True, None), (np.CopyMode.ALWAYS, None), + (False, arr), (np.CopyMode.IF_NEEDED, arr), + (np.CopyMode.NEVER, arr)]: + res = np.array(arr, copy=copy) + assert res.base is val + + def test___array__(self): + base_arr = np.arange(10) + + class ArrayLike: + def __array__(self): + # __array__ should return a copy, numpy cannot know this + # however. + return base_arr + + arr = ArrayLike() + + for copy in self.true_vals: + res = np.array(arr, copy=copy) + assert_array_equal(res, base_arr) + # An additional copy is currently forced by numpy in this case, + # you could argue, numpy does not trust the ArrayLike. This + # may be open for change: + assert res is not base_arr + + for copy in self.false_vals: + res = np.array(arr, copy=False) + assert_array_equal(res, base_arr) + assert res is base_arr # numpy trusts the ArrayLike + + assert_raises(RuntimeError, np.array, arr, copy=np.CopyMode.NEVER) + + + @pytest.mark.parametrize( + "arr", [np.ones(()), np.arange(81).reshape((9, 9))]) + @pytest.mark.parametrize("order1", ["C", "F", None]) + @pytest.mark.parametrize("order2", ["C", "F", "A", "K"]) + def test_order_mismatch(self, arr, order1, order2): + # The order is the main (python side) reason that can cause + # a never-copy to fail. + # Prepare C-order, F-order and non-contiguous arrays: + arr = arr.copy(order1) + if order1 == "C": + assert arr.flags.c_contiguous + elif order1 == "F": + assert arr.flags.f_contiguous + elif arr.ndim != 0: + # Make array non-contiguous + arr = arr[::2, ::2] + assert not arr.flags.forc + + + # Whether a copy is necessary depends on the order of arr: + if order2 == "C": + no_copy_necessary = arr.flags.c_contiguous + elif order2 == "F": + no_copy_necessary = arr.flags.f_contiguous + else: + # Keeporder and Anyorder are OK with non-contiguous output. + # This is not consistent with the `astype` behaviour which + # enforces contiguity for "A". It is probably historic from when + # "K" did not exist. + no_copy_necessary = True + + + # Test it for both the array and a memoryview + for view in [arr, memoryview(arr)]: + for copy in self.true_vals: + res = np.array(view, copy=copy, order=order2) + assert res is not arr and res.flags.owndata + assert_array_equal(arr, res) + + + if no_copy_necessary: + for copy in self.false_vals: + res = np.array(view, copy=copy, order=order2) + # res.base.obj refers to the memoryview + assert res is arr or res.base.obj is arr + + res = np.array(view, copy=np.CopyMode.NEVER, order=order2) + assert res is arr or res.base.obj is arr + else: + for copy in self.false_vals: + res = np.array(arr, copy=copy, order=order2) + assert_array_equal(arr, res) + assert_raises(ValueError, np.array, + view, copy=np.CopyMode.NEVER, order=order2) + + class TestArrayAttributeDeletion: def test_multiarray_writable_attributes_deletion(self): From 4e59da126b522dd5a894041a4c9570e130fe7ae4 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Mon, 7 Jun 2021 15:02:50 +0530 Subject: [PATCH 08/54] Testing for copy argument for existing APIs complete --- numpy/core/src/multiarray/ctors.c | 14 +++++++++++++- numpy/core/tests/test_multiarray.py | 19 +++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index ef28d7797926..670a510777ed 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1842,6 +1842,11 @@ PyArray_CheckFromAny(PyObject *op, PyArray_Descr *descr, int min_depth, if ((requires & NPY_ARRAY_ELEMENTSTRIDES) && !PyArray_ElementStrides(obj)) { PyObject *ret; + if( requires & NPY_ARRAY_ENSURENOCOPY ) { + PyErr_SetString(PyExc_RuntimeError, + "Unable to avoid copy while creating a new array."); + return NULL; + } ret = PyArray_NewCopy((PyArrayObject *)obj, NPY_ANYORDER); Py_DECREF(obj); obj = ret; @@ -1916,6 +1921,14 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) !PyArray_EquivTypes(oldtype, newtype); if (copy) { + + if( flags & NPY_ARRAY_ENSURENOCOPY ) { + PyErr_SetString(PyExc_RuntimeError, + "Unable to avoid copy while creating " + "an array from given array."); + return NULL; + } + NPY_ORDER order = NPY_KEEPORDER; int subok = 1; @@ -1988,7 +2001,6 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) if (flags & NPY_ARRAY_ENSUREARRAY) { subtype = &PyArray_Type; } - ret = (PyArrayObject *)PyArray_View(arr, NULL, subtype); if (ret == NULL) { return NULL; diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 981e59e20916..734a7cea3bbe 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7750,19 +7750,6 @@ class TestArrayCreationCopyArgument(object): true_vals = [True, np.CopyMode.ALWAYS, np.True_] false_vals = [False, np.CopyMode.IF_NEEDED, np.False_] - def test_scalars(self): - - # Test both numpy and python scalars - for dtype in np.typecodes["All"]: - arr = np.zeros((), dtype=dtype) - scalar = arr[()] - pyscalar = arr.item(0) - - # Test never-copy raises error: - assert_raises(ValueError, np.array, scalar, copy=np.CopyMode.NEVER) - assert_raises(ValueError, np.array, pyscalar, copy=np.CopyMode.NEVER) - - def test_compatible_cast(self): # Some types are compatible even though they are different, no @@ -7805,7 +7792,7 @@ def int_types(byteswap=False): assert_array_equal(res, arr) - assert_raises(ValueError, np.array, + assert_raises(RuntimeError, np.array, arr, copy=np.CopyMode.NEVER, dtype=int2) @@ -7866,7 +7853,7 @@ def __array__(self): assert_array_equal(res, base_arr) assert res is base_arr # numpy trusts the ArrayLike - assert_raises(RuntimeError, np.array, arr, copy=np.CopyMode.NEVER) + assert np.array(arr, copy=np.CopyMode.NEVER) is base_arr @pytest.mark.parametrize( @@ -7921,7 +7908,7 @@ def test_order_mismatch(self, arr, order1, order2): for copy in self.false_vals: res = np.array(arr, copy=copy, order=order2) assert_array_equal(arr, res) - assert_raises(ValueError, np.array, + assert_raises(RuntimeError, np.array, view, copy=np.CopyMode.NEVER, order=order2) From 6bff3dd72e36e009dd4417054d49b64f0458a821 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 8 Jun 2021 08:35:48 +0530 Subject: [PATCH 09/54] Added some notes for future commits --- numpy/core/src/multiarray/array_coercion.c | 1 + numpy/core/src/multiarray/ctors.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 22050a56ff6b..d567ee0a4348 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -1163,6 +1163,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( * It might be nice to deprecate this? But it allows things such as * `arr1d[...] = np.array([[1,2,3,4]])` */ +// Here we check whether a copy is being made or not. Check this function. NPY_NO_EXPORT int PyArray_DiscoverDTypeAndShape( PyObject *obj, int max_dims, diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 670a510777ed..a125ce81f2ff 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1282,7 +1282,7 @@ _array_from_array_like(PyObject *op, PyErr_Clear(); } else { - tmp = _array_from_buffer_3118(memoryview); + tmp = _array_from_buffer_3118(memoryview); // Assume: never creates a copy Py_DECREF(memoryview); if (tmp == NULL) { return NULL; @@ -1324,7 +1324,7 @@ _array_from_array_like(PyObject *op, * this should be changed! */ if (!writeable && tmp == Py_NotImplemented) { - tmp = PyArray_FromArrayAttr(op, requested_dtype, context); + tmp = PyArray_FromArrayAttr(op, requested_dtype, context); // Assume: array was copied. if (tmp == NULL) { return NULL; } From a90880a0a61cb32a23c7eb4bab463941714c59d9 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 8 Jun 2021 09:55:23 +0530 Subject: [PATCH 10/54] RuntimeError -> ValueError --- numpy/core/src/multiarray/ctors.c | 4 ++-- numpy/core/src/multiarray/methods.c | 2 +- numpy/core/src/multiarray/multiarraymodule.c | 4 ++-- numpy/core/tests/test_api.py | 2 +- numpy/core/tests/test_multiarray.py | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index a125ce81f2ff..87406bdbc826 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1843,7 +1843,7 @@ PyArray_CheckFromAny(PyObject *op, PyArray_Descr *descr, int min_depth, !PyArray_ElementStrides(obj)) { PyObject *ret; if( requires & NPY_ARRAY_ENSURENOCOPY ) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating a new array."); return NULL; } @@ -1923,7 +1923,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) if (copy) { if( flags & NPY_ARRAY_ENSURENOCOPY ) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating " "an array from given array."); return NULL; diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index d431c6f971ed..9613bf04ed2c 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -869,7 +869,7 @@ array_astype(PyArrayObject *self, } if( forcecopy == NPY_NEVER ) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while casting in np.CopyMode.NEVER"); Py_DECREF(dtype); return NULL; diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index edb2007002b4..489c51df31e4 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1582,7 +1582,7 @@ _array_fromobject_generic( } else { if( copy == NPY_NEVER ) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating a new array."); return NULL; } @@ -1601,7 +1601,7 @@ _array_fromobject_generic( } else { if( copy == NPY_NEVER ) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating a new array."); return NULL; } diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py index 38261a8c30af..3bd01e5b4f6f 100644 --- a/numpy/core/tests/test_api.py +++ b/numpy/core/tests/test_api.py @@ -611,4 +611,4 @@ def test_astype_copyflag(): assert_array_equal(res_false, arr) res_if_needed = arr.astype(np.float64, copy=np.CopyMode.IF_NEEDED) assert_array_equal(res_if_needed, arr) - assert_raises(RuntimeError, arr.astype, np.float64, copy=np.CopyMode.NEVER) + assert_raises(ValueError, arr.astype, np.float64, copy=np.CopyMode.NEVER) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 734a7cea3bbe..ba8cadddabd2 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7792,7 +7792,7 @@ def int_types(byteswap=False): assert_array_equal(res, arr) - assert_raises(RuntimeError, np.array, + assert_raises(ValueError, np.array, arr, copy=np.CopyMode.NEVER, dtype=int2) @@ -7908,7 +7908,7 @@ def test_order_mismatch(self, arr, order1, order2): for copy in self.false_vals: res = np.array(arr, copy=copy, order=order2) assert_array_equal(arr, res) - assert_raises(RuntimeError, np.array, + assert_raises(ValueError, np.array, view, copy=np.CopyMode.NEVER, order=order2) From 5481a8ac389962f855d15c91ba0fa365b3f06c90 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 8 Jun 2021 13:40:41 +0530 Subject: [PATCH 11/54] enum.IntEnum -> enum.Enum --- numpy/__init__.pyi | 2 +- numpy/_globals.py | 2 +- numpy/core/src/multiarray/conversion_utils.c | 24 ++++++++++++++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 6bf9d49a886e..f086a7a0f50f 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -3642,7 +3642,7 @@ trunc: _UFunc_Nin1_Nout1[L['trunc'], L[7], None] abs = absolute -class CopyMode(enum.IntEnum): +class CopyMode(enum.Enum): ALWAYS: L[1] IF_NEEDED: L[0] diff --git a/numpy/_globals.py b/numpy/_globals.py index a6fbec34067e..e40694e1a8c0 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -93,7 +93,7 @@ def __repr__(self): _NoValue = _NoValueType() -class CopyMode(enum.IntEnum): +class CopyMode(enum.Enum): ALWAYS = 1 IF_NEEDED = 0 diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index fc4904729d36..e16c03e31aa2 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -174,15 +174,29 @@ PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copymode) { } int int_copymode = -1; - PyArray_PythonPyIntFromInt(obj, &int_copymode); + PyObject* numpy_CopyMode = NULL; + npy_cache_import("numpy._globals", "CopyMode", &numpy_CopyMode); + if( numpy_CopyMode != NULL ) { + if(PyObject_IsInstance(obj, numpy_CopyMode)) { + PyObject* mode_value = PyObject_GetAttrString(obj, "value"); + PyArray_PythonPyIntFromInt(mode_value, &int_copymode); + } + } + + // If obj is not an instance of numpy.CopyMode then follow + // the conventional assumption that it must be value that + // can be converted to an integer. + if( int_copymode < 0 ) { + PyArray_PythonPyIntFromInt(obj, &int_copymode); + } if( int_copymode != NPY_ALWAYS && int_copymode != NPY_IF_NEEDED && int_copymode != NPY_NEVER ) { - PyErr_Format(PyExc_ValueError, - "Unrecognized copy mode %d. Please choose one of " - "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER.", - int_copymode); + PyErr_SetString(PyExc_ValueError, + "Unrecognized copy mode. Please choose one of " + "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER, " + "True/np.True_, False/np.False_"); return NPY_FAIL; } From f2e7a818561c59e014d0627a187ae359dafe7a37 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 8 Jun 2021 13:43:04 +0530 Subject: [PATCH 12/54] aligned astype with main branch --- numpy/__init__.pyi | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index f086a7a0f50f..6cbc0578245e 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1227,14 +1227,6 @@ class _ArrayOrScalarCommon: def __deepcopy__(self: _ArraySelf, __memo: Optional[dict] = ...) -> _ArraySelf: ... def __eq__(self, other): ... def __ne__(self, other): ... - def astype( - self: _ArraySelf, - dtype: DTypeLike, - order: _OrderKACF = ..., - casting: _Casting = ..., - subok: bool = ..., - copy: bool | CopyMode = ..., - ) -> _ArraySelf: ... def copy(self: _ArraySelf, order: _OrderKACF = ...) -> _ArraySelf: ... def dump(self, file: str) -> None: ... def dumps(self) -> bytes: ... @@ -1877,7 +1869,7 @@ class ndarray(_ArrayOrScalarCommon, Generic[_ShapeType, _DType_co]): order: _OrderKACF = ..., casting: _Casting = ..., subok: bool = ..., - copy: bool = ..., + copy: bool | CopyMode = ..., ) -> NDArray[_ScalarType]: ... @overload def astype( @@ -1886,7 +1878,7 @@ class ndarray(_ArrayOrScalarCommon, Generic[_ShapeType, _DType_co]): order: _OrderKACF = ..., casting: _Casting = ..., subok: bool = ..., - copy: bool = ..., + copy: bool | CopyMode = ..., ) -> NDArray[Any]: ... @overload @@ -2897,7 +2889,7 @@ class generic(_ArrayOrScalarCommon): order: _OrderKACF = ..., casting: _Casting = ..., subok: bool = ..., - copy: bool = ..., + copy: bool | CopyMode = ..., ) -> _ScalarType: ... @overload def astype( @@ -2906,7 +2898,7 @@ class generic(_ArrayOrScalarCommon): order: _OrderKACF = ..., casting: _Casting = ..., subok: bool = ..., - copy: bool = ..., + copy: bool | CopyMode = ..., ) -> Any: ... # NOTE: `view` will perform a 0D->scalar cast, From e828f1569dd65cb8a93a619fc89edec582c744cc Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 8 Jun 2021 14:56:41 +0530 Subject: [PATCH 13/54] Add tests for scalars --- numpy/core/src/multiarray/ctors.c | 7 +++++++ numpy/core/tests/test_multiarray.py | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 87406bdbc826..c2abc299c806 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1715,6 +1715,12 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, /* Create a new array and copy the data */ Py_INCREF(dtype); /* hold on in case of a subarray that is replaced */ + if( flags & NPY_ARRAY_ENSURENOCOPY ) { + PyErr_SetString(PyExc_ValueError, + "Unable to avoid copy while creating " + "an array from descriptor."); + return NULL; + } ret = (PyArrayObject *)PyArray_NewFromDescr( &PyArray_Type, dtype, ndim, dims, NULL, NULL, flags&NPY_ARRAY_F_CONTIGUOUS, NULL); @@ -1839,6 +1845,7 @@ PyArray_CheckFromAny(PyObject *op, PyArray_Descr *descr, int min_depth, if (obj == NULL) { return NULL; } + if ((requires & NPY_ARRAY_ELEMENTSTRIDES) && !PyArray_ElementStrides(obj)) { PyObject *ret; diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index ba8cadddabd2..8fd7ccf4e486 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7750,6 +7750,17 @@ class TestArrayCreationCopyArgument(object): true_vals = [True, np.CopyMode.ALWAYS, np.True_] false_vals = [False, np.CopyMode.IF_NEEDED, np.False_] + def test_scalars(self): + # Test both numpy and python scalars + for dtype in np.typecodes["All"]: + arr = np.zeros((), dtype=dtype) + scalar = arr[()] + pyscalar = arr.item(0) + + # Test never-copy raises error: + assert_raises(ValueError, np.array, scalar, copy=np.CopyMode.NEVER) + assert_raises(ValueError, np.array, pyscalar, copy=np.CopyMode.NEVER) + def test_compatible_cast(self): # Some types are compatible even though they are different, no From 91d669396a7f73abecd511f1c49076f2bc441e5c Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 8 Jun 2021 15:05:20 +0530 Subject: [PATCH 14/54] resolved linting issues --- numpy/core/tests/test_multiarray.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 70e4f1044268..22816643aaac 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7765,8 +7765,10 @@ def test_scalars(self): pyscalar = arr.item(0) # Test never-copy raises error: - assert_raises(ValueError, np.array, scalar, copy=np.CopyMode.NEVER) - assert_raises(ValueError, np.array, pyscalar, copy=np.CopyMode.NEVER) + assert_raises(ValueError, np.array, scalar, + copy=np.CopyMode.NEVER) + assert_raises(ValueError, np.array, pyscalar, + copy=np.CopyMode.NEVER) def test_compatible_cast(self): @@ -7809,7 +7811,6 @@ def int_types(byteswap=False): assert res is not arr and res.flags.owndata assert_array_equal(res, arr) - assert_raises(ValueError, np.array, arr, copy=np.CopyMode.NEVER, dtype=int2) From 7b53a02c647eb9b1c627799e7ce1b7fecb0bf928 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 8 Jun 2021 15:09:29 +0530 Subject: [PATCH 15/54] Fixed blank line linting issue --- numpy/_globals.py | 3 ++- numpy/core/tests/test_api.py | 1 - numpy/core/tests/test_multiarray.py | 10 ---------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/numpy/_globals.py b/numpy/_globals.py index e40694e1a8c0..9a99acde388c 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -109,4 +109,5 @@ def __bool__(self): raise TypeError(f"{self} is neither True nor False.") -CopyMode.__module__ = 'numpy' \ No newline at end of file + +CopyMode.__module__ = 'numpy' diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py index 0204d79c9f64..d48b208001da 100644 --- a/numpy/core/tests/test_api.py +++ b/numpy/core/tests/test_api.py @@ -618,7 +618,6 @@ def test_astype_copyflag(): res_never = arr.astype(np.intp, copy=np.CopyMode.NEVER) assert np.may_share_memory(arr, res_never) - # Simple tests for when a copy is necessary: res_false = arr.astype(np.float64, copy=False) assert_array_equal(res_false, arr) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 22816643aaac..d1246f11cc19 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7791,19 +7791,15 @@ def int_types(byteswap=False): assert res is not arr and res.flags.owndata assert_array_equal(res, arr) - - if int1 == int2: # Casting is not necessary, base check is sufficient here for copy in self.false_vals: res = np.array(arr, copy=copy, dtype=int2) assert res is arr or res.base is arr - res = np.array(arr, copy=np.CopyMode.NEVER, dtype=int2) assert res is arr or res.base is arr - else: # Casting is necessary, assert copy works: for copy in self.false_vals: @@ -7814,7 +7810,6 @@ def int_types(byteswap=False): assert_raises(ValueError, np.array, arr, copy=np.CopyMode.NEVER, dtype=int2) - def test_buffer_interface(self): # Buffer interface gives direct memory access (no copy) @@ -7836,7 +7831,6 @@ def test_array_interfaces(self): # Array interface gives direct memory access (much like a memoryview) base_arr = np.arange(10) - class ArrayLike: __array_interface__ = base_arr.__array_interface__ @@ -7874,7 +7868,6 @@ def __array__(self): assert np.array(arr, copy=np.CopyMode.NEVER) is base_arr - @pytest.mark.parametrize( "arr", [np.ones(()), np.arange(81).reshape((9, 9))]) @pytest.mark.parametrize("order1", ["C", "F", None]) @@ -7893,7 +7886,6 @@ def test_order_mismatch(self, arr, order1, order2): arr = arr[::2, ::2] assert not arr.flags.forc - # Whether a copy is necessary depends on the order of arr: if order2 == "C": no_copy_necessary = arr.flags.c_contiguous @@ -7906,7 +7898,6 @@ def test_order_mismatch(self, arr, order1, order2): # "K" did not exist. no_copy_necessary = True - # Test it for both the array and a memoryview for view in [arr, memoryview(arr)]: for copy in self.true_vals: @@ -7914,7 +7905,6 @@ def test_order_mismatch(self, arr, order1, order2): assert res is not arr and res.flags.owndata assert_array_equal(arr, res) - if no_copy_necessary: for copy in self.false_vals: res = np.array(view, copy=copy, order=order2) From eedb30087d2c1de95b2a65aef074f9aade1b9658 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 9 Jun 2021 08:52:57 +0530 Subject: [PATCH 16/54] Apply suggestions from code review Co-authored-by: Eric Wieser --- numpy/_globals.py | 4 ++-- numpy/core/include/numpy/ndarraytypes.h | 8 ++++---- numpy/core/src/multiarray/conversion_utils.c | 16 +++++++++++----- numpy/core/src/multiarray/methods.c | 5 +++-- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/numpy/_globals.py b/numpy/_globals.py index 9a99acde388c..b7a399a793b9 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -95,8 +95,8 @@ def __repr__(self): class CopyMode(enum.Enum): - ALWAYS = 1 - IF_NEEDED = 0 + ALWAYS = True + IF_NEEDED = False NEVER = 2 def __bool__(self): diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 6626da1e9b6c..6a861dc1d9ce 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -452,10 +452,10 @@ typedef struct { } PyArray_Dims; typedef enum PyNpCopyMode { - NPY_IF_NEEDED, - NPY_ALWAYS, - NPY_NEVER -} PyNpCopyMode_Enum; + NPY_COPY_IF_NEEDED, + NPY_COPY_ALWAYS, + NPY_COPY_NEVER +} PyArray_CopyMode; typedef struct { /* diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index e16c03e31aa2..8f608ea3ed69 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -176,17 +176,23 @@ PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copymode) { int int_copymode = -1; PyObject* numpy_CopyMode = NULL; npy_cache_import("numpy._globals", "CopyMode", &numpy_CopyMode); - if( numpy_CopyMode != NULL ) { - if(PyObject_IsInstance(obj, numpy_CopyMode)) { - PyObject* mode_value = PyObject_GetAttrString(obj, "value"); - PyArray_PythonPyIntFromInt(mode_value, &int_copymode); + if (numpy_CopyMode == NULL) { + return NPY_FAIL; + } + if (PyObject_IsInstance(obj, numpy_CopyMode)) { + PyObject* mode_value = PyObject_GetAttrString(obj, "value"); + if (mode_value == NULL) { + return NPY_FAIL; + } + if (!PyArray_PythonPyIntFromInt(mode_value, &int_copymode)) { + return NPY_FAIL; } } // If obj is not an instance of numpy.CopyMode then follow // the conventional assumption that it must be value that // can be converted to an integer. - if( int_copymode < 0 ) { + else { PyArray_PythonPyIntFromInt(obj, &int_copymode); } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 9613bf04ed2c..a3b97ea7c048 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -828,7 +828,8 @@ array_astype(PyArrayObject *self, */ NPY_CASTING casting = NPY_UNSAFE_CASTING; NPY_ORDER order = NPY_KEEPORDER; - PyNpCopyMode_Enum forcecopy = 1, subok = 1; + PyNpCopyMode_Enum forcecopy = 1; + int subok = 1; NPY_PREPARE_ARGPARSER; if (npy_parse_arguments("astype", args, len_args, kwnames, "dtype", &PyArray_DescrConverter, &dtype, @@ -852,7 +853,7 @@ array_astype(PyArrayObject *self, * and it's not a subtype if subok is False, then we * can skip the copy. */ - if ( (forcecopy == NPY_IF_NEEDED || forcecopy == NPY_NEVER) && + if (forcecopy != NPY_ALWAYS && (order == NPY_KEEPORDER || (order == NPY_ANYORDER && (PyArray_IS_C_CONTIGUOUS(self) || From bc8903d5fe64b705d03011a472357175d6aadce4 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Wed, 9 Jun 2021 09:20:57 +0530 Subject: [PATCH 17/54] renaming complete --- numpy/core/src/multiarray/conversion_utils.c | 10 +++++----- numpy/core/src/multiarray/conversion_utils.h | 2 +- numpy/core/src/multiarray/methods.c | 6 +++--- numpy/core/src/multiarray/multiarraymodule.c | 16 ++++++++-------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 8f608ea3ed69..22b77070c2bb 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -165,7 +165,7 @@ PyArray_OptionalIntpConverter(PyObject *obj, PyArray_Dims *seq) } NPY_NO_EXPORT int -PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copymode) { +PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { if (obj == Py_None) { PyErr_SetString(PyExc_ValueError, "NoneType copy mode not allowed. Please choose one of " @@ -196,9 +196,9 @@ PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copymode) { PyArray_PythonPyIntFromInt(obj, &int_copymode); } - if( int_copymode != NPY_ALWAYS && - int_copymode != NPY_IF_NEEDED && - int_copymode != NPY_NEVER ) { + if( int_copymode != NPY_COPY_ALWAYS && + int_copymode != NPY_COPY_IF_NEEDED && + int_copymode != NPY_COPY_NEVER ) { PyErr_SetString(PyExc_ValueError, "Unrecognized copy mode. Please choose one of " "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER, " @@ -206,7 +206,7 @@ PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copymode) { return NPY_FAIL; } - *copymode = (PyNpCopyMode_Enum) int_copymode; + *copymode = (PyArray_CopyMode) int_copymode; return NPY_SUCCEED; } diff --git a/numpy/core/src/multiarray/conversion_utils.h b/numpy/core/src/multiarray/conversion_utils.h index d2874aafa843..d1f17c58ebd4 100644 --- a/numpy/core/src/multiarray/conversion_utils.h +++ b/numpy/core/src/multiarray/conversion_utils.h @@ -10,7 +10,7 @@ NPY_NO_EXPORT int PyArray_OptionalIntpConverter(PyObject *obj, PyArray_Dims *seq); NPY_NO_EXPORT int -PyArray_CopyConverter(PyObject *obj, PyNpCopyMode_Enum *copyflag); +PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copyflag); NPY_NO_EXPORT int PyArray_BufferConverter(PyObject *obj, PyArray_Chunk *buf); diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index a3b97ea7c048..5e1a8b0e5d34 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -828,7 +828,7 @@ array_astype(PyArrayObject *self, */ NPY_CASTING casting = NPY_UNSAFE_CASTING; NPY_ORDER order = NPY_KEEPORDER; - PyNpCopyMode_Enum forcecopy = 1; + PyArray_CopyMode forcecopy = 1; int subok = 1; NPY_PREPARE_ARGPARSER; if (npy_parse_arguments("astype", args, len_args, kwnames, @@ -853,7 +853,7 @@ array_astype(PyArrayObject *self, * and it's not a subtype if subok is False, then we * can skip the copy. */ - if (forcecopy != NPY_ALWAYS && + if (forcecopy != NPY_COPY_ALWAYS && (order == NPY_KEEPORDER || (order == NPY_ANYORDER && (PyArray_IS_C_CONTIGUOUS(self) || @@ -869,7 +869,7 @@ array_astype(PyArrayObject *self, return (PyObject *)self; } - if( forcecopy == NPY_NEVER ) { + if( forcecopy == NPY_COPY_NEVER ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while casting in np.CopyMode.NEVER"); Py_DECREF(dtype); diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 489c51df31e4..7e1311ea34ca 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1557,7 +1557,7 @@ _prepend_ones(PyArrayObject *arr, int nd, int ndmin, NPY_ORDER order) static NPY_INLINE PyObject * _array_fromobject_generic( - PyObject *op, PyArray_Descr *type, PyNpCopyMode_Enum copy, NPY_ORDER order, + PyObject *op, PyArray_Descr *type, PyArray_CopyMode copy, NPY_ORDER order, npy_bool subok, int ndmin) { PyArrayObject *oparr = NULL, *ret = NULL; @@ -1574,14 +1574,14 @@ _array_fromobject_generic( if (PyArray_CheckExact(op) || (subok && PyArray_Check(op))) { oparr = (PyArrayObject *)op; if (type == NULL) { - if ((copy == NPY_IF_NEEDED || copy == NPY_NEVER) && + if ((copy == NPY_COPY_IF_NEEDED || copy == NPY_COPY_NEVER) && STRIDING_OK(oparr, order)) { ret = oparr; Py_INCREF(ret); goto finish; } else { - if( copy == NPY_NEVER ) { + if( copy == NPY_COPY_NEVER ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating a new array."); return NULL; @@ -1593,14 +1593,14 @@ _array_fromobject_generic( /* One more chance */ oldtype = PyArray_DESCR(oparr); if (PyArray_EquivTypes(oldtype, type)) { - if ((copy == NPY_IF_NEEDED || copy == NPY_NEVER) && + if ((copy == NPY_COPY_IF_NEEDED || copy == NPY_COPY_NEVER) && STRIDING_OK(oparr, order)) { Py_INCREF(op); ret = oparr; goto finish; } else { - if( copy == NPY_NEVER ) { + if( copy == NPY_COPY_NEVER ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating a new array."); return NULL; @@ -1617,9 +1617,9 @@ _array_fromobject_generic( } } - if (copy == NPY_ALWAYS) { + if (copy == NPY_COPY_ALWAYS) { flags = NPY_ARRAY_ENSURECOPY; - } else if( copy == NPY_NEVER ) { + } else if( copy == NPY_COPY_NEVER ) { flags = NPY_ARRAY_ENSURENOCOPY; } if (order == NPY_CORDER) { @@ -1665,7 +1665,7 @@ array_array(PyObject *NPY_UNUSED(ignored), { PyObject *op; npy_bool subok = NPY_FALSE; - PyNpCopyMode_Enum copy = NPY_ALWAYS; + PyArray_CopyMode copy = NPY_COPY_ALWAYS; int ndmin = 0; PyArray_Descr *type = NULL; NPY_ORDER order = NPY_KEEPORDER; From 3268a48ba4c0e7ae97dc358fa85e7c1b09d7cb21 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Wed, 9 Jun 2021 12:08:08 +0530 Subject: [PATCH 18/54] fixed failures due to Python 3.8.2 --- numpy/core/src/multiarray/conversion_utils.c | 52 +++++++++++++++----- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 22b77070c2bb..c5e1ef808fd6 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -175,11 +175,11 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { int int_copymode = -1; PyObject* numpy_CopyMode = NULL; + PyObject* numpy_bool_ = NULL; npy_cache_import("numpy._globals", "CopyMode", &numpy_CopyMode); - if (numpy_CopyMode == NULL) { - return NPY_FAIL; - } - if (PyObject_IsInstance(obj, numpy_CopyMode)) { + npy_cache_import("numpy", "bool_", &numpy_bool_); + + if (numpy_CopyMode != NULL && PyObject_IsInstance(obj, numpy_CopyMode)) { PyObject* mode_value = PyObject_GetAttrString(obj, "value"); if (mode_value == NULL) { return NPY_FAIL; @@ -187,22 +187,52 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { if (!PyArray_PythonPyIntFromInt(mode_value, &int_copymode)) { return NPY_FAIL; } + } else if(numpy_bool_ != NULL && PyObject_IsInstance(obj, numpy_bool_)) { + /* + In Python 3.8.2, numpy.bool_ objects are not converted + to integers using PyLong_AsLong. Hence, the following + if condition checks whether the input obj is a numpy.bool_ object + and extracts the values by perfroming comparisions between + obj and numpy.True_/numpy.False_. + */ + PyObject *numpy_True_ = NULL, *numpy_False_ = NULL; + npy_cache_import("numpy", "True_", &numpy_True_); + npy_cache_import("numpy", "False_", &numpy_False_); + if( numpy_True_ != NULL && + PyObject_RichCompareBool(obj, numpy_True_, Py_EQ) ) { + int_copymode = 1; + } else if( numpy_False_ != NULL && + PyObject_RichCompareBool(obj, numpy_False_, Py_EQ) ) { + int_copymode = 0; + } + } else if( PyBool_Check(obj) ) { + /* + In Python 3.8.2, Boolean objects are not converted + to integers using PyLong_AsLong. Hence, the following + if condition checks whether the input obj is a Boolean object + and extracts the values by perfroming comparisions between + obj and Py_True/Py_False. + */ + if( PyObject_RichCompareBool(obj, Py_True, Py_EQ) ) { + int_copymode = 1; + } else if( PyObject_RichCompareBool(obj, Py_False, Py_EQ) ) { + int_copymode = 0; + } } - - // If obj is not an instance of numpy.CopyMode then follow - // the conventional assumption that it must be value that - // can be converted to an integer. else { + // If obj is not an instance of numpy.CopyMode then follow + // the conventional assumption that it must be value that + // can be converted to an integer. PyArray_PythonPyIntFromInt(obj, &int_copymode); } if( int_copymode != NPY_COPY_ALWAYS && int_copymode != NPY_COPY_IF_NEEDED && int_copymode != NPY_COPY_NEVER ) { - PyErr_SetString(PyExc_ValueError, - "Unrecognized copy mode. Please choose one of " + PyErr_Format(PyExc_ValueError, + "Unrecognized copy mode %d. Please choose one of " "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER, " - "True/np.True_, False/np.False_"); + "True/np.True_, False/np.False_", int_copymode); return NPY_FAIL; } From 782d823301ac932ce0a41b816403b83cce54d20b Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Thu, 10 Jun 2021 09:04:44 +0530 Subject: [PATCH 19/54] fixed PYPY errors --- numpy/core/tests/test_multiarray.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index d1246f11cc19..54386de956ed 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7909,10 +7909,12 @@ def test_order_mismatch(self, arr, order1, order2): for copy in self.false_vals: res = np.array(view, copy=copy, order=order2) # res.base.obj refers to the memoryview - assert res is arr or res.base.obj is arr + if not IS_PYPY: + assert res is arr or res.base.obj is arr res = np.array(view, copy=np.CopyMode.NEVER, order=order2) - assert res is arr or res.base.obj is arr + if not IS_PYPY: + assert res is arr or res.base.obj is arr else: for copy in self.false_vals: res = np.array(arr, copy=copy, order=order2) From 342e0c55d97a49749ed685d7f0f4ab03f5c8551b Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Thu, 10 Jun 2021 09:18:17 +0530 Subject: [PATCH 20/54] removed complicated checks for copy_mode --- numpy/core/src/multiarray/conversion_utils.c | 43 +++----------------- 1 file changed, 5 insertions(+), 38 deletions(-) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index c5e1ef808fd6..8208bd067dac 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -175,9 +175,7 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { int int_copymode = -1; PyObject* numpy_CopyMode = NULL; - PyObject* numpy_bool_ = NULL; npy_cache_import("numpy._globals", "CopyMode", &numpy_CopyMode); - npy_cache_import("numpy", "bool_", &numpy_bool_); if (numpy_CopyMode != NULL && PyObject_IsInstance(obj, numpy_CopyMode)) { PyObject* mode_value = PyObject_GetAttrString(obj, "value"); @@ -187,43 +185,12 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { if (!PyArray_PythonPyIntFromInt(mode_value, &int_copymode)) { return NPY_FAIL; } - } else if(numpy_bool_ != NULL && PyObject_IsInstance(obj, numpy_bool_)) { - /* - In Python 3.8.2, numpy.bool_ objects are not converted - to integers using PyLong_AsLong. Hence, the following - if condition checks whether the input obj is a numpy.bool_ object - and extracts the values by perfroming comparisions between - obj and numpy.True_/numpy.False_. - */ - PyObject *numpy_True_ = NULL, *numpy_False_ = NULL; - npy_cache_import("numpy", "True_", &numpy_True_); - npy_cache_import("numpy", "False_", &numpy_False_); - if( numpy_True_ != NULL && - PyObject_RichCompareBool(obj, numpy_True_, Py_EQ) ) { - int_copymode = 1; - } else if( numpy_False_ != NULL && - PyObject_RichCompareBool(obj, numpy_False_, Py_EQ) ) { - int_copymode = 0; - } - } else if( PyBool_Check(obj) ) { - /* - In Python 3.8.2, Boolean objects are not converted - to integers using PyLong_AsLong. Hence, the following - if condition checks whether the input obj is a Boolean object - and extracts the values by perfroming comparisions between - obj and Py_True/Py_False. - */ - if( PyObject_RichCompareBool(obj, Py_True, Py_EQ) ) { - int_copymode = 1; - } else if( PyObject_RichCompareBool(obj, Py_False, Py_EQ) ) { - int_copymode = 0; + } else { + npy_bool bool_copymode; + if( !PyArray_BoolConverter(obj, &bool_copymode) ) { + return NPY_FAIL; } - } - else { - // If obj is not an instance of numpy.CopyMode then follow - // the conventional assumption that it must be value that - // can be converted to an integer. - PyArray_PythonPyIntFromInt(obj, &int_copymode); + int_copymode = (int) bool_copymode; } if( int_copymode != NPY_COPY_ALWAYS && From 30e34729c68288514a27211dd4475bc74441aa49 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 15 Jun 2021 08:52:55 +0530 Subject: [PATCH 21/54] interface shifted --- numpy/core/multiarray.pyi | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/numpy/core/multiarray.pyi b/numpy/core/multiarray.pyi index bc33165be73b..6d6d90da2295 100644 --- a/numpy/core/multiarray.pyi +++ b/numpy/core/multiarray.pyi @@ -20,6 +20,7 @@ from typing import ( from numpy import ( # Re-exports + CopyMode, busdaycalendar as busdaycalendar, broadcast as broadcast, dtype as dtype, @@ -177,7 +178,7 @@ def array( object: _ArrayType, dtype: None = ..., *, - copy: bool = ..., + copy: bool | CopyMode = ..., order: _OrderKACF = ..., subok: L[True], ndmin: int = ..., @@ -188,7 +189,7 @@ def array( object: _ArrayLike[_SCT], dtype: None = ..., *, - copy: bool = ..., + copy: bool | CopyMode = ..., order: _OrderKACF = ..., subok: bool = ..., ndmin: int = ..., @@ -199,7 +200,7 @@ def array( object: object, dtype: None = ..., *, - copy: bool = ..., + copy: bool | CopyMode = ..., order: _OrderKACF = ..., subok: bool = ..., ndmin: int = ..., @@ -210,7 +211,7 @@ def array( object: Any, dtype: _DTypeLike[_SCT], *, - copy: bool = ..., + copy: bool | CopyMode = ..., order: _OrderKACF = ..., subok: bool = ..., ndmin: int = ..., @@ -221,7 +222,7 @@ def array( object: Any, dtype: DTypeLike, *, - copy: bool = ..., + copy: bool | CopyMode = ..., order: _OrderKACF = ..., subok: bool = ..., ndmin: int = ..., @@ -948,3 +949,9 @@ def compare_chararrays( ) -> NDArray[bool_]: ... def add_docstring(__obj: Callable[..., Any], __docstring: str) -> None: ... + +class CopyMode(enum.Enum): + + ALWAYS: L[1] + IF_NEEDED: L[0] + NEVER: L[2] From a6caf9c781e1ea318b1f128ea01d8901d21d6516 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 15 Jun 2021 08:54:25 +0530 Subject: [PATCH 22/54] Apply suggestions from code review Co-authored-by: Stefan van der Walt --- numpy/core/src/multiarray/conversion_utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 8208bd067dac..f4063373a6b6 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -169,7 +169,7 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { if (obj == Py_None) { PyErr_SetString(PyExc_ValueError, "NoneType copy mode not allowed. Please choose one of " - "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER."); + "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER."); return NPY_FAIL; } From 321e028b90526ec40df281ae543f18aa4434bcd3 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Sat, 7 Aug 2021 11:05:40 +0530 Subject: [PATCH 23/54] Shifted to CopyMode to np.array_api --- numpy/__init__.py | 5 ++-- numpy/__init__.pyi | 2 +- numpy/_globals.py | 25 ++----------------- numpy/array_api.py | 24 ++++++++++++++++++ numpy/core/multiarray.pyi | 5 +++- numpy/core/src/multiarray/conversion_utils.c | 9 ++++--- numpy/core/src/multiarray/methods.c | 3 ++- numpy/core/tests/test_api.py | 10 ++++---- numpy/core/tests/test_multiarray.py | 26 ++++++++++---------- numpy/tests/test_public_api.py | 2 ++ 10 files changed, 61 insertions(+), 50 deletions(-) create mode 100644 numpy/array_api.py diff --git a/numpy/__init__.py b/numpy/__init__.py index 476815520ed3..83d2c279efa1 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -111,7 +111,7 @@ from ._globals import ( ModuleDeprecationWarning, VisibleDeprecationWarning, - _NoValue, CopyMode + _NoValue ) # We first need to detect if we're being called as part of the numpy setup @@ -133,8 +133,7 @@ raise ImportError(msg) from e __all__ = ['ModuleDeprecationWarning', - 'VisibleDeprecationWarning', - 'CopyMode'] + 'VisibleDeprecationWarning'] # get the version using versioneer from ._version import get_versions diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index e036a32feac1..c8bdae5f09b0 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -3876,4 +3876,4 @@ class iinfo(Generic[_IntType]): @overload def __new__(cls, dtype: int | Type[int]) -> iinfo[int_]: ... @overload - def __new__(cls, dtype: str) -> iinfo[Any]: ... + def __new__(cls, dtype: str) -> iinfo[Any]: ... \ No newline at end of file diff --git a/numpy/_globals.py b/numpy/_globals.py index b7a399a793b9..97007b51e230 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -15,11 +15,9 @@ def foo(arg=np._NoValue): motivated this module. """ -import enum - __ALL__ = [ 'ModuleDeprecationWarning', 'VisibleDeprecationWarning', - '_NoValue', 'CopyMode' + '_NoValue' ] @@ -91,23 +89,4 @@ def __repr__(self): return "" -_NoValue = _NoValueType() - -class CopyMode(enum.Enum): - - ALWAYS = True - IF_NEEDED = False - NEVER = 2 - - def __bool__(self): - # For backwards compatiblity - if self == CopyMode.ALWAYS: - return True - - if self == CopyMode.IF_NEEDED: - return False - - raise TypeError(f"{self} is neither True nor False.") - - -CopyMode.__module__ = 'numpy' +_NoValue = _NoValueType() \ No newline at end of file diff --git a/numpy/array_api.py b/numpy/array_api.py new file mode 100644 index 000000000000..527ccc3fa2ba --- /dev/null +++ b/numpy/array_api.py @@ -0,0 +1,24 @@ +import enum + +__all__ = [ + 'CopyMode' + ] + +class CopyMode(enum.Enum): + + ALWAYS = True + IF_NEEDED = False + NEVER = 2 + + def __bool__(self): + # For backwards compatiblity + if self == CopyMode.ALWAYS: + return True + + if self == CopyMode.IF_NEEDED: + return False + + raise TypeError(f"{self} is neither True nor False.") + + +CopyMode.__module__ = 'numpy.array_api' diff --git a/numpy/core/multiarray.pyi b/numpy/core/multiarray.pyi index b02f7f391729..8c4526a5dd55 100644 --- a/numpy/core/multiarray.pyi +++ b/numpy/core/multiarray.pyi @@ -20,7 +20,6 @@ from typing import ( from numpy import ( # Re-exports - CopyMode, busdaycalendar as busdaycalendar, broadcast as broadcast, dtype as dtype, @@ -79,6 +78,10 @@ from numpy.typing import ( _TD64Like_co, ) +from numpy.array_api import ( + CopyMode +) + if sys.version_info >= (3, 8): from typing import SupportsIndex, final, Final, Literal as L else: diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index a1f3dafb841a..7aa1e9abe08d 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -169,13 +169,15 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { if (obj == Py_None) { PyErr_SetString(PyExc_ValueError, "NoneType copy mode not allowed. Please choose one of " - "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER."); + "np.array_api.CopyMode.ALWAYS, " + "np.array_api.CopyMode.IF_NEEDED, " + "np.array_api.CopyMode.NEVER."); return NPY_FAIL; } int int_copymode = -1; PyObject* numpy_CopyMode = NULL; - npy_cache_import("numpy._globals", "CopyMode", &numpy_CopyMode); + npy_cache_import("numpy.array_api", "CopyMode", &numpy_CopyMode); if (numpy_CopyMode != NULL && PyObject_IsInstance(obj, numpy_CopyMode)) { PyObject* mode_value = PyObject_GetAttrString(obj, "value"); @@ -198,7 +200,8 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { int_copymode != NPY_COPY_NEVER ) { PyErr_Format(PyExc_ValueError, "Unrecognized copy mode %d. Please choose one of " - "np.CopyMode.ALWAYS, np.CopyMode.IF_NEEDED, np.CopyMode.NEVER, " + "np.array_api.CopyMode.ALWAYS, np.array_api.CopyMode.IF_NEEDED, " + "np.array_api.CopyMode.NEVER, " "True/np.True_, False/np.False_", int_copymode); return NPY_FAIL; } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index f20082c4025a..3e66e55f3fe7 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -874,7 +874,8 @@ array_astype(PyArrayObject *self, if( forcecopy == NPY_COPY_NEVER ) { PyErr_SetString(PyExc_ValueError, - "Unable to avoid copy while casting in np.CopyMode.NEVER"); + "Unable to avoid copy while casting in " + "np.array_api.CopyMode.NEVER"); Py_DECREF(dtype); return NULL; } diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py index d48b208001da..e3e07138d4e0 100644 --- a/numpy/core/tests/test_api.py +++ b/numpy/core/tests/test_api.py @@ -605,22 +605,22 @@ def test_astype_copyflag(): res_true = arr.astype(np.intp, copy=True) assert not np.may_share_memory(arr, res_true) - res_always = arr.astype(np.intp, copy=np.CopyMode.ALWAYS) + res_always = arr.astype(np.intp, copy=np.array_api.CopyMode.ALWAYS) assert not np.may_share_memory(arr, res_always) res_false = arr.astype(np.intp, copy=False) # `res_false is arr` currently, but check `may_share_memory`. assert np.may_share_memory(arr, res_false) - res_if_needed = arr.astype(np.intp, copy=np.CopyMode.IF_NEEDED) + res_if_needed = arr.astype(np.intp, copy=np.array_api.CopyMode.IF_NEEDED) # `res_if_needed is arr` currently, but check `may_share_memory`. assert np.may_share_memory(arr, res_if_needed) - res_never = arr.astype(np.intp, copy=np.CopyMode.NEVER) + res_never = arr.astype(np.intp, copy=np.array_api.CopyMode.NEVER) assert np.may_share_memory(arr, res_never) # Simple tests for when a copy is necessary: res_false = arr.astype(np.float64, copy=False) assert_array_equal(res_false, arr) - res_if_needed = arr.astype(np.float64, copy=np.CopyMode.IF_NEEDED) + res_if_needed = arr.astype(np.float64, copy=np.array_api.CopyMode.IF_NEEDED) assert_array_equal(res_if_needed, arr) - assert_raises(ValueError, arr.astype, np.float64, copy=np.CopyMode.NEVER) + assert_raises(ValueError, arr.astype, np.float64, copy=np.array_api.CopyMode.NEVER) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index b9dbda43b9df..c51d2766ebc1 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7818,8 +7818,8 @@ def test_error_if_stored_buffer_info_is_corrupted(self, obj): class TestArrayCreationCopyArgument(object): - true_vals = [True, np.CopyMode.ALWAYS, np.True_] - false_vals = [False, np.CopyMode.IF_NEEDED, np.False_] + true_vals = [True, np.array_api.CopyMode.ALWAYS, np.True_] + false_vals = [False, np.array_api.CopyMode.IF_NEEDED, np.False_] def test_scalars(self): # Test both numpy and python scalars @@ -7830,9 +7830,9 @@ def test_scalars(self): # Test never-copy raises error: assert_raises(ValueError, np.array, scalar, - copy=np.CopyMode.NEVER) + copy=np.array_api.CopyMode.NEVER) assert_raises(ValueError, np.array, pyscalar, - copy=np.CopyMode.NEVER) + copy=np.array_api.CopyMode.NEVER) def test_compatible_cast(self): @@ -7861,7 +7861,7 @@ def int_types(byteswap=False): res = np.array(arr, copy=copy, dtype=int2) assert res is arr or res.base is arr - res = np.array(arr, copy=np.CopyMode.NEVER, dtype=int2) + res = np.array(arr, copy=np.array_api.CopyMode.NEVER, dtype=int2) assert res is arr or res.base is arr else: @@ -7872,7 +7872,7 @@ def int_types(byteswap=False): assert_array_equal(res, arr) assert_raises(ValueError, np.array, - arr, copy=np.CopyMode.NEVER, dtype=int2) + arr, copy=np.array_api.CopyMode.NEVER, dtype=int2) def test_buffer_interface(self): @@ -7888,7 +7888,7 @@ def test_buffer_interface(self): for copy in self.false_vals: res = np.array(view, copy=copy) assert np.may_share_memory(arr, res) - res = np.array(view, copy=np.CopyMode.NEVER) + res = np.array(view, copy=np.array_api.CopyMode.NEVER) assert np.may_share_memory(arr, res) def test_array_interfaces(self): @@ -7900,9 +7900,9 @@ class ArrayLike: arr = ArrayLike() - for copy, val in [(True, None), (np.CopyMode.ALWAYS, None), - (False, arr), (np.CopyMode.IF_NEEDED, arr), - (np.CopyMode.NEVER, arr)]: + for copy, val in [(True, None), (np.array_api.CopyMode.ALWAYS, None), + (False, arr), (np.array_api.CopyMode.IF_NEEDED, arr), + (np.array_api.CopyMode.NEVER, arr)]: res = np.array(arr, copy=copy) assert res.base is val @@ -7930,7 +7930,7 @@ def __array__(self): assert_array_equal(res, base_arr) assert res is base_arr # numpy trusts the ArrayLike - assert np.array(arr, copy=np.CopyMode.NEVER) is base_arr + assert np.array(arr, copy=np.array_api.CopyMode.NEVER) is base_arr @pytest.mark.parametrize( "arr", [np.ones(()), np.arange(81).reshape((9, 9))]) @@ -7976,7 +7976,7 @@ def test_order_mismatch(self, arr, order1, order2): if not IS_PYPY: assert res is arr or res.base.obj is arr - res = np.array(view, copy=np.CopyMode.NEVER, order=order2) + res = np.array(view, copy=np.array_api.CopyMode.NEVER, order=order2) if not IS_PYPY: assert res is arr or res.base.obj is arr else: @@ -7984,7 +7984,7 @@ def test_order_mismatch(self, arr, order1, order2): res = np.array(arr, copy=copy, order=order2) assert_array_equal(arr, res) assert_raises(ValueError, np.array, - view, copy=np.CopyMode.NEVER, order=order2) + view, copy=np.array_api.CopyMode.NEVER, order=order2) class TestArrayAttributeDeletion: diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 6e4a8dee0a7c..5fb06f71fd45 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -137,6 +137,7 @@ def test_NPY_NO_EXPORT(): # current status is fine. For others it may make sense to work on making them # private, to clean up our public API and avoid confusion. PUBLIC_MODULES = ['numpy.' + s for s in [ + "array_api", "ctypeslib", "distutils", "distutils.cpuinfo", @@ -321,6 +322,7 @@ def is_unexpected(name): # These are present in a directory with an __init__.py but cannot be imported # code_generators/ isn't installed, but present for an inplace build SKIP_LIST = [ + "numpy.array_api.CopyMode", "numpy.core.code_generators", "numpy.core.code_generators.genapi", "numpy.core.code_generators.generate_umath", From 844883cff35996edeeeebc294e00c4aaf4fe9144 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Sat, 7 Aug 2021 11:07:45 +0530 Subject: [PATCH 24/54] corrected linting issues --- numpy/_globals.py | 2 +- numpy/core/tests/test_api.py | 3 ++- numpy/core/tests/test_multiarray.py | 10 +++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/numpy/_globals.py b/numpy/_globals.py index 97007b51e230..5e8a4557a226 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -89,4 +89,4 @@ def __repr__(self): return "" -_NoValue = _NoValueType() \ No newline at end of file +_NoValue = _NoValueType() diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py index e3e07138d4e0..c2a2e712c6e0 100644 --- a/numpy/core/tests/test_api.py +++ b/numpy/core/tests/test_api.py @@ -621,6 +621,7 @@ def test_astype_copyflag(): # Simple tests for when a copy is necessary: res_false = arr.astype(np.float64, copy=False) assert_array_equal(res_false, arr) - res_if_needed = arr.astype(np.float64, copy=np.array_api.CopyMode.IF_NEEDED) + res_if_needed = arr.astype(np.float64, + copy=np.array_api.CopyMode.IF_NEEDED) assert_array_equal(res_if_needed, arr) assert_raises(ValueError, arr.astype, np.float64, copy=np.array_api.CopyMode.NEVER) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index c51d2766ebc1..4c598da1fe05 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7861,7 +7861,9 @@ def int_types(byteswap=False): res = np.array(arr, copy=copy, dtype=int2) assert res is arr or res.base is arr - res = np.array(arr, copy=np.array_api.CopyMode.NEVER, dtype=int2) + res = np.array(arr, + copy=np.array_api.CopyMode.NEVER, + dtype=int2) assert res is arr or res.base is arr else: @@ -7872,7 +7874,8 @@ def int_types(byteswap=False): assert_array_equal(res, arr) assert_raises(ValueError, np.array, - arr, copy=np.array_api.CopyMode.NEVER, dtype=int2) + arr, copy=np.array_api.CopyMode.NEVER, + dtype=int2) def test_buffer_interface(self): @@ -7976,7 +7979,8 @@ def test_order_mismatch(self, arr, order1, order2): if not IS_PYPY: assert res is arr or res.base.obj is arr - res = np.array(view, copy=np.array_api.CopyMode.NEVER, order=order2) + res = np.array(view, copy=np.array_api.CopyMode.NEVER, + order=order2) if not IS_PYPY: assert res is arr or res.base.obj is arr else: From 0f8d4c5f23a5fa55efba96034c0215cb683f9e02 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Sat, 7 Aug 2021 11:23:48 +0530 Subject: [PATCH 25/54] fixed linting issues --- numpy/core/tests/test_api.py | 3 ++- numpy/core/tests/test_multiarray.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py index c2a2e712c6e0..ea2bc8e3d2d1 100644 --- a/numpy/core/tests/test_api.py +++ b/numpy/core/tests/test_api.py @@ -624,4 +624,5 @@ def test_astype_copyflag(): res_if_needed = arr.astype(np.float64, copy=np.array_api.CopyMode.IF_NEEDED) assert_array_equal(res_if_needed, arr) - assert_raises(ValueError, arr.astype, np.float64, copy=np.array_api.CopyMode.NEVER) + assert_raises(ValueError, arr.astype, np.float64, + copy=np.array_api.CopyMode.NEVER) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 4c598da1fe05..9621b700cc1b 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7988,7 +7988,8 @@ def test_order_mismatch(self, arr, order1, order2): res = np.array(arr, copy=copy, order=order2) assert_array_equal(arr, res) assert_raises(ValueError, np.array, - view, copy=np.array_api.CopyMode.NEVER, order=order2) + view, copy=np.array_api.CopyMode.NEVER, + order=order2) class TestArrayAttributeDeletion: From 698fb6db917e63621df9e9d50335245dd0ab5a2e Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Wed, 18 Aug 2021 12:15:00 +0530 Subject: [PATCH 26/54] Made _CopyMode private --- numpy/__init__.py | 2 +- numpy/__init__.pyi | 12 ++++----- numpy/_globals.py | 24 +++++++++++++++++- numpy/array_api.py | 24 ------------------ numpy/core/multiarray.pyi | 14 +++++------ numpy/core/src/multiarray/conversion_utils.c | 12 ++++----- numpy/core/src/multiarray/methods.c | 2 +- numpy/core/tests/test_api.py | 10 ++++---- numpy/core/tests/test_multiarray.py | 26 ++++++++++---------- numpy/tests/test_public_api.py | 2 -- 10 files changed, 62 insertions(+), 66 deletions(-) delete mode 100644 numpy/array_api.py diff --git a/numpy/__init__.py b/numpy/__init__.py index 83d2c279efa1..5e7f74059688 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -111,7 +111,7 @@ from ._globals import ( ModuleDeprecationWarning, VisibleDeprecationWarning, - _NoValue + _NoValue, _CopyMode ) # We first need to detect if we're being called as part of the numpy setup diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index c8bdae5f09b0..49c17a015345 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1912,7 +1912,7 @@ class ndarray(_ArrayOrScalarCommon, Generic[_ShapeType, _DType_co]): order: _OrderKACF = ..., casting: _CastingKind = ..., subok: bool = ..., - copy: bool | CopyMode = ..., + copy: bool | _CopyMode = ..., ) -> NDArray[_ScalarType]: ... @overload def astype( @@ -1921,7 +1921,7 @@ class ndarray(_ArrayOrScalarCommon, Generic[_ShapeType, _DType_co]): order: _OrderKACF = ..., casting: _CastingKind = ..., subok: bool = ..., - copy: bool | CopyMode = ..., + copy: bool | _CopyMode = ..., ) -> NDArray[Any]: ... @overload @@ -2928,7 +2928,7 @@ class generic(_ArrayOrScalarCommon): order: _OrderKACF = ..., casting: _CastingKind = ..., subok: bool = ..., - copy: bool | CopyMode = ..., + copy: bool | _CopyMode = ..., ) -> _ScalarType: ... @overload def astype( @@ -2937,7 +2937,7 @@ class generic(_ArrayOrScalarCommon): order: _OrderKACF = ..., casting: _CastingKind = ..., subok: bool = ..., - copy: bool | CopyMode = ..., + copy: bool | _CopyMode = ..., ) -> Any: ... # NOTE: `view` will perform a 0D->scalar cast, @@ -3690,7 +3690,7 @@ trunc: _UFunc_Nin1_Nout1[L['trunc'], L[7], None] abs = absolute -class CopyMode(enum.Enum): +class _CopyMode(enum.Enum): ALWAYS: L[1] IF_NEEDED: L[0] @@ -3876,4 +3876,4 @@ class iinfo(Generic[_IntType]): @overload def __new__(cls, dtype: int | Type[int]) -> iinfo[int_]: ... @overload - def __new__(cls, dtype: str) -> iinfo[Any]: ... \ No newline at end of file + def __new__(cls, dtype: str) -> iinfo[Any]: ... diff --git a/numpy/_globals.py b/numpy/_globals.py index 5e8a4557a226..fd8025443dd8 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -15,9 +15,11 @@ def foo(arg=np._NoValue): motivated this module. """ +import enum + __ALL__ = [ 'ModuleDeprecationWarning', 'VisibleDeprecationWarning', - '_NoValue' + '_NoValue', '_CopyMode' ] @@ -90,3 +92,23 @@ def __repr__(self): _NoValue = _NoValueType() + + +class _CopyMode(enum.Enum): + + ALWAYS = True + IF_NEEDED = False + NEVER = 2 + + def __bool__(self): + # For backwards compatiblity + if self == _CopyMode.ALWAYS: + return True + + if self == _CopyMode.IF_NEEDED: + return False + + raise TypeError(f"{self} is neither True nor False.") + + +_CopyMode.__module__ = 'numpy' \ No newline at end of file diff --git a/numpy/array_api.py b/numpy/array_api.py deleted file mode 100644 index 527ccc3fa2ba..000000000000 --- a/numpy/array_api.py +++ /dev/null @@ -1,24 +0,0 @@ -import enum - -__all__ = [ - 'CopyMode' - ] - -class CopyMode(enum.Enum): - - ALWAYS = True - IF_NEEDED = False - NEVER = 2 - - def __bool__(self): - # For backwards compatiblity - if self == CopyMode.ALWAYS: - return True - - if self == CopyMode.IF_NEEDED: - return False - - raise TypeError(f"{self} is neither True nor False.") - - -CopyMode.__module__ = 'numpy.array_api' diff --git a/numpy/core/multiarray.pyi b/numpy/core/multiarray.pyi index 8c4526a5dd55..7ae831b533c8 100644 --- a/numpy/core/multiarray.pyi +++ b/numpy/core/multiarray.pyi @@ -79,7 +79,7 @@ from numpy.typing import ( ) from numpy.array_api import ( - CopyMode + _CopyMode ) if sys.version_info >= (3, 8): @@ -181,7 +181,7 @@ def array( object: _ArrayType, dtype: None = ..., *, - copy: bool | CopyMode = ..., + copy: bool | _CopyMode = ..., order: _OrderKACF = ..., subok: L[True], ndmin: int = ..., @@ -192,7 +192,7 @@ def array( object: _ArrayLike[_SCT], dtype: None = ..., *, - copy: bool | CopyMode = ..., + copy: bool | _CopyMode = ..., order: _OrderKACF = ..., subok: bool = ..., ndmin: int = ..., @@ -203,7 +203,7 @@ def array( object: object, dtype: None = ..., *, - copy: bool | CopyMode = ..., + copy: bool | _CopyMode = ..., order: _OrderKACF = ..., subok: bool = ..., ndmin: int = ..., @@ -214,7 +214,7 @@ def array( object: Any, dtype: _DTypeLike[_SCT], *, - copy: bool | CopyMode = ..., + copy: bool | _CopyMode = ..., order: _OrderKACF = ..., subok: bool = ..., ndmin: int = ..., @@ -225,7 +225,7 @@ def array( object: Any, dtype: DTypeLike, *, - copy: bool | CopyMode = ..., + copy: bool | _CopyMode = ..., order: _OrderKACF = ..., subok: bool = ..., ndmin: int = ..., @@ -1005,7 +1005,7 @@ class flagsobj: def __getitem__(self, key: _GetItemKeys) -> bool: ... def __setitem__(self, key: _SetItemKeys, value: bool) -> None: ... -class CopyMode(enum.Enum): +class _CopyMode(enum.Enum): ALWAYS: L[1] IF_NEEDED: L[0] diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 7aa1e9abe08d..15fe4bde2506 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -169,15 +169,15 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { if (obj == Py_None) { PyErr_SetString(PyExc_ValueError, "NoneType copy mode not allowed. Please choose one of " - "np.array_api.CopyMode.ALWAYS, " - "np.array_api.CopyMode.IF_NEEDED, " - "np.array_api.CopyMode.NEVER."); + "np.array_api._CopyMode.ALWAYS, " + "np.array_api._CopyMode.IF_NEEDED, " + "np.array_api._CopyMode.NEVER."); return NPY_FAIL; } int int_copymode = -1; PyObject* numpy_CopyMode = NULL; - npy_cache_import("numpy.array_api", "CopyMode", &numpy_CopyMode); + npy_cache_import("numpy", "_CopyMode", &numpy_CopyMode); if (numpy_CopyMode != NULL && PyObject_IsInstance(obj, numpy_CopyMode)) { PyObject* mode_value = PyObject_GetAttrString(obj, "value"); @@ -200,8 +200,8 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { int_copymode != NPY_COPY_NEVER ) { PyErr_Format(PyExc_ValueError, "Unrecognized copy mode %d. Please choose one of " - "np.array_api.CopyMode.ALWAYS, np.array_api.CopyMode.IF_NEEDED, " - "np.array_api.CopyMode.NEVER, " + "np._CopyMode.ALWAYS, np._CopyMode.IF_NEEDED, " + "np._CopyMode.NEVER, " "True/np.True_, False/np.False_", int_copymode); return NPY_FAIL; } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 3e66e55f3fe7..ffa735b38a48 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -875,7 +875,7 @@ array_astype(PyArrayObject *self, if( forcecopy == NPY_COPY_NEVER ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while casting in " - "np.array_api.CopyMode.NEVER"); + "np._CopyMode.NEVER"); Py_DECREF(dtype); return NULL; } diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py index ea2bc8e3d2d1..d3c7211cd1b9 100644 --- a/numpy/core/tests/test_api.py +++ b/numpy/core/tests/test_api.py @@ -605,24 +605,24 @@ def test_astype_copyflag(): res_true = arr.astype(np.intp, copy=True) assert not np.may_share_memory(arr, res_true) - res_always = arr.astype(np.intp, copy=np.array_api.CopyMode.ALWAYS) + res_always = arr.astype(np.intp, copy=np._CopyMode.ALWAYS) assert not np.may_share_memory(arr, res_always) res_false = arr.astype(np.intp, copy=False) # `res_false is arr` currently, but check `may_share_memory`. assert np.may_share_memory(arr, res_false) - res_if_needed = arr.astype(np.intp, copy=np.array_api.CopyMode.IF_NEEDED) + res_if_needed = arr.astype(np.intp, copy=np._CopyMode.IF_NEEDED) # `res_if_needed is arr` currently, but check `may_share_memory`. assert np.may_share_memory(arr, res_if_needed) - res_never = arr.astype(np.intp, copy=np.array_api.CopyMode.NEVER) + res_never = arr.astype(np.intp, copy=np._CopyMode.NEVER) assert np.may_share_memory(arr, res_never) # Simple tests for when a copy is necessary: res_false = arr.astype(np.float64, copy=False) assert_array_equal(res_false, arr) res_if_needed = arr.astype(np.float64, - copy=np.array_api.CopyMode.IF_NEEDED) + copy=np._CopyMode.IF_NEEDED) assert_array_equal(res_if_needed, arr) assert_raises(ValueError, arr.astype, np.float64, - copy=np.array_api.CopyMode.NEVER) + copy=np._CopyMode.NEVER) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 9621b700cc1b..9c56df2ba3e2 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7818,8 +7818,8 @@ def test_error_if_stored_buffer_info_is_corrupted(self, obj): class TestArrayCreationCopyArgument(object): - true_vals = [True, np.array_api.CopyMode.ALWAYS, np.True_] - false_vals = [False, np.array_api.CopyMode.IF_NEEDED, np.False_] + true_vals = [True, np._CopyMode.ALWAYS, np.True_] + false_vals = [False, np._CopyMode.IF_NEEDED, np.False_] def test_scalars(self): # Test both numpy and python scalars @@ -7830,9 +7830,9 @@ def test_scalars(self): # Test never-copy raises error: assert_raises(ValueError, np.array, scalar, - copy=np.array_api.CopyMode.NEVER) + copy=np._CopyMode.NEVER) assert_raises(ValueError, np.array, pyscalar, - copy=np.array_api.CopyMode.NEVER) + copy=np._CopyMode.NEVER) def test_compatible_cast(self): @@ -7862,7 +7862,7 @@ def int_types(byteswap=False): assert res is arr or res.base is arr res = np.array(arr, - copy=np.array_api.CopyMode.NEVER, + copy=np._CopyMode.NEVER, dtype=int2) assert res is arr or res.base is arr @@ -7874,7 +7874,7 @@ def int_types(byteswap=False): assert_array_equal(res, arr) assert_raises(ValueError, np.array, - arr, copy=np.array_api.CopyMode.NEVER, + arr, copy=np._CopyMode.NEVER, dtype=int2) def test_buffer_interface(self): @@ -7891,7 +7891,7 @@ def test_buffer_interface(self): for copy in self.false_vals: res = np.array(view, copy=copy) assert np.may_share_memory(arr, res) - res = np.array(view, copy=np.array_api.CopyMode.NEVER) + res = np.array(view, copy=np._CopyMode.NEVER) assert np.may_share_memory(arr, res) def test_array_interfaces(self): @@ -7903,9 +7903,9 @@ class ArrayLike: arr = ArrayLike() - for copy, val in [(True, None), (np.array_api.CopyMode.ALWAYS, None), - (False, arr), (np.array_api.CopyMode.IF_NEEDED, arr), - (np.array_api.CopyMode.NEVER, arr)]: + for copy, val in [(True, None), (np._CopyMode.ALWAYS, None), + (False, arr), (np._CopyMode.IF_NEEDED, arr), + (np._CopyMode.NEVER, arr)]: res = np.array(arr, copy=copy) assert res.base is val @@ -7933,7 +7933,7 @@ def __array__(self): assert_array_equal(res, base_arr) assert res is base_arr # numpy trusts the ArrayLike - assert np.array(arr, copy=np.array_api.CopyMode.NEVER) is base_arr + assert np.array(arr, copy=np._CopyMode.NEVER) is base_arr @pytest.mark.parametrize( "arr", [np.ones(()), np.arange(81).reshape((9, 9))]) @@ -7979,7 +7979,7 @@ def test_order_mismatch(self, arr, order1, order2): if not IS_PYPY: assert res is arr or res.base.obj is arr - res = np.array(view, copy=np.array_api.CopyMode.NEVER, + res = np.array(view, copy=np._CopyMode.NEVER, order=order2) if not IS_PYPY: assert res is arr or res.base.obj is arr @@ -7988,7 +7988,7 @@ def test_order_mismatch(self, arr, order1, order2): res = np.array(arr, copy=copy, order=order2) assert_array_equal(arr, res) assert_raises(ValueError, np.array, - view, copy=np.array_api.CopyMode.NEVER, + view, copy=np._CopyMode.NEVER, order=order2) diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 5fb06f71fd45..6e4a8dee0a7c 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -137,7 +137,6 @@ def test_NPY_NO_EXPORT(): # current status is fine. For others it may make sense to work on making them # private, to clean up our public API and avoid confusion. PUBLIC_MODULES = ['numpy.' + s for s in [ - "array_api", "ctypeslib", "distutils", "distutils.cpuinfo", @@ -322,7 +321,6 @@ def is_unexpected(name): # These are present in a directory with an __init__.py but cannot be imported # code_generators/ isn't installed, but present for an inplace build SKIP_LIST = [ - "numpy.array_api.CopyMode", "numpy.core.code_generators", "numpy.core.code_generators.genapi", "numpy.core.code_generators.generate_umath", From b341e4c3249817d2e14ddf71aa850a8a896b9303 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Wed, 18 Aug 2021 12:15:23 +0530 Subject: [PATCH 27/54] Fixed linting issues --- numpy/_globals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/_globals.py b/numpy/_globals.py index fd8025443dd8..133ab11ccf3b 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -111,4 +111,4 @@ def __bool__(self): raise TypeError(f"{self} is neither True nor False.") -_CopyMode.__module__ = 'numpy' \ No newline at end of file +_CopyMode.__module__ = 'numpy' From 45dbdc9d8fd3fa7fbabbddf690f6c892cc24aa1d Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Fri, 3 Sep 2021 15:10:23 +0530 Subject: [PATCH 28/54] CopyMode added to np.array_api --- numpy/array_api/_creation_functions.py | 2 +- numpy/array_api/tests/test_creation_functions.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/numpy/array_api/_creation_functions.py b/numpy/array_api/_creation_functions.py index e9c01e7e6106..c15d54db114b 100644 --- a/numpy/array_api/_creation_functions.py +++ b/numpy/array_api/_creation_functions.py @@ -43,7 +43,7 @@ def asarray( *, dtype: Optional[Dtype] = None, device: Optional[Device] = None, - copy: Optional[bool] = None, + copy: Optional[Union[bool | np._CopyMode]] = None, ) -> Array: """ Array API compatible wrapper for :py:func:`np.asarray `. diff --git a/numpy/array_api/tests/test_creation_functions.py b/numpy/array_api/tests/test_creation_functions.py index 3cb8865cd1c5..b0c99cd45e39 100644 --- a/numpy/array_api/tests/test_creation_functions.py +++ b/numpy/array_api/tests/test_creation_functions.py @@ -56,12 +56,17 @@ def test_asarray_copy(): a[0] = 0 assert all(b[0] == 1) assert all(a[0] == 0) - # Once copy=False is implemented, replace this with - # a = asarray([1]) - # b = asarray(a, copy=False) - # a[0] = 0 - # assert all(b[0] == 0) + a = asarray([1]) + b = asarray(a, copy=np._CopyMode.ALWAYS) + a[0] = 0 + assert all(b[0] == 1) + assert all(a[0] == 0) + a = asarray([1]) + b = asarray(a, copy=np._CopyMode.NEVER) + a[0] = 0 + assert all(b[0] == 0) assert_raises(NotImplementedError, lambda: asarray(a, copy=False)) + assert_raises(NotImplementedError, lambda: asarray(a, copy=np._CopyMode.NEVER)) def test_arange_errors(): From a39312cf4eca9dc9573737a01f16fee24efb6c0a Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Fri, 3 Sep 2021 15:15:18 +0530 Subject: [PATCH 29/54] fixed linting issues --- numpy/array_api/tests/test_creation_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/array_api/tests/test_creation_functions.py b/numpy/array_api/tests/test_creation_functions.py index b0c99cd45e39..9d2909e91754 100644 --- a/numpy/array_api/tests/test_creation_functions.py +++ b/numpy/array_api/tests/test_creation_functions.py @@ -66,7 +66,8 @@ def test_asarray_copy(): a[0] = 0 assert all(b[0] == 0) assert_raises(NotImplementedError, lambda: asarray(a, copy=False)) - assert_raises(NotImplementedError, lambda: asarray(a, copy=np._CopyMode.NEVER)) + assert_raises(NotImplementedError, lambda: asarray(a, + copy=np._CopyMode.NEVER)) def test_arange_errors(): From c2acd5b25a04783fbbe3ba32426039e4dbe9207e Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Fri, 3 Sep 2021 15:17:58 +0530 Subject: [PATCH 30/54] fixed linting issues --- numpy/array_api/tests/test_creation_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/array_api/tests/test_creation_functions.py b/numpy/array_api/tests/test_creation_functions.py index 9d2909e91754..7182209dcd48 100644 --- a/numpy/array_api/tests/test_creation_functions.py +++ b/numpy/array_api/tests/test_creation_functions.py @@ -66,8 +66,8 @@ def test_asarray_copy(): a[0] = 0 assert all(b[0] == 0) assert_raises(NotImplementedError, lambda: asarray(a, copy=False)) - assert_raises(NotImplementedError, lambda: asarray(a, - copy=np._CopyMode.NEVER)) + assert_raises(NotImplementedError, + lambda: asarray(a, copy=np._CopyMode.NEVER)) def test_arange_errors(): From 56647dd47345a7fd24b4ee8d9d52025fcdc3b9ae Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Sat, 4 Sep 2021 22:33:52 +0530 Subject: [PATCH 31/54] Addressed reviews --- numpy/__init__.pyi | 1 - numpy/_globals.py | 2 +- numpy/array_api/_creation_functions.py | 6 +++--- numpy/array_api/tests/test_creation_functions.py | 2 +- numpy/core/multiarray.pyi | 7 +------ 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index dafedeb56a35..30b0944fd3b9 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -3689,7 +3689,6 @@ trunc: _UFunc_Nin1_Nout1[L['trunc'], L[7], None] abs = absolute class _CopyMode(enum.Enum): - ALWAYS: L[1] IF_NEEDED: L[0] NEVER: L[2] diff --git a/numpy/_globals.py b/numpy/_globals.py index 133ab11ccf3b..d458fc9c463a 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -108,7 +108,7 @@ def __bool__(self): if self == _CopyMode.IF_NEEDED: return False - raise TypeError(f"{self} is neither True nor False.") + raise ValueError(f"{self} is neither True nor False.") _CopyMode.__module__ = 'numpy' diff --git a/numpy/array_api/_creation_functions.py b/numpy/array_api/_creation_functions.py index c15d54db114b..2d6cf4414228 100644 --- a/numpy/array_api/_creation_functions.py +++ b/numpy/array_api/_creation_functions.py @@ -43,7 +43,7 @@ def asarray( *, dtype: Optional[Dtype] = None, device: Optional[Device] = None, - copy: Optional[Union[bool | np._CopyMode]] = None, + copy: Optional[Union[bool, np._CopyMode]] = None, ) -> Array: """ Array API compatible wrapper for :py:func:`np.asarray `. @@ -57,11 +57,11 @@ def asarray( _check_valid_dtype(dtype) if device not in ["cpu", None]: raise ValueError(f"Unsupported device {device!r}") - if copy is False: + if copy in (False, np._CopyMode.IF_NEEDED): # Note: copy=False is not yet implemented in np.asarray raise NotImplementedError("copy=False is not yet implemented") if isinstance(obj, Array) and (dtype is None or obj.dtype == dtype): - if copy is True: + if copy in (True, np._CopyMode.ALWAYS): return Array._new(np.array(obj._array, copy=True, dtype=dtype)) return obj if dtype is None and isinstance(obj, int) and (obj > 2 ** 64 or obj < -(2 ** 63)): diff --git a/numpy/array_api/tests/test_creation_functions.py b/numpy/array_api/tests/test_creation_functions.py index 7182209dcd48..2ee23a47bc13 100644 --- a/numpy/array_api/tests/test_creation_functions.py +++ b/numpy/array_api/tests/test_creation_functions.py @@ -67,7 +67,7 @@ def test_asarray_copy(): assert all(b[0] == 0) assert_raises(NotImplementedError, lambda: asarray(a, copy=False)) assert_raises(NotImplementedError, - lambda: asarray(a, copy=np._CopyMode.NEVER)) + lambda: asarray(a, copy=np._CopyMode.IF_NEEDED)) def test_arange_errors(): diff --git a/numpy/core/multiarray.pyi b/numpy/core/multiarray.pyi index 97e9c3498276..501e55634a85 100644 --- a/numpy/core/multiarray.pyi +++ b/numpy/core/multiarray.pyi @@ -51,6 +51,7 @@ from numpy import ( _ModeKind, _SupportsBuffer, _IOProtocol, + _CopyMode ) from numpy.typing import ( @@ -1012,9 +1013,3 @@ class flagsobj: def owndata(self) -> bool: ... def __getitem__(self, key: _GetItemKeys) -> bool: ... def __setitem__(self, key: _SetItemKeys, value: bool) -> None: ... - -class _CopyMode(enum.Enum): - - ALWAYS: L[1] - IF_NEEDED: L[0] - NEVER: L[2] \ No newline at end of file From 790f927fcedb7debc477746ffb7a4a2eb1668693 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 2 Nov 2021 13:04:51 +0530 Subject: [PATCH 32/54] Addressed reviews --- doc/source/reference/c-api/array.rst | 5 ---- numpy/core/include/numpy/ndarraytypes.h | 6 ++--- numpy/core/src/multiarray/conversion_utils.c | 25 +++++--------------- numpy/core/src/multiarray/conversion_utils.h | 2 +- numpy/core/src/multiarray/ctors.c | 12 +++++----- numpy/core/src/multiarray/methods.c | 4 ++-- numpy/core/src/multiarray/multiarraymodule.c | 6 ++--- numpy/core/tests/test_multiarray.py | 15 ++++++++++++ 8 files changed, 36 insertions(+), 39 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index bb440582548c..60037af8a0a1 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -456,11 +456,6 @@ From other objects Make sure the returned array can be written to. - .. c:macro:: NPY_ARRAY_ENSURECOPY - - Make sure a copy is made of *op*. If this flag is not - present, data is not copied if it can be avoided. - .. c:macro:: NPY_ARRAY_ENSUREARRAY Make sure the result is a base-class ndarray. By diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index a1d1c01dc301..566eae357ffc 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -449,11 +449,11 @@ typedef struct { int len; } PyArray_Dims; -typedef enum PyNpCopyMode { +typedef enum { NPY_COPY_IF_NEEDED, NPY_COPY_ALWAYS, NPY_COPY_NEVER -} PyArray_CopyMode; +} _PyArray_CopyMode; typedef struct { /* @@ -938,7 +938,7 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); #define NPY_ARRAY_UPDATEIFCOPY 0x1000 /* Deprecated in 1.14 */ #define NPY_ARRAY_WRITEBACKIFCOPY 0x2000 -#define NPY_ARRAY_ENSURENOCOPY 0x4000 +#define _NPY_ARRAY_ENSURENOCOPY 0x4000 /* * NOTE: there are also internal flags defined in multiarray/arrayobject.h, diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 59e3b49228f6..983890433e6b 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -166,13 +166,10 @@ PyArray_OptionalIntpConverter(PyObject *obj, PyArray_Dims *seq) } NPY_NO_EXPORT int -PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { +PyArray_CopyConverter(PyObject *obj, _PyArray_CopyMode *copymode) { if (obj == Py_None) { PyErr_SetString(PyExc_ValueError, - "NoneType copy mode not allowed. Please choose one of " - "np.array_api._CopyMode.ALWAYS, " - "np.array_api._CopyMode.IF_NEEDED, " - "np.array_api._CopyMode.NEVER."); + "NoneType copy mode not allowed."); return NPY_FAIL; } @@ -180,7 +177,7 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { PyObject* numpy_CopyMode = NULL; npy_cache_import("numpy", "_CopyMode", &numpy_CopyMode); - if (numpy_CopyMode != NULL && PyObject_IsInstance(obj, numpy_CopyMode)) { + if (numpy_CopyMode != NULL && PyObject_TypeCheck(obj, numpy_CopyMode)) { PyObject* mode_value = PyObject_GetAttrString(obj, "value"); if (mode_value == NULL) { return NPY_FAIL; @@ -188,7 +185,8 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { if (!PyArray_PythonPyIntFromInt(mode_value, &int_copymode)) { return NPY_FAIL; } - } else { + } + else { npy_bool bool_copymode; if( !PyArray_BoolConverter(obj, &bool_copymode) ) { return NPY_FAIL; @@ -196,18 +194,7 @@ PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copymode) { int_copymode = (int) bool_copymode; } - if( int_copymode != NPY_COPY_ALWAYS && - int_copymode != NPY_COPY_IF_NEEDED && - int_copymode != NPY_COPY_NEVER ) { - PyErr_Format(PyExc_ValueError, - "Unrecognized copy mode %d. Please choose one of " - "np._CopyMode.ALWAYS, np._CopyMode.IF_NEEDED, " - "np._CopyMode.NEVER, " - "True/np.True_, False/np.False_", int_copymode); - return NPY_FAIL; - } - - *copymode = (PyArray_CopyMode) int_copymode; + *copymode = (_PyArray_CopyMode) int_copymode; return NPY_SUCCEED; } diff --git a/numpy/core/src/multiarray/conversion_utils.h b/numpy/core/src/multiarray/conversion_utils.h index 4662c6a8b76f..643b67d5992e 100644 --- a/numpy/core/src/multiarray/conversion_utils.h +++ b/numpy/core/src/multiarray/conversion_utils.h @@ -10,7 +10,7 @@ NPY_NO_EXPORT int PyArray_OptionalIntpConverter(PyObject *obj, PyArray_Dims *seq); NPY_NO_EXPORT int -PyArray_CopyConverter(PyObject *obj, PyArray_CopyMode *copyflag); +PyArray_CopyConverter(PyObject *obj, _PyArray_CopyMode *copyflag); NPY_NO_EXPORT int PyArray_BufferConverter(PyObject *obj, PyArray_Chunk *buf); diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 27fd3a057c93..fdc393b976de 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1305,7 +1305,7 @@ _array_from_array_like(PyObject *op, PyErr_Clear(); } else { - tmp = _array_from_buffer_3118(memoryview); // Assume: never creates a copy + tmp = _array_from_buffer_3118(memoryview); Py_DECREF(memoryview); if (tmp == NULL) { return NULL; @@ -1347,7 +1347,7 @@ _array_from_array_like(PyObject *op, * this should be changed! */ if (!writeable && tmp == Py_NotImplemented) { - tmp = PyArray_FromArrayAttr(op, requested_dtype, context); // Assume: array was copied. + tmp = PyArray_FromArrayAttr(op, requested_dtype, context); if (tmp == NULL) { return NULL; } @@ -1738,7 +1738,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, /* Create a new array and copy the data */ Py_INCREF(dtype); /* hold on in case of a subarray that is replaced */ - if( flags & NPY_ARRAY_ENSURENOCOPY ) { + if( flags & _NPY_ARRAY_ENSURENOCOPY ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating " "an array from descriptor."); @@ -1802,7 +1802,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, * NPY_ARRAY_ALIGNED, * NPY_ARRAY_WRITEABLE, * NPY_ARRAY_NOTSWAPPED, - * NPY_ARRAY_ENSURECOPY, + * _NPY_ARRAY_ENSURECOPY, * NPY_ARRAY_UPDATEIFCOPY, * NPY_ARRAY_WRITEBACKIFCOPY, * NPY_ARRAY_FORCECAST, @@ -1872,7 +1872,7 @@ PyArray_CheckFromAny(PyObject *op, PyArray_Descr *descr, int min_depth, if ((requires & NPY_ARRAY_ELEMENTSTRIDES) && !PyArray_ElementStrides(obj)) { PyObject *ret; - if( requires & NPY_ARRAY_ENSURENOCOPY ) { + if( requires & _NPY_ARRAY_ENSURENOCOPY ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating a new array."); return NULL; @@ -1952,7 +1952,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) if (copy) { - if( flags & NPY_ARRAY_ENSURENOCOPY ) { + if( flags & _NPY_ARRAY_ENSURENOCOPY ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating " "an array from given array."); diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index bb0006e327e9..7530a2e36cce 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -833,7 +833,7 @@ array_astype(PyArrayObject *self, */ NPY_CASTING casting = NPY_UNSAFE_CASTING; NPY_ORDER order = NPY_KEEPORDER; - PyArray_CopyMode forcecopy = 1; + _PyArray_CopyMode forcecopy = 1; int subok = 1; NPY_PREPARE_ARGPARSER; if (npy_parse_arguments("astype", args, len_args, kwnames, @@ -877,7 +877,7 @@ array_astype(PyArrayObject *self, if( forcecopy == NPY_COPY_NEVER ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while casting in " - "np._CopyMode.NEVER"); + "never copy mode."); Py_DECREF(dtype); return NULL; } diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index c00f1404518e..bf60314ad494 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1560,7 +1560,7 @@ _prepend_ones(PyArrayObject *arr, int nd, int ndmin, NPY_ORDER order) static NPY_INLINE PyObject * _array_fromobject_generic( - PyObject *op, PyArray_Descr *type, PyArray_CopyMode copy, NPY_ORDER order, + PyObject *op, PyArray_Descr *type, _PyArray_CopyMode copy, NPY_ORDER order, npy_bool subok, int ndmin) { PyArrayObject *oparr = NULL, *ret = NULL; @@ -1623,7 +1623,7 @@ _array_fromobject_generic( if (copy == NPY_COPY_ALWAYS) { flags = NPY_ARRAY_ENSURECOPY; } else if( copy == NPY_COPY_NEVER ) { - flags = NPY_ARRAY_ENSURENOCOPY; + flags = _NPY_ARRAY_ENSURENOCOPY; } if (order == NPY_CORDER) { flags |= NPY_ARRAY_C_CONTIGUOUS; @@ -1668,7 +1668,7 @@ array_array(PyObject *NPY_UNUSED(ignored), { PyObject *op; npy_bool subok = NPY_FALSE; - PyArray_CopyMode copy = NPY_COPY_ALWAYS; + _PyArray_CopyMode copy = NPY_COPY_ALWAYS; int ndmin = 0; PyArray_Descr *type = NULL; NPY_ORDER order = NPY_KEEPORDER; diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index fa7f254a694f..3919f31d8940 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7816,6 +7816,11 @@ def test_error_if_stored_buffer_info_is_corrupted(self, obj): class TestArrayCreationCopyArgument(object): + class RaiseOnBool: + + def __bool__(self): + raise ValueError + true_vals = [True, np._CopyMode.ALWAYS, np.True_] false_vals = [False, np._CopyMode.IF_NEEDED, np.False_] @@ -7831,6 +7836,10 @@ def test_scalars(self): copy=np._CopyMode.NEVER) assert_raises(ValueError, np.array, pyscalar, copy=np._CopyMode.NEVER) + assert_raises(ValueError, np.array, pyscalar, + copy=None) + assert_raises(ValueError, np.array, pyscalar, + copy=self.RaiseOnBool()) def test_compatible_cast(self): @@ -7874,6 +7883,9 @@ def int_types(byteswap=False): assert_raises(ValueError, np.array, arr, copy=np._CopyMode.NEVER, dtype=int2) + assert_raises(ValueError, np.array, + arr, copy=None, + dtype=int2) def test_buffer_interface(self): @@ -7988,6 +8000,9 @@ def test_order_mismatch(self, arr, order1, order2): assert_raises(ValueError, np.array, view, copy=np._CopyMode.NEVER, order=order2) + assert_raises(ValueError, np.array, + view, copy=None, + order=order2) class TestArrayAttributeDeletion: From 267778847df8afdc25c91ced9e467902eb1cb94f Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 2 Nov 2021 16:44:06 +0530 Subject: [PATCH 33/54] Fixed type check --- numpy/core/src/multiarray/conversion_utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 983890433e6b..d8b5ea6b42d8 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -177,7 +177,7 @@ PyArray_CopyConverter(PyObject *obj, _PyArray_CopyMode *copymode) { PyObject* numpy_CopyMode = NULL; npy_cache_import("numpy", "_CopyMode", &numpy_CopyMode); - if (numpy_CopyMode != NULL && PyObject_TypeCheck(obj, numpy_CopyMode)) { + if (numpy_CopyMode != NULL && PyObject_Type(obj) == numpy_CopyMode) { PyObject* mode_value = PyObject_GetAttrString(obj, "value"); if (mode_value == NULL) { return NPY_FAIL; From d9a9785ada0b4a2b8694511dd854121f618f4f56 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Thu, 4 Nov 2021 14:11:54 +0530 Subject: [PATCH 34/54] Addressed reviews and increased code coverage --- numpy/_globals.py | 12 ++++++++++++ numpy/core/src/multiarray/_multiarray_tests.c.src | 14 ++++++++++++++ numpy/core/src/multiarray/array_coercion.c | 1 - numpy/core/src/multiarray/conversion_utils.c | 6 +++--- numpy/core/src/multiarray/ctors.c | 5 +++-- numpy/core/tests/test_multiarray.py | 13 +++++++++++-- 6 files changed, 43 insertions(+), 8 deletions(-) diff --git a/numpy/_globals.py b/numpy/_globals.py index d458fc9c463a..b17ca19798be 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -95,6 +95,18 @@ def __repr__(self): class _CopyMode(enum.Enum): + """ + An enumeration for the copy modes supported + by numpy. The following three modes are supported, + + - ALWAYS: This means that a deep copy of the input + array will always be taken. + - IF_NEEDED: This means that a deep copy of the input + array will be taken only if necessary. + - NEVER: This means that the deep copy will never be taken. + If a copy cannot be avoided then a `ValueError` will be + raised. + """ ALWAYS = True IF_NEEDED = False diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index e945d07712e5..f21a901c1d9a 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -2363,6 +2363,17 @@ run_intp_converter(PyObject* NPY_UNUSED(self), PyObject *args) return tup; } +/* used to test _NPY_ARRAY_ENSURENOCOPY raises ValueError */ +static PyObject* +npy_ensurenocopy(PyObject* NPY_UNUSED(self), PyObject* args) +{ + int flags = _NPY_ARRAY_ENSURENOCOPY; + if (!PyArray_CheckFromAny(args, NULL, 0, 0, flags, NULL)) { + return NULL; + } + Py_RETURN_NONE; +} + static PyMethodDef Multiarray_TestsMethods[] = { {"argparse_example_function", (PyCFunction)argparse_example_function, @@ -2424,6 +2435,9 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"npy_discard", npy_discard, METH_O, NULL}, + {"npy_ensurenocopy", + npy_ensurenocopy, + METH_O, NULL}, {"get_buffer_info", get_buffer_info, METH_VARARGS, NULL}, diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index b335b64a0711..847bdafc3c6f 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -1163,7 +1163,6 @@ PyArray_DiscoverDTypeAndShape_Recursive( * It might be nice to deprecate this? But it allows things such as * `arr1d[...] = np.array([[1,2,3,4]])` */ -// Here we check whether a copy is being made or not. Check this function. NPY_NO_EXPORT int PyArray_DiscoverDTypeAndShape( PyObject *obj, int max_dims, diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index d8b5ea6b42d8..4df46cffa302 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -18,8 +18,6 @@ #include "alloc.h" #include "npy_buffer.h" -#include "npy_argparse.h" - static int PyArray_PyIntAsInt_ErrMsg(PyObject *o, const char * msg) NPY_GCC_NONNULL(2); static npy_intp @@ -182,7 +180,9 @@ PyArray_CopyConverter(PyObject *obj, _PyArray_CopyMode *copymode) { if (mode_value == NULL) { return NPY_FAIL; } - if (!PyArray_PythonPyIntFromInt(mode_value, &int_copymode)) { + + int_copymode = PyLong_AsLong(mode_value); + if (int_copymode < 0 || PyErr_Occurred()) { return NPY_FAIL; } } diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index fdc393b976de..a8f41f58264c 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1802,12 +1802,13 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, * NPY_ARRAY_ALIGNED, * NPY_ARRAY_WRITEABLE, * NPY_ARRAY_NOTSWAPPED, - * _NPY_ARRAY_ENSURECOPY, + * NPY_ARRAY_ENSURECOPY, * NPY_ARRAY_UPDATEIFCOPY, * NPY_ARRAY_WRITEBACKIFCOPY, * NPY_ARRAY_FORCECAST, * NPY_ARRAY_ENSUREARRAY, - * NPY_ARRAY_ELEMENTSTRIDES + * NPY_ARRAY_ELEMENTSTRIDES, + * _NPY_ARRAY_ENSURENOCOPY * * or'd (|) together * diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 3919f31d8940..50133798c7c7 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7836,10 +7836,10 @@ def test_scalars(self): copy=np._CopyMode.NEVER) assert_raises(ValueError, np.array, pyscalar, copy=np._CopyMode.NEVER) - assert_raises(ValueError, np.array, pyscalar, - copy=None) assert_raises(ValueError, np.array, pyscalar, copy=self.RaiseOnBool()) + assert_raises(ValueError, _multiarray_tests.npy_ensurenocopy, + [1]) def test_compatible_cast(self): @@ -8004,6 +8004,15 @@ def test_order_mismatch(self, arr, order1, order2): view, copy=None, order=order2) + def test_striding_not_ok(self): + arr = np.array([[1, 2, 4], [3, 4, 5]]) + assert_raises(ValueError, np.array, + arr.T, copy=np._CopyMode.NEVER, + order='C') + assert_raises(ValueError, np.array, + arr, copy=np._CopyMode.NEVER, + order='F') + class TestArrayAttributeDeletion: From 5cb2d64eb00b8fd8531c0c722dbb6105cbe8e277 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 9 Nov 2021 17:22:54 +0530 Subject: [PATCH 35/54] Addressed reviews and increased code coverage --- doc/source/reference/c-api/array.rst | 5 +++++ numpy/core/include/numpy/ndarraytypes.h | 2 +- numpy/core/src/multiarray/_multiarray_tests.c.src | 4 ++-- numpy/core/src/multiarray/ctors.c | 8 ++++---- numpy/core/src/multiarray/multiarraymodule.c | 2 +- numpy/core/tests/test_multiarray.py | 6 ++++++ 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 60037af8a0a1..bb440582548c 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -456,6 +456,11 @@ From other objects Make sure the returned array can be written to. + .. c:macro:: NPY_ARRAY_ENSURECOPY + + Make sure a copy is made of *op*. If this flag is not + present, data is not copied if it can be avoided. + .. c:macro:: NPY_ARRAY_ENSUREARRAY Make sure the result is a base-class ndarray. By diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 566eae357ffc..cc3b7c00689f 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -938,7 +938,7 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); #define NPY_ARRAY_UPDATEIFCOPY 0x1000 /* Deprecated in 1.14 */ #define NPY_ARRAY_WRITEBACKIFCOPY 0x2000 -#define _NPY_ARRAY_ENSURENOCOPY 0x4000 +#define NPY_ARRAY_ENSURENOCOPY 0x4000 /* * NOTE: there are also internal flags defined in multiarray/arrayobject.h, diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index f21a901c1d9a..be3eab6d6967 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -2363,11 +2363,11 @@ run_intp_converter(PyObject* NPY_UNUSED(self), PyObject *args) return tup; } -/* used to test _NPY_ARRAY_ENSURENOCOPY raises ValueError */ +/* used to test NPY_ARRAY_ENSURENOCOPY raises ValueError */ static PyObject* npy_ensurenocopy(PyObject* NPY_UNUSED(self), PyObject* args) { - int flags = _NPY_ARRAY_ENSURENOCOPY; + int flags = NPY_ARRAY_ENSURENOCOPY; if (!PyArray_CheckFromAny(args, NULL, 0, 0, flags, NULL)) { return NULL; } diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index a8f41f58264c..470b3b2f8f54 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1738,7 +1738,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, /* Create a new array and copy the data */ Py_INCREF(dtype); /* hold on in case of a subarray that is replaced */ - if( flags & _NPY_ARRAY_ENSURENOCOPY ) { + if( flags & NPY_ARRAY_ENSURENOCOPY ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating " "an array from descriptor."); @@ -1808,7 +1808,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, * NPY_ARRAY_FORCECAST, * NPY_ARRAY_ENSUREARRAY, * NPY_ARRAY_ELEMENTSTRIDES, - * _NPY_ARRAY_ENSURENOCOPY + * NPY_ARRAY_ENSURENOCOPY * * or'd (|) together * @@ -1873,7 +1873,7 @@ PyArray_CheckFromAny(PyObject *op, PyArray_Descr *descr, int min_depth, if ((requires & NPY_ARRAY_ELEMENTSTRIDES) && !PyArray_ElementStrides(obj)) { PyObject *ret; - if( requires & _NPY_ARRAY_ENSURENOCOPY ) { + if( requires & NPY_ARRAY_ENSURENOCOPY ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating a new array."); return NULL; @@ -1953,7 +1953,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) if (copy) { - if( flags & _NPY_ARRAY_ENSURENOCOPY ) { + if( flags & NPY_ARRAY_ENSURENOCOPY ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating " "an array from given array."); diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index bf60314ad494..15c2e3c10131 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1623,7 +1623,7 @@ _array_fromobject_generic( if (copy == NPY_COPY_ALWAYS) { flags = NPY_ARRAY_ENSURECOPY; } else if( copy == NPY_COPY_NEVER ) { - flags = _NPY_ARRAY_ENSURENOCOPY; + flags = NPY_ARRAY_ENSURENOCOPY; } if (order == NPY_CORDER) { flags |= NPY_ARRAY_C_CONTIGUOUS; diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 50133798c7c7..4ef58d90415f 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -8009,9 +8009,15 @@ def test_striding_not_ok(self): assert_raises(ValueError, np.array, arr.T, copy=np._CopyMode.NEVER, order='C') + assert_raises(ValueError, np.array, + arr.T, copy=np._CopyMode.NEVER, + order='C', dtype=np.int64) assert_raises(ValueError, np.array, arr, copy=np._CopyMode.NEVER, order='F') + assert_raises(ValueError, np.array, + arr, copy=np._CopyMode.NEVER, + order='F', dtype=np.int64) class TestArrayAttributeDeletion: From 8b939c9e9a4264a4d3b10fcecbfd5ad21ef9901d Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 9 Nov 2021 17:27:12 +0530 Subject: [PATCH 36/54] Intentional RuntimeError --- numpy/core/src/multiarray/ctors.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 470b3b2f8f54..1ff26a311c46 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1872,15 +1872,17 @@ PyArray_CheckFromAny(PyObject *op, PyArray_Descr *descr, int min_depth, if ((requires & NPY_ARRAY_ELEMENTSTRIDES) && !PyArray_ElementStrides(obj)) { - PyObject *ret; - if( requires & NPY_ARRAY_ENSURENOCOPY ) { - PyErr_SetString(PyExc_ValueError, - "Unable to avoid copy while creating a new array."); - return NULL; - } - ret = PyArray_NewCopy((PyArrayObject *)obj, NPY_ANYORDER); - Py_DECREF(obj); - obj = ret; + // Testing if this code can be ever hit by existing tests + PyErr_SetString(PyExc_RuntimeError, "Not Implemented"); + // PyObject *ret; + // if( requires & NPY_ARRAY_ENSURENOCOPY ) { + // PyErr_SetString(PyExc_ValueError, + // "Unable to avoid copy while creating a new array."); + // return NULL; + // } + // ret = PyArray_NewCopy((PyArrayObject *)obj, NPY_ANYORDER); + // Py_DECREF(obj); + // obj = ret; } return obj; } From 7658ad93ba845fd1e7cd6ef81f266b5d18fd565d Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 11:48:06 +0530 Subject: [PATCH 37/54] Removed dead code --- numpy/core/src/multiarray/ctors.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 1ff26a311c46..2170a9a99354 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1872,17 +1872,7 @@ PyArray_CheckFromAny(PyObject *op, PyArray_Descr *descr, int min_depth, if ((requires & NPY_ARRAY_ELEMENTSTRIDES) && !PyArray_ElementStrides(obj)) { - // Testing if this code can be ever hit by existing tests - PyErr_SetString(PyExc_RuntimeError, "Not Implemented"); - // PyObject *ret; - // if( requires & NPY_ARRAY_ENSURENOCOPY ) { - // PyErr_SetString(PyExc_ValueError, - // "Unable to avoid copy while creating a new array."); - // return NULL; - // } - // ret = PyArray_NewCopy((PyArrayObject *)obj, NPY_ANYORDER); - // Py_DECREF(obj); - // obj = ret; + PyErr_SetString(PyExc_RuntimeError, "Not Implemented."); } return obj; } From 2cf561ba0284dd4dd70d0396262974495cb4edfa Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 12:32:03 +0530 Subject: [PATCH 38/54] Prohibited calling ``__array__`` method in never copy mode --- numpy/core/src/multiarray/array_coercion.c | 13 +++++++------ numpy/core/src/multiarray/array_coercion.h | 2 +- numpy/core/src/multiarray/arrayobject.c | 2 +- numpy/core/src/multiarray/common.c | 2 +- numpy/core/src/multiarray/ctors.c | 14 +++++++++++--- numpy/core/src/multiarray/ctors.h | 3 ++- numpy/core/tests/test_multiarray.py | 3 ++- 7 files changed, 25 insertions(+), 14 deletions(-) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 847bdafc3c6f..60525026caf2 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -865,7 +865,8 @@ PyArray_DiscoverDTypeAndShape_Recursive( PyObject *obj, int curr_dims, int max_dims, PyArray_Descr**out_descr, npy_intp out_shape[NPY_MAXDIMS], coercion_cache_obj ***coercion_cache_tail_ptr, - PyArray_DTypeMeta *fixed_DType, enum _dtype_discovery_flags *flags) + PyArray_DTypeMeta *fixed_DType, enum _dtype_discovery_flags *flags, + int do_copy) { PyArrayObject *arr = NULL; PyObject *seq; @@ -923,7 +924,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( requested_descr = *out_descr; } arr = (PyArrayObject *)_array_from_array_like(obj, - requested_descr, 0, NULL); + requested_descr, 0, NULL, do_copy); if (arr == NULL) { return -1; } @@ -1117,7 +1118,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( max_dims = PyArray_DiscoverDTypeAndShape_Recursive( objects[i], curr_dims + 1, max_dims, out_descr, out_shape, coercion_cache_tail_ptr, fixed_DType, - flags); + flags, do_copy); if (max_dims < 0) { return -1; @@ -1169,7 +1170,7 @@ PyArray_DiscoverDTypeAndShape( npy_intp out_shape[NPY_MAXDIMS], coercion_cache_obj **coercion_cache, PyArray_DTypeMeta *fixed_DType, PyArray_Descr *requested_descr, - PyArray_Descr **out_descr) + PyArray_Descr **out_descr, int do_copy) { coercion_cache_obj **coercion_cache_head = coercion_cache; *coercion_cache = NULL; @@ -1214,7 +1215,7 @@ PyArray_DiscoverDTypeAndShape( int ndim = PyArray_DiscoverDTypeAndShape_Recursive( obj, 0, max_dims, out_descr, out_shape, &coercion_cache, - fixed_DType, &flags); + fixed_DType, &flags, do_copy); if (ndim < 0) { goto fail; } @@ -1499,7 +1500,7 @@ _discover_array_parameters(PyObject *NPY_UNUSED(self), int ndim = PyArray_DiscoverDTypeAndShape( obj, NPY_MAXDIMS, shape, &coercion_cache, - fixed_DType, fixed_descriptor, (PyArray_Descr **)&out_dtype); + fixed_DType, fixed_descriptor, (PyArray_Descr **)&out_dtype, 0); Py_XDECREF(fixed_DType); Py_XDECREF(fixed_descriptor); if (ndim < 0) { diff --git a/numpy/core/src/multiarray/array_coercion.h b/numpy/core/src/multiarray/array_coercion.h index db0e479fe35b..fe59b731c8d8 100644 --- a/numpy/core/src/multiarray/array_coercion.h +++ b/numpy/core/src/multiarray/array_coercion.h @@ -31,7 +31,7 @@ PyArray_DiscoverDTypeAndShape( npy_intp out_shape[NPY_MAXDIMS], coercion_cache_obj **coercion_cache, PyArray_DTypeMeta *fixed_DType, PyArray_Descr *requested_descr, - PyArray_Descr **out_descr); + PyArray_Descr **out_descr, int do_copy); NPY_NO_EXPORT int PyArray_ExtractDTypeAndDescriptor(PyObject *dtype, diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index c8aaced4e2af..1b197d0f246b 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -263,7 +263,7 @@ PyArray_CopyObject(PyArrayObject *dest, PyObject *src_object) */ ndim = PyArray_DiscoverDTypeAndShape(src_object, PyArray_NDIM(dest), dims, &cache, - NPY_DTYPE(PyArray_DESCR(dest)), PyArray_DESCR(dest), &dtype); + NPY_DTYPE(PyArray_DESCR(dest)), PyArray_DESCR(dest), &dtype, 0); if (ndim < 0) { return -1; } diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 82d34193d28b..aa95d285a8ca 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -119,7 +119,7 @@ PyArray_DTypeFromObject(PyObject *obj, int maxdims, PyArray_Descr **out_dtype) int ndim; ndim = PyArray_DiscoverDTypeAndShape( - obj, maxdims, shape, &cache, NULL, NULL, out_dtype); + obj, maxdims, shape, &cache, NULL, NULL, out_dtype, 0); if (ndim < 0) { return -1; } diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 2170a9a99354..23d868c2cfe1 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1291,7 +1291,8 @@ _array_from_buffer_3118(PyObject *memoryview) */ NPY_NO_EXPORT PyObject * _array_from_array_like(PyObject *op, - PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context) { + PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context, + int do_copy) { PyObject* tmp; /* @@ -1347,6 +1348,12 @@ _array_from_array_like(PyObject *op, * this should be changed! */ if (!writeable && tmp == Py_NotImplemented) { + PyObject* array_meth = PyArray_LookupSpecial_OnInstance(op, "__array__"); + PyObject* has_get = array_meth && PyType_Check(op) && PyObject_HasAttrString(array_meth, "__get__"); + if (array_meth != NULL && !has_get && do_copy) { + PyErr_SetString(PyExc_ValueError, "Calling __array__ in never copy mode is not allowed."); + return NULL; + } tmp = PyArray_FromArrayAttr(op, requested_dtype, context); if (tmp == NULL) { return NULL; @@ -1447,7 +1454,7 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, } /* Try __array__ before using s as a sequence */ - PyObject *tmp = _array_from_array_like(s, NULL, 0, NULL); + PyObject *tmp = _array_from_array_like(s, NULL, 0, NULL, 0); if (tmp == NULL) { goto fail; } @@ -1575,7 +1582,8 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, Py_XDECREF(newtype); ndim = PyArray_DiscoverDTypeAndShape(op, - NPY_MAXDIMS, dims, &cache, fixed_DType, fixed_descriptor, &dtype); + NPY_MAXDIMS, dims, &cache, fixed_DType, fixed_descriptor, &dtype, + flags & NPY_ARRAY_ENSURENOCOPY); Py_XDECREF(fixed_descriptor); Py_XDECREF(fixed_DType); diff --git a/numpy/core/src/multiarray/ctors.h b/numpy/core/src/multiarray/ctors.h index e59e86e8b9aa..4f8bd5a82887 100644 --- a/numpy/core/src/multiarray/ctors.h +++ b/numpy/core/src/multiarray/ctors.h @@ -32,7 +32,8 @@ PyArray_New( NPY_NO_EXPORT PyObject * _array_from_array_like(PyObject *op, - PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context); + PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context, + int do_copy); NPY_NO_EXPORT PyObject * PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 4ef58d90415f..18302543adde 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7943,7 +7943,8 @@ def __array__(self): assert_array_equal(res, base_arr) assert res is base_arr # numpy trusts the ArrayLike - assert np.array(arr, copy=np._CopyMode.NEVER) is base_arr + with pytest.raises(ValueError): + np.array(arr, copy=np._CopyMode.NEVER) @pytest.mark.parametrize( "arr", [np.ones(()), np.arange(81).reshape((9, 9))]) From 37cd05eb4fa732230345785de53722a80c27fc62 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 13:15:59 +0530 Subject: [PATCH 39/54] Fixed warning and updated docs --- numpy/core/src/multiarray/array_coercion.c | 1 + numpy/core/src/multiarray/ctors.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 60525026caf2..93957242b662 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -857,6 +857,7 @@ PyArray_AdaptDescriptorToArray(PyArrayObject *arr, PyObject *dtype) * (Initially it is a pointer to the user-provided head pointer). * @param fixed_DType User provided fixed DType class * @param flags Discovery flags (reporting and behaviour flags, see def.) + * @param do_copy Specifies if a copy is to be made during array creation. * @return The updated number of maximum dimensions (i.e. scalars will set * this to the current dimensions). */ diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 23d868c2cfe1..0fbf0f91feb0 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1349,7 +1349,7 @@ _array_from_array_like(PyObject *op, */ if (!writeable && tmp == Py_NotImplemented) { PyObject* array_meth = PyArray_LookupSpecial_OnInstance(op, "__array__"); - PyObject* has_get = array_meth && PyType_Check(op) && PyObject_HasAttrString(array_meth, "__get__"); + int has_get = array_meth && PyType_Check(op) && PyObject_HasAttrString(array_meth, "__get__"); if (array_meth != NULL && !has_get && do_copy) { PyErr_SetString(PyExc_ValueError, "Calling __array__ in never copy mode is not allowed."); return NULL; From 9f9a34843016faea897cd591a5c4c104fa68c883 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 14:27:35 +0530 Subject: [PATCH 40/54] Apply suggestions from code review Co-authored-by: Matti Picus --- numpy/_globals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/_globals.py b/numpy/_globals.py index b17ca19798be..41adaae25a58 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -97,7 +97,7 @@ def __repr__(self): class _CopyMode(enum.Enum): """ An enumeration for the copy modes supported - by numpy. The following three modes are supported, + by numpy.copy() and numpy.array(). The following three modes are supported, - ALWAYS: This means that a deep copy of the input array will always be taken. From 946ab2461870047b1c210b93152234d185684927 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 14:28:06 +0530 Subject: [PATCH 41/54] Updated docs --- numpy/core/src/multiarray/array_coercion.c | 1 + numpy/core/src/multiarray/ctors.c | 1 + 2 files changed, 2 insertions(+) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 93957242b662..aa5f1f2fd6fc 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -1159,6 +1159,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( * The result may be unchanged (remain NULL) when converting a * sequence with no elements. In this case it is callers responsibility * to choose a default. + * @param do_copy Specifies if a copy is to be made during array creation. * @return dimensions of the discovered object or -1 on error. * WARNING: If (and only if) the output is a single array, the ndim * returned _can_ exceed the maximum allowed number of dimensions. diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 0fbf0f91feb0..ca3a29d53663 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1284,6 +1284,7 @@ _array_from_buffer_3118(PyObject *memoryview) * DType may be used, but is not enforced. * @param writeable whether the result must be writeable. * @param context Unused parameter, must be NULL (should be removed later). + * @param do_copy Specifies if a copy is to be made during array creation. * * @returns The array object, Py_NotImplemented if op is not array-like, * or NULL with an error set. (A new reference to Py_NotImplemented From 5ede7eb15058e10b218756a6f5c5066b97af2839 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 14:35:40 +0530 Subject: [PATCH 42/54] Added release notes entry --- doc/release/upcoming_changes/19173.change.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/release/upcoming_changes/19173.change.rst diff --git a/doc/release/upcoming_changes/19173.change.rst b/doc/release/upcoming_changes/19173.change.rst new file mode 100644 index 000000000000..ff5047da6768 --- /dev/null +++ b/doc/release/upcoming_changes/19173.change.rst @@ -0,0 +1,6 @@ +Calling `__array__` raises `ValueError` in never copy mode +---------------------------------------------------------- + +Since `__array__` lacks a copy keyword argument, currently +never copy (speficied by `np._CopyMode.NEVER`) will raise +a `ValueError` if `__array__` is called. \ No newline at end of file From d1cb6624f5b52b1de5aa89dd18062dbca79ab00d Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 15:40:55 +0530 Subject: [PATCH 43/54] L[0]->L[False], L[1]->L[True] --- numpy/__init__.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 4e6969e322ae..d510acaa5d1f 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -3243,8 +3243,8 @@ trunc: _UFunc_Nin1_Nout1[L['trunc'], L[7], None] abs = absolute class _CopyMode(enum.Enum): - ALWAYS: L[1] - IF_NEEDED: L[0] + ALWAYS: L[True] + IF_NEEDED: L[False] NEVER: L[2] # Warnings From 6c01915fd37652e6960b2e563a93df79c92ae18e Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 21:44:35 +0530 Subject: [PATCH 44/54] Deleted release notes entry --- doc/release/upcoming_changes/19173.change.rst | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 doc/release/upcoming_changes/19173.change.rst diff --git a/doc/release/upcoming_changes/19173.change.rst b/doc/release/upcoming_changes/19173.change.rst deleted file mode 100644 index ff5047da6768..000000000000 --- a/doc/release/upcoming_changes/19173.change.rst +++ /dev/null @@ -1,6 +0,0 @@ -Calling `__array__` raises `ValueError` in never copy mode ----------------------------------------------------------- - -Since `__array__` lacks a copy keyword argument, currently -never copy (speficied by `np._CopyMode.NEVER`) will raise -a `ValueError` if `__array__` is called. \ No newline at end of file From 5f1965fd36d9d28c0f8743384485e346564a919b Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 21:49:45 +0530 Subject: [PATCH 45/54] do_copy -> allow_copy --- numpy/core/src/multiarray/array_coercion.c | 14 +++++++------- numpy/core/src/multiarray/array_coercion.h | 2 +- numpy/core/src/multiarray/ctors.c | 6 +++--- numpy/core/src/multiarray/ctors.h | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index aa5f1f2fd6fc..8778ec20c77b 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -857,7 +857,7 @@ PyArray_AdaptDescriptorToArray(PyArrayObject *arr, PyObject *dtype) * (Initially it is a pointer to the user-provided head pointer). * @param fixed_DType User provided fixed DType class * @param flags Discovery flags (reporting and behaviour flags, see def.) - * @param do_copy Specifies if a copy is to be made during array creation. + * @param allow_copy Specifies if a copy is allowed during array creation. * @return The updated number of maximum dimensions (i.e. scalars will set * this to the current dimensions). */ @@ -867,7 +867,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( npy_intp out_shape[NPY_MAXDIMS], coercion_cache_obj ***coercion_cache_tail_ptr, PyArray_DTypeMeta *fixed_DType, enum _dtype_discovery_flags *flags, - int do_copy) + int allow_copy) { PyArrayObject *arr = NULL; PyObject *seq; @@ -925,7 +925,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( requested_descr = *out_descr; } arr = (PyArrayObject *)_array_from_array_like(obj, - requested_descr, 0, NULL, do_copy); + requested_descr, 0, NULL, allow_copy); if (arr == NULL) { return -1; } @@ -1119,7 +1119,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( max_dims = PyArray_DiscoverDTypeAndShape_Recursive( objects[i], curr_dims + 1, max_dims, out_descr, out_shape, coercion_cache_tail_ptr, fixed_DType, - flags, do_copy); + flags, allow_copy); if (max_dims < 0) { return -1; @@ -1159,7 +1159,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( * The result may be unchanged (remain NULL) when converting a * sequence with no elements. In this case it is callers responsibility * to choose a default. - * @param do_copy Specifies if a copy is to be made during array creation. + * @param allow_copy Specifies if a copy is allowed during array creation. * @return dimensions of the discovered object or -1 on error. * WARNING: If (and only if) the output is a single array, the ndim * returned _can_ exceed the maximum allowed number of dimensions. @@ -1172,7 +1172,7 @@ PyArray_DiscoverDTypeAndShape( npy_intp out_shape[NPY_MAXDIMS], coercion_cache_obj **coercion_cache, PyArray_DTypeMeta *fixed_DType, PyArray_Descr *requested_descr, - PyArray_Descr **out_descr, int do_copy) + PyArray_Descr **out_descr, int allow_copy) { coercion_cache_obj **coercion_cache_head = coercion_cache; *coercion_cache = NULL; @@ -1217,7 +1217,7 @@ PyArray_DiscoverDTypeAndShape( int ndim = PyArray_DiscoverDTypeAndShape_Recursive( obj, 0, max_dims, out_descr, out_shape, &coercion_cache, - fixed_DType, &flags, do_copy); + fixed_DType, &flags, allow_copy); if (ndim < 0) { goto fail; } diff --git a/numpy/core/src/multiarray/array_coercion.h b/numpy/core/src/multiarray/array_coercion.h index fe59b731c8d8..4790d80301b0 100644 --- a/numpy/core/src/multiarray/array_coercion.h +++ b/numpy/core/src/multiarray/array_coercion.h @@ -31,7 +31,7 @@ PyArray_DiscoverDTypeAndShape( npy_intp out_shape[NPY_MAXDIMS], coercion_cache_obj **coercion_cache, PyArray_DTypeMeta *fixed_DType, PyArray_Descr *requested_descr, - PyArray_Descr **out_descr, int do_copy); + PyArray_Descr **out_descr, int allow_copy); NPY_NO_EXPORT int PyArray_ExtractDTypeAndDescriptor(PyObject *dtype, diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index ca3a29d53663..d9289dc96c31 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1284,7 +1284,7 @@ _array_from_buffer_3118(PyObject *memoryview) * DType may be used, but is not enforced. * @param writeable whether the result must be writeable. * @param context Unused parameter, must be NULL (should be removed later). - * @param do_copy Specifies if a copy is to be made during array creation. + * @param allow_copy Specifies if a copy is allowed during array creation. * * @returns The array object, Py_NotImplemented if op is not array-like, * or NULL with an error set. (A new reference to Py_NotImplemented @@ -1293,7 +1293,7 @@ _array_from_buffer_3118(PyObject *memoryview) NPY_NO_EXPORT PyObject * _array_from_array_like(PyObject *op, PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context, - int do_copy) { + int allow_copy) { PyObject* tmp; /* @@ -1351,7 +1351,7 @@ _array_from_array_like(PyObject *op, if (!writeable && tmp == Py_NotImplemented) { PyObject* array_meth = PyArray_LookupSpecial_OnInstance(op, "__array__"); int has_get = array_meth && PyType_Check(op) && PyObject_HasAttrString(array_meth, "__get__"); - if (array_meth != NULL && !has_get && do_copy) { + if (array_meth != NULL && !has_get && allow_copy) { PyErr_SetString(PyExc_ValueError, "Calling __array__ in never copy mode is not allowed."); return NULL; } diff --git a/numpy/core/src/multiarray/ctors.h b/numpy/core/src/multiarray/ctors.h index 4f8bd5a82887..cf01a6256034 100644 --- a/numpy/core/src/multiarray/ctors.h +++ b/numpy/core/src/multiarray/ctors.h @@ -33,7 +33,7 @@ PyArray_New( NPY_NO_EXPORT PyObject * _array_from_array_like(PyObject *op, PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context, - int do_copy); + int allow_copy); NPY_NO_EXPORT PyObject * PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, From 05ff102c00558de31f2f5ef8145241490156267b Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 21:52:32 +0530 Subject: [PATCH 46/54] Apply suggestions from code review Co-authored-by: Sebastian Berg --- numpy/core/src/multiarray/conversion_utils.c | 2 +- numpy/core/src/multiarray/ctors.c | 8 +++----- numpy/core/src/multiarray/multiarraymodule.c | 3 ++- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 4df46cffa302..ee3e7b50c4b5 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -182,7 +182,7 @@ PyArray_CopyConverter(PyObject *obj, _PyArray_CopyMode *copymode) { } int_copymode = PyLong_AsLong(mode_value); - if (int_copymode < 0 || PyErr_Occurred()) { + if (error_converting(int_copymode)) { return NPY_FAIL; } } diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index ca3a29d53663..fcb19096b546 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1747,7 +1747,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, /* Create a new array and copy the data */ Py_INCREF(dtype); /* hold on in case of a subarray that is replaced */ - if( flags & NPY_ARRAY_ENSURENOCOPY ) { + if (flags & NPY_ARRAY_ENSURENOCOPY ) { PyErr_SetString(PyExc_ValueError, "Unable to avoid copy while creating " "an array from descriptor."); @@ -1953,11 +1953,9 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) !PyArray_EquivTypes(oldtype, newtype); if (copy) { - - if( flags & NPY_ARRAY_ENSURENOCOPY ) { + if (flags & NPY_ARRAY_ENSURENOCOPY ) { PyErr_SetString(PyExc_ValueError, - "Unable to avoid copy while creating " - "an array from given array."); + "Unable to avoid copy while creating an array from given array."); return NULL; } diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 15c2e3c10131..5f72674fec74 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1622,7 +1622,8 @@ _array_fromobject_generic( if (copy == NPY_COPY_ALWAYS) { flags = NPY_ARRAY_ENSURECOPY; - } else if( copy == NPY_COPY_NEVER ) { + } + else if (copy == NPY_COPY_NEVER ) { flags = NPY_ARRAY_ENSURENOCOPY; } if (order == NPY_CORDER) { From 2db65c9db122d780462c6fdcd8d54b85c6631365 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 22:02:46 +0530 Subject: [PATCH 47/54] Addressed reviews --- numpy/core/src/multiarray/methods.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 7530a2e36cce..dddfb35f6405 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -859,16 +859,16 @@ array_astype(PyArrayObject *self, * can skip the copy. */ if (forcecopy != NPY_COPY_ALWAYS && - (order == NPY_KEEPORDER || - (order == NPY_ANYORDER && - (PyArray_IS_C_CONTIGUOUS(self) || - PyArray_IS_F_CONTIGUOUS(self))) || - (order == NPY_CORDER && - PyArray_IS_C_CONTIGUOUS(self)) || - (order == NPY_FORTRANORDER && - PyArray_IS_F_CONTIGUOUS(self))) && - (subok || PyArray_CheckExact(self)) && - PyArray_EquivTypes(dtype, PyArray_DESCR(self))) { + (order == NPY_KEEPORDER || + (order == NPY_ANYORDER && + (PyArray_IS_C_CONTIGUOUS(self) || + PyArray_IS_F_CONTIGUOUS(self))) || + (order == NPY_CORDER && + PyArray_IS_C_CONTIGUOUS(self)) || + (order == NPY_FORTRANORDER && + PyArray_IS_F_CONTIGUOUS(self))) && + (subok || PyArray_CheckExact(self)) && + PyArray_EquivTypes(dtype, PyArray_DESCR(self))) { Py_DECREF(dtype); Py_INCREF(self); return (PyObject *)self; From d0d75f39f28ac26d4cc1aa3a4cbea63a6a027929 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Wed, 10 Nov 2021 22:27:45 +0530 Subject: [PATCH 48/54] Reverted dead code removal --- numpy/core/src/multiarray/ctors.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index dd92f715c8fd..286e45e39a28 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1881,7 +1881,16 @@ PyArray_CheckFromAny(PyObject *op, PyArray_Descr *descr, int min_depth, if ((requires & NPY_ARRAY_ELEMENTSTRIDES) && !PyArray_ElementStrides(obj)) { - PyErr_SetString(PyExc_RuntimeError, "Not Implemented."); + PyObject *ret; + if (requires & NPY_ARRAY_ENSURENOCOPY) { + PyErr_SetString(PyExc_ValueError, + "Unable to avoid copy " + "while creating a new array."); + return NULL; + } + ret = PyArray_NewCopy((PyArrayObject *)obj, NPY_ANYORDER); + Py_DECREF(obj); + obj = ret; } return obj; } From 4b2cd27a5eceb288685020c8efa925ee3c72aed1 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 12 Nov 2021 15:20:49 -0600 Subject: [PATCH 49/54] STY: Small style fixups for never-copy changes Yes, these may be slightly biased towards my own opinions. --- numpy/core/include/numpy/ndarraytypes.h | 6 +++--- numpy/core/src/multiarray/conversion_utils.c | 12 ++++++------ numpy/core/src/multiarray/ctors.c | 7 +++---- numpy/core/src/multiarray/methods.c | 5 ++--- numpy/core/src/multiarray/multiarraymodule.c | 14 ++++++-------- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 6f6a00b8f81e..9a610908fb8a 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -450,9 +450,9 @@ typedef struct { } PyArray_Dims; typedef enum { - NPY_COPY_IF_NEEDED, - NPY_COPY_ALWAYS, - NPY_COPY_NEVER + NPY_COPY_IF_NEEDED = 0, + NPY_COPY_ALWAYS = 1, + NPY_COPY_NEVER = 2, } _PyArray_CopyMode; typedef struct { diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index ee3e7b50c4b5..ef101a78bc89 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -171,30 +171,30 @@ PyArray_CopyConverter(PyObject *obj, _PyArray_CopyMode *copymode) { return NPY_FAIL; } - int int_copymode = -1; + int int_copymode; PyObject* numpy_CopyMode = NULL; npy_cache_import("numpy", "_CopyMode", &numpy_CopyMode); - if (numpy_CopyMode != NULL && PyObject_Type(obj) == numpy_CopyMode) { + if (numpy_CopyMode != NULL && (PyObject *)Py_TYPE(obj) == numpy_CopyMode) { PyObject* mode_value = PyObject_GetAttrString(obj, "value"); if (mode_value == NULL) { return NPY_FAIL; } - int_copymode = PyLong_AsLong(mode_value); + int_copymode = (int)PyLong_AsLong(mode_value); if (error_converting(int_copymode)) { return NPY_FAIL; } } else { npy_bool bool_copymode; - if( !PyArray_BoolConverter(obj, &bool_copymode) ) { + if (!PyArray_BoolConverter(obj, &bool_copymode)) { return NPY_FAIL; } - int_copymode = (int) bool_copymode; + int_copymode = (int)bool_copymode; } - *copymode = (_PyArray_CopyMode) int_copymode; + *copymode = (_PyArray_CopyMode)int_copymode; return NPY_SUCCEED; } diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 286e45e39a28..7c3ac61c08bb 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1879,13 +1879,12 @@ PyArray_CheckFromAny(PyObject *op, PyArray_Descr *descr, int min_depth, return NULL; } - if ((requires & NPY_ARRAY_ELEMENTSTRIDES) && - !PyArray_ElementStrides(obj)) { + if ((requires & NPY_ARRAY_ELEMENTSTRIDES) + && !PyArray_ElementStrides(obj)) { PyObject *ret; if (requires & NPY_ARRAY_ENSURENOCOPY) { PyErr_SetString(PyExc_ValueError, - "Unable to avoid copy " - "while creating a new array."); + "Unable to avoid copy while creating a new array."); return NULL; } ret = PyArray_NewCopy((PyArrayObject *)obj, NPY_ANYORDER); diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 0a471da921f3..627096b3cfb2 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -875,10 +875,9 @@ array_astype(PyArrayObject *self, return (PyObject *)self; } - if( forcecopy == NPY_COPY_NEVER ) { + if (forcecopy == NPY_COPY_NEVER) { PyErr_SetString(PyExc_ValueError, - "Unable to avoid copy while casting in " - "never copy mode."); + "Unable to avoid copy while casting in never copy mode."); Py_DECREF(dtype); return NULL; } diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index d9dce25172d7..d28c033f8235 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1579,16 +1579,15 @@ _array_fromobject_generic( if (PyArray_CheckExact(op) || (subok && PyArray_Check(op))) { oparr = (PyArrayObject *)op; if (type == NULL) { - if ((copy == NPY_COPY_IF_NEEDED || copy == NPY_COPY_NEVER) && - STRIDING_OK(oparr, order)) { + if (copy != NPY_COPY_ALWAYS && STRIDING_OK(oparr, order)) { ret = oparr; Py_INCREF(ret); goto finish; } else { - if( copy == NPY_COPY_NEVER ) { + if (copy == NPY_COPY_NEVER) { PyErr_SetString(PyExc_ValueError, - "Unable to avoid copy while creating a new array."); + "Unable to avoid copy while creating a new array."); return NULL; } ret = (PyArrayObject *)PyArray_NewCopy(oparr, order); @@ -1598,16 +1597,15 @@ _array_fromobject_generic( /* One more chance */ oldtype = PyArray_DESCR(oparr); if (PyArray_EquivTypes(oldtype, type)) { - if ((copy == NPY_COPY_IF_NEEDED || copy == NPY_COPY_NEVER) && - STRIDING_OK(oparr, order)) { + if (copy != NPY_COPY_ALWAYS && STRIDING_OK(oparr, order)) { Py_INCREF(op); ret = oparr; goto finish; } else { - if( copy == NPY_COPY_NEVER ) { + if (copy == NPY_COPY_NEVER) { PyErr_SetString(PyExc_ValueError, - "Unable to avoid copy while creating a new array."); + "Unable to avoid copy while creating a new array."); return NULL; } ret = (PyArrayObject *)PyArray_NewCopy(oparr, order); From 84951a63df4dce887f31969e2a109936b368198a Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 12 Nov 2021 15:24:57 -0600 Subject: [PATCH 50/54] MAINT: Remove private CopyMode enum to private header --- numpy/core/include/numpy/ndarraytypes.h | 6 ------ numpy/core/src/multiarray/conversion_utils.h | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 9a610908fb8a..616738d565f1 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -449,12 +449,6 @@ typedef struct { int len; } PyArray_Dims; -typedef enum { - NPY_COPY_IF_NEEDED = 0, - NPY_COPY_ALWAYS = 1, - NPY_COPY_NEVER = 2, -} _PyArray_CopyMode; - typedef struct { /* * Functions to cast to most other standard types diff --git a/numpy/core/src/multiarray/conversion_utils.h b/numpy/core/src/multiarray/conversion_utils.h index 643b67d5992e..4072841ee1c7 100644 --- a/numpy/core/src/multiarray/conversion_utils.h +++ b/numpy/core/src/multiarray/conversion_utils.h @@ -9,6 +9,12 @@ PyArray_IntpConverter(PyObject *obj, PyArray_Dims *seq); NPY_NO_EXPORT int PyArray_OptionalIntpConverter(PyObject *obj, PyArray_Dims *seq); +typedef enum { + NPY_COPY_IF_NEEDED = 0, + NPY_COPY_ALWAYS = 1, + NPY_COPY_NEVER = 2, +} _PyArray_CopyMode; + NPY_NO_EXPORT int PyArray_CopyConverter(PyObject *obj, _PyArray_CopyMode *copyflag); From f31c4a66ad27b428b6a20f9fa28c1b33854ea949 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 12 Nov 2021 15:45:26 -0600 Subject: [PATCH 51/54] MAINT,BUG: Refactor __array__ and never-copy to move check later Doing the check before calling `PyArray_FromArrayAttr` means we look up the attribute twice. It further fixes two bugs: 1. The reference counting was wrong. 2. In debug mode there was a failure for some deprecation warnings (probably due to error propagation) --- numpy/core/src/multiarray/ctors.c | 61 +++++++++++++++++++++++-------- numpy/core/src/multiarray/ctors.h | 4 ++ 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 7c3ac61c08bb..3f1d7835e01b 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1349,13 +1349,7 @@ _array_from_array_like(PyObject *op, * this should be changed! */ if (!writeable && tmp == Py_NotImplemented) { - PyObject* array_meth = PyArray_LookupSpecial_OnInstance(op, "__array__"); - int has_get = array_meth && PyType_Check(op) && PyObject_HasAttrString(array_meth, "__get__"); - if (array_meth != NULL && !has_get && allow_copy) { - PyErr_SetString(PyExc_ValueError, "Calling __array__ in never copy mode is not allowed."); - return NULL; - } - tmp = PyArray_FromArrayAttr(op, requested_dtype, context); + tmp = PyArray_FromArrayAttr_int(op, requested_dtype, allow_copy); if (tmp == NULL) { return NULL; } @@ -2463,18 +2457,30 @@ PyArray_FromInterface(PyObject *origin) return NULL; } -/*NUMPY_API + +/** + * Check for an __array__ attribute and call it when it exists. + * + * .. warning: + * If returned, `NotImplemented` is borrowed and must not be Decref'd + * + * @param op The Python object to convert to an array. + * @param descr The desired `arr.dtype`, passed into the `__array__` call, + * as information but is not checked/enforced! + * @param never_copy Indicator that a copy is not allowed. + * NOTE: Currently, this means an error is raised instead of calling + * `op.__array__()`. In the future we could call for example call + * `op.__array__(never_copy=True)` instead. + * @returns NotImplemented if `__array__` is not defined or a NumPy array + * (or subclass). On error, return NULL. */ NPY_NO_EXPORT PyObject * -PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode, PyObject *context) +PyArray_FromArrayAttr_int( + PyObject *op, PyArray_Descr *descr, int never_copy) { PyObject *new; PyObject *array_meth; - if (context != NULL) { - PyErr_SetString(PyExc_RuntimeError, "'context' must be NULL"); - return NULL; - } array_meth = PyArray_LookupSpecial_OnInstance(op, "__array__"); if (array_meth == NULL) { if (PyErr_Occurred()) { @@ -2490,6 +2496,16 @@ PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode, PyObject *context) } return Py_NotImplemented; } + if (never_copy) { + /* Currently, we must always assume that `__array__` returns a copy */ + PyErr_SetString(PyExc_ValueError, + "Unable to avoid copy while converting from an object " + "implementing the `__array__` protocol. NumPy cannot ensure " + "that no copy will be made."); + Py_DECREF(array_meth); + return NULL; + } + if (PyType_Check(op) && PyObject_HasAttrString(array_meth, "__get__")) { /* * If the input is a class `array_meth` may be a property-like object. @@ -2500,11 +2516,11 @@ PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode, PyObject *context) Py_DECREF(array_meth); return Py_NotImplemented; } - if (typecode == NULL) { + if (descr == NULL) { new = PyObject_CallFunction(array_meth, NULL); } else { - new = PyObject_CallFunction(array_meth, "O", typecode); + new = PyObject_CallFunction(array_meth, "O", descr); } Py_DECREF(array_meth); if (new == NULL) { @@ -2520,6 +2536,21 @@ PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode, PyObject *context) return new; } + +/*NUMPY_API + */ +NPY_NO_EXPORT PyObject * +PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode, PyObject *context) +{ + if (context != NULL) { + PyErr_SetString(PyExc_RuntimeError, "'context' must be NULL"); + return NULL; + } + + return PyArray_FromArrayAttr_int(op, typecode, 0); +} + + /*NUMPY_API * new reference -- accepts NULL for mintype */ diff --git a/numpy/core/src/multiarray/ctors.h b/numpy/core/src/multiarray/ctors.h index cf01a6256034..2f9e8547dec5 100644 --- a/numpy/core/src/multiarray/ctors.h +++ b/numpy/core/src/multiarray/ctors.h @@ -52,6 +52,10 @@ PyArray_FromStructInterface(PyObject *input); NPY_NO_EXPORT PyObject * PyArray_FromInterface(PyObject *input); +NPY_NO_EXPORT PyObject * +PyArray_FromArrayAttr_int( + PyObject *op, PyArray_Descr *descr, int never_copy); + NPY_NO_EXPORT PyObject * PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode, PyObject *context); From dea40555bd90afb368917dab31e613e00f1f6665 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 12 Nov 2021 15:51:12 -0600 Subject: [PATCH 52/54] MAINT: Rename `allow_copy` to `never_copy` in never-copy machinery The first name was `do_copy`, which was confusing because it sounds like a copy is forced (opposite is the case!) `allow_copy` would have been better, but corrects it into the wrong direction. `never_copy` is correct, and aligns with the Python side names --- numpy/core/src/multiarray/array_coercion.c | 14 +++++++------- numpy/core/src/multiarray/array_coercion.h | 2 +- numpy/core/src/multiarray/ctors.c | 8 ++++---- numpy/core/src/multiarray/ctors.h | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index d58dd5d216ff..2598e4bde6ea 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -858,7 +858,7 @@ PyArray_AdaptDescriptorToArray(PyArrayObject *arr, PyObject *dtype) * (Initially it is a pointer to the user-provided head pointer). * @param fixed_DType User provided fixed DType class * @param flags Discovery flags (reporting and behaviour flags, see def.) - * @param allow_copy Specifies if a copy is allowed during array creation. + * @param never_copy Specifies if a copy is allowed during array creation. * @return The updated number of maximum dimensions (i.e. scalars will set * this to the current dimensions). */ @@ -868,7 +868,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( npy_intp out_shape[NPY_MAXDIMS], coercion_cache_obj ***coercion_cache_tail_ptr, PyArray_DTypeMeta *fixed_DType, enum _dtype_discovery_flags *flags, - int allow_copy) + int never_copy) { PyArrayObject *arr = NULL; PyObject *seq; @@ -926,7 +926,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( requested_descr = *out_descr; } arr = (PyArrayObject *)_array_from_array_like(obj, - requested_descr, 0, NULL, allow_copy); + requested_descr, 0, NULL, never_copy); if (arr == NULL) { return -1; } @@ -1120,7 +1120,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( max_dims = PyArray_DiscoverDTypeAndShape_Recursive( objects[i], curr_dims + 1, max_dims, out_descr, out_shape, coercion_cache_tail_ptr, fixed_DType, - flags, allow_copy); + flags, never_copy); if (max_dims < 0) { return -1; @@ -1160,7 +1160,7 @@ PyArray_DiscoverDTypeAndShape_Recursive( * The result may be unchanged (remain NULL) when converting a * sequence with no elements. In this case it is callers responsibility * to choose a default. - * @param allow_copy Specifies if a copy is allowed during array creation. + * @param never_copy Specifies that a copy is not allowed. * @return dimensions of the discovered object or -1 on error. * WARNING: If (and only if) the output is a single array, the ndim * returned _can_ exceed the maximum allowed number of dimensions. @@ -1173,7 +1173,7 @@ PyArray_DiscoverDTypeAndShape( npy_intp out_shape[NPY_MAXDIMS], coercion_cache_obj **coercion_cache, PyArray_DTypeMeta *fixed_DType, PyArray_Descr *requested_descr, - PyArray_Descr **out_descr, int allow_copy) + PyArray_Descr **out_descr, int never_copy) { coercion_cache_obj **coercion_cache_head = coercion_cache; *coercion_cache = NULL; @@ -1218,7 +1218,7 @@ PyArray_DiscoverDTypeAndShape( int ndim = PyArray_DiscoverDTypeAndShape_Recursive( obj, 0, max_dims, out_descr, out_shape, &coercion_cache, - fixed_DType, &flags, allow_copy); + fixed_DType, &flags, never_copy); if (ndim < 0) { goto fail; } diff --git a/numpy/core/src/multiarray/array_coercion.h b/numpy/core/src/multiarray/array_coercion.h index 4790d80301b0..f2482cecc005 100644 --- a/numpy/core/src/multiarray/array_coercion.h +++ b/numpy/core/src/multiarray/array_coercion.h @@ -31,7 +31,7 @@ PyArray_DiscoverDTypeAndShape( npy_intp out_shape[NPY_MAXDIMS], coercion_cache_obj **coercion_cache, PyArray_DTypeMeta *fixed_DType, PyArray_Descr *requested_descr, - PyArray_Descr **out_descr, int allow_copy); + PyArray_Descr **out_descr, int never_copy); NPY_NO_EXPORT int PyArray_ExtractDTypeAndDescriptor(PyObject *dtype, diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 3f1d7835e01b..819bb22bedd6 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1284,7 +1284,7 @@ _array_from_buffer_3118(PyObject *memoryview) * DType may be used, but is not enforced. * @param writeable whether the result must be writeable. * @param context Unused parameter, must be NULL (should be removed later). - * @param allow_copy Specifies if a copy is allowed during array creation. + * @param never_copy Specifies that a copy is not allowed. * * @returns The array object, Py_NotImplemented if op is not array-like, * or NULL with an error set. (A new reference to Py_NotImplemented @@ -1293,7 +1293,7 @@ _array_from_buffer_3118(PyObject *memoryview) NPY_NO_EXPORT PyObject * _array_from_array_like(PyObject *op, PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context, - int allow_copy) { + int never_copy) { PyObject* tmp; /* @@ -1349,7 +1349,7 @@ _array_from_array_like(PyObject *op, * this should be changed! */ if (!writeable && tmp == Py_NotImplemented) { - tmp = PyArray_FromArrayAttr_int(op, requested_dtype, allow_copy); + tmp = PyArray_FromArrayAttr_int(op, requested_dtype, never_copy); if (tmp == NULL) { return NULL; } @@ -2467,7 +2467,7 @@ PyArray_FromInterface(PyObject *origin) * @param op The Python object to convert to an array. * @param descr The desired `arr.dtype`, passed into the `__array__` call, * as information but is not checked/enforced! - * @param never_copy Indicator that a copy is not allowed. + * @param never_copy Specifies that a copy is not allowed. * NOTE: Currently, this means an error is raised instead of calling * `op.__array__()`. In the future we could call for example call * `op.__array__(never_copy=True)` instead. diff --git a/numpy/core/src/multiarray/ctors.h b/numpy/core/src/multiarray/ctors.h index 2f9e8547dec5..98160b1cc48f 100644 --- a/numpy/core/src/multiarray/ctors.h +++ b/numpy/core/src/multiarray/ctors.h @@ -33,7 +33,7 @@ PyArray_New( NPY_NO_EXPORT PyObject * _array_from_array_like(PyObject *op, PyArray_Descr *requested_dtype, npy_bool writeable, PyObject *context, - int allow_copy); + int never_copy); NPY_NO_EXPORT PyObject * PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, From 9fee0f8db5def3d119a38845111724f436fcfe2e Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 12 Nov 2021 16:02:06 -0600 Subject: [PATCH 53/54] DOC: Slightly extend to docs to note that we assume no-copy buffer protocol --- numpy/_globals.py | 3 +++ numpy/core/include/numpy/ndarraytypes.h | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/numpy/_globals.py b/numpy/_globals.py index 41adaae25a58..c888747258c7 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -106,6 +106,9 @@ class _CopyMode(enum.Enum): - NEVER: This means that the deep copy will never be taken. If a copy cannot be avoided then a `ValueError` will be raised. + + Note that the buffer-protocol could in theory do copies. NumPy currently + assumes an object exporting the buffer protocol will never do this. """ ALWAYS = True diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 616738d565f1..6240adc0c7f1 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -868,7 +868,7 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); /* * Always copy the array. Returned arrays are always CONTIGUOUS, - * ALIGNED, and WRITEABLE. + * ALIGNED, and WRITEABLE. See also: NPY_ARRAY_ENSURENOCOPY = 0x4000. * * This flag may be requested in constructor functions. */ @@ -937,6 +937,11 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); #define NPY_ARRAY_UPDATEIFCOPY 0x1000 /* Deprecated in 1.14 */ #define NPY_ARRAY_WRITEBACKIFCOPY 0x2000 +/* + * No copy may be made while converting from an object/array (result is a view) + * + * This flag may be requested in constructor functions. + */ #define NPY_ARRAY_ENSURENOCOPY 0x4000 /* From f058aea694327b517eb5c3d786c7f6a3e9c9898d Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 12 Nov 2021 16:40:50 -0600 Subject: [PATCH 54/54] BUG: Fix refcounting issue and missed scalar special case for never-copy logic --- numpy/core/src/multiarray/ctors.c | 18 +++++++++++------- numpy/core/tests/test_multiarray.py | 3 +++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 819bb22bedd6..b6242685447b 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1703,7 +1703,17 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, ((PyVoidScalarObject *)op)->flags, NULL, op); } - else if (cache == 0 && newtype != NULL && + /* + * If we got this far, we definitely have to create a copy, since we are + * converting either from a scalar (cache == NULL) or a (nested) sequence. + */ + if (flags & NPY_ARRAY_ENSURENOCOPY ) { + PyErr_SetString(PyExc_ValueError, + "Unable to avoid copy while creating an array."); + return NULL; + } + + if (cache == 0 && newtype != NULL && PyDataType_ISSIGNED(newtype) && PyArray_IsScalar(op, Generic)) { assert(ndim == 0); /* @@ -1741,12 +1751,6 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, /* Create a new array and copy the data */ Py_INCREF(dtype); /* hold on in case of a subarray that is replaced */ - if (flags & NPY_ARRAY_ENSURENOCOPY ) { - PyErr_SetString(PyExc_ValueError, - "Unable to avoid copy while creating " - "an array from descriptor."); - return NULL; - } ret = (PyArrayObject *)PyArray_NewFromDescr( &PyArray_Type, dtype, ndim, dims, NULL, NULL, flags&NPY_ARRAY_F_CONTIGUOUS, NULL); diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 18302543adde..4413cd0d0e69 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7840,6 +7840,9 @@ def test_scalars(self): copy=self.RaiseOnBool()) assert_raises(ValueError, _multiarray_tests.npy_ensurenocopy, [1]) + # Casting with a dtype (to unsigned integers) can be special: + with pytest.raises(ValueError): + np.array(pyscalar, dtype=np.int64, copy=np._CopyMode.NEVER) def test_compatible_cast(self):