8000 Merge pull request #17401 from seberg/restructure-casting · numpy/numpy@ba77419 · GitHub
[go: up one dir, main page]
Skip to content
< 8000 script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_gsap_index_js-028cb2a18f5a.js" defer="defer">

Commit ba77419

Browse files
charrischarris
authored
Merge pull request #17401 from seberg/restructure-casting
MAINT: Rewrite can-cast logic in terms of NEP 42
2 parents 44f8f9d + a806c21 commit ba77419

25 files changed

+4162
-708
lines changed

‎azure-pipelines.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,9 @@ stages:
153153
CC: /usr/bin/clang
154154
condition: eq(variables['USE_OPENBLAS'], '1')
155155
- script: python setup.py build -j 4 build_ext --inplace install
156-
displayName: 'Build NumPy without OpenBLAS'
156+
displayName: 'Build NumPy without OpenBLAS and new casting'
157157
env:
158+
NPY_USE_NEW_CASTINGIMPL: 1
158159
BLAS: None
159160
LAPACK: None
160161
ATLAS: None

‎doc/neps/nep-0042-new-dtypes.rst

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -784,7 +784,7 @@ Its ``resolve_descriptors`` function may look like::
784784
# This is always an "unsafe" cast, but for int64, we can represent
785785
# it by a simple view (if the dtypes are both canonical).
786786
# (represented as C-side flags here).
787-
safety_and_view = NPY_UNSAFE_CASTING | NPY_CAST_IS_VIEW
787+
safety_and_view = NPY_UNSAFE_CASTING | _NPY_CAST_IS_VIEW
788788
return safety_and_view, (from_dtype, to_dtype)
789789

790790
.. note::
@@ -1305,7 +1305,7 @@ The external API for ``CastingImpl`` will be limited initially to defining:
13051305
``casting`` will be set to ``NPY_EQUIV_CASTING``, ``NPY_SAFE_CASTING``,
13061306
``NPY_UNSAFE_CASTING``, or ``NPY_SAME_KIND_CASTING``.
13071307
A new, additional flag,
1308-
``NPY_CAST_IS_VIEW``, can be set to indicate that no cast is necessary and a
1308+
``_NPY_CAST_IS_VIEW``, can be set to indicate that no cast is necessary and a
13091309
view is sufficient to perform the cast. The cast should return
13101310
``-1`` when a custom error is set and ``NPY_NO_CASTING`` to indicate
13111311
that a generic casting error should be set (this is in most cases

‎doc/source/reference/global_state.rst

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,18 @@ in C which iterates through arrays that may or may not be
8383
contiguous in memory.
8484
Most users will have no reason to change these; for details
8585
see the :ref:`memory layout <memory-layout>` documentation.
86+
87+
Using the new casting implementation
88+
------------------------------------
89+
90+
Within NumPy 1.20 it is possible to enable the new experimental casting
91+
implementation for testing purposes. To do this set::
92+
93+
NPY_USE_NEW_CASTINGIMPL=1
94+
95+
Setting the flag is only useful to aid with NumPy developement to ensure the
96+
new version is bug free and should be avoided for production code.
97+
It is a helpful test for projects that either create custom datatypes or
98+
use for example complicated structured dtypes. The flag is expected to be
99+
removed in 1.21 with the new version being always in use.
100+

‎numpy/core/code_generators/genapi.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
join('multiarray', 'array_assign_array.c'),
2 F438 727
join('multiarray', 'array_assign_scalar.c'),
2828
join('multiarray', 'array_coercion.c'),
29+
join('multiarray', 'array_method.c'),
2930
join('multiarray', 'arrayobject.c'),
3031
join('multiarray', 'arraytypes.c.src'),
3132
join('multiarray', 'buffer.c'),

‎numpy/core/include/numpy/ndarraytypes.h

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,7 @@ typedef enum {
210210

211211
/* For specifying allowed casting in operations which support it */
212212
typedef enum {
213+
_NPY_ERROR_OCCURRED_IN_CAST = -1,
213214
/* Only allow identical types */
214215
NPY_NO_CASTING=0,
215216
/* Allow identical and byte swapped types */
@@ -219,7 +220,14 @@ typedef enum {
219220
/* Allow safe casts or casts within the same kind */
220221
NPY_SAME_KIND_CASTING=3,
221222
/* Allow any casts */
222-
NPY_UNSAFE_CASTING=4
223+
NPY_UNSAFE_CASTING=4,
224+
/*
225+
* Flag to allow signalling that a cast is a view, this flag is not
226+
* valid when requesting a cast of specific safety.
227+
* _NPY_CAST_IS_VIEW|NPY_EQUIV_CASTING means the same as NPY_NO_CASTING.
228+
*/
229+
// TODO-DTYPES: Needs to be documented.
230+
_NPY_CAST_IS_VIEW = 1 << 16,
223231
} NPY_CASTING;
224232

225233
typedef enum {
@@ -1900,6 +1908,12 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size,
19001908
default_descr_function *default_descr;
19011909
common_dtype_function *common_dtype;
19021910
common_instance_function *common_instance;
1911+
/*
1912+
* Dictionary of ArrayMethods representing most possible casts
1913+
* (structured and object are exceptions).
1914+
* This should potentially become a weak mapping in the future.
1915+
*/
1916+
PyObject *castingimpls;
19031917
};
19041918

19051919
#endif /* NPY_INTERNAL_BUILD */

‎numpy/core/setup.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@
2323
NPY_RELAXED_STRIDES_DEBUG = (os.environ.get('NPY_RELAXED_STRIDES_DEBUG', "0") != "0")
2424
NPY_RELAXED_STRIDES_DEBUG = NPY_RELAXED_STRIDES_DEBUG and NPY_RELAXED_STRIDES_CHECKING
2525

26+
# Set to True to use the new casting implementation as much as implemented.
27+
# Allows running the full test suit to exercise the new machinery until
28+
# it is used as default and the old version is eventually deleted.
29+
NPY_USE_NEW_CASTINGIMPL = os.environ.get('NPY_USE_NEW_CASTINGIMPL', "0") != "0"
30+
2631
# XXX: ugly, we use a class to avoid calling twice some expensive functions in
2732
# config.h/numpyconfig.h. I don't see a better way because distutils force
2833
# config.h generation inside an Extension class, and as such sharing
@@ -468,6 +473,10 @@ def generate_config_h(ext, build_dir):
468473
if NPY_RELAXED_STRIDES_DEBUG:
469474
moredefs.append(('NPY_RELAXED_STRIDES_DEBUG', 1))
470475

476+
# Use the new experimental casting implementation in NumPy 1.20:
477+
if NPY_USE_NEW_CASTINGIMPL:
478+
moredefs.append(('NPY_USE_NEW_CASTINGIMPL', 1))
479+
471480
# Get long double representation
472481
rep = check_long_double_representation(config_cmd)
473482
moredefs.append(('HAVE_LDOUBLE_%s' % rep, 1))
@@ -769,6 +778,7 @@ def get_mathlib_info(*args):
769778
join('src', 'multiarray', 'arraytypes.h'),
770779
join('src', 'multiarray', 'arrayfunction_override.h'),
771780
join('src', 'multiarray', 'array_coercion.h'),
781+
join('src', 'multiarray', 'array_method.h'),
772782
join('src', 'multiarray', 'npy_buffer.h'),
773783
join('src', 'multiarray', 'calculation.h'),
774784
join('src', 'multiarray', 'common.h'),
@@ -784,6 +794,7 @@ def get_mathlib_info(*args):
784794
join('src', 'multiarray', 'getset.h'),
785795
join('src', 'multiarray', 'hashdescr.h'),
786796
join('src', 'multiarray', 'iterators.h'),
797+
join('src', 'multiarray', 'legacy_dtype_implementation.h'),
787798
join('src', 'multiarray', 'mapping.h'),
788799
join('src', 'multiarray', 'methods.h'),
789800
join('src', 'multiarray', 'multiarraymodule.h'),
@@ -824,6 +835,7 @@ def get_mathlib_info(*args):
824835
join('src', 'multiarray', 'arrayobject.c'),
825836
join('src', 'multiarray', 'arraytypes.c.src'),
826837
join('src', 'multiarray', 'array_coercion.c'),
838+
join('src', 'multiarray', 'array_method.c'),
827839
join('src', 'multiarray', 'array_assign_scalar.c'),
828840
join('src', 'multiarray', 'array_assign_array.c'),
829841
join('src', 'multiarray', 'arrayfunction_override.c'),
@@ -850,6 +862,7 @@ def get_mathlib_info(*args):
850862
join('src', 'multiarray', 'hashdescr.c'),
851863
join('src', 'multiarray', 'item_selection.c'),
852864
join('src', 'multiarray', 'iterators.c'),
865+
join('src', 'multiarray', 'legacy_dtype_implementation.c'),
853866
join('src', 'multiarray', 'lowlevel_strided_loops.c.src'),
854867
join('src', 'multiarray', 'mapping.c'),
855868
join('src', 'multiarray', 'methods.c'),

‎numpy/core/src/multiarray/_datetime.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,4 +373,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step,
373373
NPY_NO_EXPORT PyArray_Descr *
374374
find_object_datetime_type(PyObject *obj, int type_num);
375375

376+
NPY_NO_EXPORT int
377+
PyArray_InitializeDatetimeCasts();
378+
376379
#endif

‎numpy/core/src/multiarray/_multiarray_tests.c.src

Lines changed: 96 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "common.h"
1010
#include "mem_overlap.h"
1111
#include "npy_extint128.h"
12+
#include "array_method.h"
< 179B code>1213

1314
#if defined(MS_WIN32) || defined(__CYGWIN__)
1415
#define EXPORT(x) __declspec(dllexport) x
@@ -977,6 +978,79 @@ get_c_wrapping_array(PyObject* NPY_UNUSED(self), PyObject* arg)
977978
}
978979

979980

981+
static PyObject *
982+
get_all_cast_information(PyObject *NPY_UNUSED(mod), PyObject *NPY_UNUSED(args))
983+
{
984+
PyObject *result = PyList_New(0);
985+
if (result == NULL) {
986+
return NULL;
987+
}
988+
PyObject *classes = PyObject_CallMethod(
989+
(PyObject *)&PyArrayDescr_Type, "__subclasses__", "");
990+
if (classes == NULL) {
991+
return NULL;
992+
}
993+
Py_SETREF(classes, PySequence_Fast(classes, NULL));
994+
if (classes == NULL) {
995+
goto fail;
996+
}
997+
998+
Py_ssize_t nclass = PySequence_Length(classes);
999+
for (Py_ssize_t i = 0; i < nclass; i++) {
1000+
PyArray_DTypeMeta *from_dtype = (
1001+
(PyArray_DTypeMeta *)PySequence_Fast_GET_ITEM(classes, i));
1002+
if (from_dtype->abstract) {
1003+
/*
1004+
* TODO: In principle probably needs to recursively check this,
1005+
* also we may allow casts to abstract dtypes at some point.
1006+
*/
1007+
continue;
1008+
}
1009+
1010+
PyObject *to_dtype, *cast_obj;
1011+
Py_ssize_t pos = 0;
1012+
1013+
while (PyDict_Next(from_dtype->castingimpls, &pos, &to_dtype, &cast_obj)) {
1014+
if (cast_obj == Py_None) {
1015+
continue;
1016+
}
1017+
PyArrayMethodObject *cast = (PyArrayMethodObject *)cast_obj;
1018+
1019+
/* Pass some information about this cast out! */
1020+
PyObject *cast_info = Py_BuildValue("{sOsOsisisisisisssi}",
1021+
"from", from_dtype,
1022+
"to", to_dtype,
1023+
"legacy", (cast->name != NULL &&
1024+
strncmp(cast->name, "legacy_", 7) == 0),
1025+
"casting", cast->casting & ~_NPY_CAST_IS_VIEW,
1026+
"requires_pyapi", cast->flags & NPY_METH_REQUIRES_PYAPI,
1027+
"supports_unaligned",
1028+
cast->flags & NPY_METH_SUPPORTS_UNALIGNED,
1029+
"no_floatingpoint_errors",
1030+
cast->flags & NPY_METH_NO_FLOATINGPOINT_ERRORS,
1031+
"name", cast->name,
1032+
"cast_is_view",
1033+
cast->casting & _NPY_CAST_IS_VIEW);
1034+
if (cast_info == NULL) {
1035+
goto fail;
1036+
}
1037+
int res = PyList_Append(result, cast_info);
1038+
Py_DECREF(cast_info);
1039+
if (res < 0) {
1040+
goto fail;
1041+
}
1042+
}
1043+
}
1044+
Py_DECREF(classes);
1045+
return result;
1046+
1047+
fail:
1048+
Py_XDECREF(classes);
1049+
Py_XDECREF(result);
1050+
return NULL;
1051+
}
1052+
1053+
9801054
/*
9811055
* Test C-api level item getting.
9821056
*/
@@ -2010,6 +2084,18 @@ getset_numericops(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args))
20102084
return ret;
20112085
}
20122086

2087+
2088+
static PyObject *
2089+
uses_new_casts(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args))
2090+
{
2091+
#if NPY_USE_NEW_CASTINGIMPL
2092+
Py_RETURN_TRUE;
2093+
#else
2094+
Py_RETURN_FALSE;
2095+
#endif
2096+
}
2097+
2098+
20132099
static PyObject *
20142100
run_byteorder_converter(PyObject* NPY_UNUSED(self), PyObject *args)
20152101
{
@@ -2113,8 +2199,8 @@ run_casting_converter(PyObject* NPY_UNUSED(self), PyObject *args)
21132199
case NPY_SAFE_CASTING: return PyUnicode_FromString("NPY_SAFE_CASTING");
21142200
case NPY_SAME_KIND_CASTING: return PyUnicode_FromString("NPY_SAME_KIND_CASTING");
21152201
case NPY_UNSAFE_CASTING: return PyUnicode_FromString("NPY_UNSAFE_CASTING");
2202+
default: return PyLong_FromLong(casting);
21162203
}
2117-
return PyLong_FromLong(casting);
21182204
}
21192205

21202206
static PyObject *
@@ -2194,6 +2280,12 @@ static PyMethodDef Multiarray_TestsMethods[] = {
21942280
{"get_c_wrapping_array",
21952281
get_c_wrapping_array,
21962282
METH_O, NULL},
2283+
{"get_all_cast_information",
2284+
get_all_cast_information,
2285+
METH_NOARGS,
2286+
"Return a list with info on all available casts. Some of the info"
2287+
"may differ for an actual cast if it uses value-based casting "
2288+
"(flexible types)."},
21972289
{"array_indexing",
21982290
array_indexing,
21992291
METH_VARARGS, NULL},
@@ -2254,6 +2346,9 @@ static PyMethodDef Multiarray_TestsMethods[] = {
22542346
{"getset_numericops",
22552347
getset_numericops,
22562348
METH_NOARGS, NULL},
2349+
{"uses_new_casts",
2350+
uses_new_casts,
2351+
METH_NOARGS, NULL},
22572352
/**begin repeat
22582353
* #name = cabs, carg#
22592354
*/

0 commit comments

Comments
 (0)
0