8000 Merge pull request #17401 from seberg/restructure-casting · numpy/numpy@ba77419 · GitHub
[go: up one dir, main page]
Skip to content

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit ba77419

Browse files
charrischarris
authored
Merge pull request #17401 from seberg/restructure-casting
MAINT: Rewrite can-cast logic in terms of NEP 42
2 parents 44f8f9d + a806c21 commit ba77419

25 files changed

+4162
-708
lines changed

‎azure-pipelines.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,9 @@ stages:
153153
CC: /usr/bin/clang
154154
condition: eq(variables['USE_OPENBLAS'], '1')
155155
- script: python setup.py build -j 4 build_ext --inplace install
156-
displayName: 'Build NumPy without OpenBLAS'
156+
displayName: 'Build NumPy without OpenBLAS and new casting'
157157
env:
158+
NPY_USE_NEW_CASTINGIMPL: 1
158159
BLAS: None
159160
LAPACK: None
160161
ATLAS: None

‎doc/neps/nep-0042-new-dtypes.rst

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -784,7 +784,7 @@ Its ``resolve_descriptors`` function may look like::
784784
# This is always an "unsafe" cast, but for int64, we can represent
785785
# it by a simple view (if the dtypes are both canonical).
786786
# (represented as C-side flags here).
787-
safety_and_view = NPY_UNSAFE_CASTING | NPY_CAST_IS_VIEW
787+
safety_and_view = NPY_UNSAFE_CASTING | _NPY_CAST_IS_VIEW
788788
return safety_and_view, (from_dtype, to_dtype)
789789

790790
.. note::
@@ -1305,7 +1305,7 @@ The external API for ``CastingImpl`` will be limited initially to defining:
13051305
``casting`` will be set to ``NPY_EQUIV_CASTING``, ``NPY_SAFE_CASTING``,
13061306
``NPY_UNSAFE_CASTING``, or ``NPY_SAME_KIND_CASTING``.
13071307
A new, additional flag,
1308-
``NPY_CAST_IS_VIEW``, can be set to indicate that no cast is necessary and a
1308+
``_NPY_CAST_IS_VIEW``, can be set to indicate that no cast is necessary and a
13091309
view is sufficient to perform the cast. The cast should return
13101310
``-1`` when a custom error is set and ``NPY_NO_CASTING`` to indicate
13111311
that a generic casting error should be set (this is in most cases

‎doc/source/reference/global_state.rst

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,18 @@ in C which iterates through arrays that may or may not be
8383
contiguous in memory.
8484
Most users will have no reason to change these; for details
8585
see the :ref:`memory layout <memory-layout>` documentation.
86+
87+
Using the new casting implementation
88+
------------------------------------
89+
90+
Within NumPy 1.20 it is possible to enable the new experimental casting
91+
implementation for testing purposes. To do this set::
92+
93+
NPY_USE_NEW_CASTINGIMPL=1
94+
95+
Setting the flag is only useful to aid with NumPy developement to ensure the
96+
new version is bug free and should be avoided for production code.
97+
It is a helpful test for projects that either create custom datatypes or
98+
use for example complicated structured dtypes. The flag is expected to be
99+
removed in 1.21 with the new version being always in use.
100+

‎numpy/core/code_generators/genapi.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
join('multiarray', 'array_assign_array.c'),
2727
join('multiarray', 'array_assign_scalar.c'),
2828
join('multiarray', 'array_coercion.c'),
29+
join('multiarray', 'array_method.c'),
2930
join('multiarray', 'arrayobject.c'),
3031
join('multiarray', 'arraytypes.c.src'),
3132
join('multiarray', 'buffer.c'),

‎numpy/core/include/numpy/ndarraytypes.h

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,7 @@ typedef enum {
210210

211211
/* For specifying allowed casting in operations which support it */
212212
typedef enum {
213+
_NPY_ERROR_OCCURRED_IN_CAST = -1,
213214
/* Only allow identical types */
214215
NPY_NO_CASTING=0,
215216
/* Allow identical and byte swapped types */
@@ -219,7 +220,14 @@ typedef enum {
219220
/* Allow safe casts or casts within the same kind */
220221
NPY_SAME_KIND_CASTING=3,
221222
/* Allow any casts */
222-
NPY_UNSAFE_CASTING=4
223+
NPY_UNSAFE_CASTING=4,
224+
/*
225+
* Flag to allow signalling that a cast is a view, this flag is not
226+
* valid when requesting a cast of specific safety.
227+
* _NPY_CAST_IS_VIEW|NPY_EQUIV_CASTING means the same as NPY_NO_CASTING.
228+
*/
229+
// TODO-DTYPES: Needs to be documented.
230+
_NPY_CAST_IS_VIEW = 1 << 16,
223231
} NPY_CASTING;
224232

225233
typedef enum {
@@ -1900,6 +1908,12 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size,
19001908
default_descr_function *default_descr;
19011909
common_dtype_function *common_dtype;
19021910
common_instance_function *common_instance;
1911+
/*
1912+
* Dictionary of ArrayMethods representing most possible casts
1913+
* (structured and object are exceptions).
1914+
* This should potentially become a weak mapping in the future.
1915+
*/
1916+
PyObject *castingimpls;
19031917
};
19041918

19051919
#endif /* NPY_INTERNAL_BUILD */

‎numpy/core/setup.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@
2323
NPY_RELAXED_STRIDES_DEBUG = (os.environ.get('NPY_RELAXED_STRIDES_DEBUG', "0") != "0")
2424
NPY_RELAXED_STRIDES_DEBUG = NPY_RELAXED_STRIDES_DEBUG and NPY_RELAXED_STRIDES_CHECKING
2525

26+
# Set to True to use the new casting implementation as much as implemented.
27+
# Allows running the full test suit to exercise the new machinery until
28+
# it is used as default and the old version is eventually deleted.
29+
NPY_USE_NEW_CASTINGIMPL = os.environ.get('NPY_USE_NEW_CASTINGIMPL', "0") != "0"
30+
2631
# XXX: ugly, we use a class to avoid calling twice some expensive functions in
2732
# config.h/numpyconfig.h. I don't see a better way because distutils force
2833
# config.h generation inside an Extension class, and as such sharing
@@ -468,6 +473,10 @@ def generate_config_h(ext, build_dir):
468473
if NPY_RELAXED_STRIDES_DEBUG:
469474
moredefs.append(('NPY_RELAXED_STRIDES_DEBUG', 1))
470475

476+
# Use the new experimental casting implementation in NumPy 1.20:
477+
if NPY_USE_NEW_CASTINGIMPL:
478+
moredefs.append(('NPY_USE_NEW_CASTINGIMPL', 1))
479+
471480
# Get long double representation
472481
rep = check_long_double_representation(config_cmd)
473482
moredefs.append(('HAVE_LDOUBLE_%s' % rep, 1))
@@ -769,6 +778,7 @@ def get_mathlib_info(*args):
769778
join('src', 'multiarray', 'arraytypes.h'),
770779
join('src', 'multiarray', 'arrayfunction_override.h'),
771780
join('src', 'multiarray', 'array_coercion.h'),
781+
join('src', 'multiarray', 'array_method.h'),
772782
join('src', 'multiarray', 'npy_buffer.h'),
773783
join('src', 'multiarray', 'calculation.h'),
774784
join('src', 'multiarray', 'common.h'),
@@ -784,6 +794,7 @@ def get_mathlib_info(*args):
784794
join('src', 'multiarray', 'getset.h'),
785795
join('src', 'multiarray', 'hashdescr.h'),
786796
join('src', 'multiarray', 'iterators.h'),
797+
join('src', 'multiarray', 'legacy_dtype_implementation.h'),
787798
join('src', 'multiarray', 'mapping.h'),
788799
join('src', 'multiarray', 'methods.h'),
789800
join('src', 'multiarray', 'multiarraymodule.h'),
@@ -824,6 +835,7 @@ def get_mathlib_info(*args):
824835
join('src', 'multiarray', 'arrayobject.c'),
825836
join('src', 'multiarray', 'arraytypes.c.src'),
826837
join('src', 'multiarray', 'array_coercion.c'),
838+
join('src', 'multiarray', 'array_method.c'),
827839
join('src', 'multiarray', 'array_assign_scalar.c'),
828840
join('src', 'multiarray', 'array_assign_array.c'),
829841
join('src', 'multiarray', 'arrayfunction_override.c'),
@@ -850,6 +862,7 @@ def get_mathlib_info(*args):
850862
join('src', 'multiarray', 'hashdescr.c'),
851863
join('src', 'multiarray', 'item_selection.c'),
852864
join('src', 'multiarray', 'iterators.c'),
865+
join('src', 'multiarray', 'legacy_dtype_implementation.c'),
853866
join('src', 'multiarray', 'lowlevel_strided_loops.c.src'),
854867
join('src', 'multiarray', 'mapping.c'),
855868
join('src', 'multiarray', 'methods.c'),

‎numpy/core/src/multiarray/_datetime.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,4 +373,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step,
373373
NPY_NO_EXPORT PyArray_Descr *
374374
find_object_datetime_type(PyObject *obj, int type_num);
375375

376+
NPY_NO_EXPORT int
377+
PyArray_InitializeDatetimeCasts();
378+
376379
#endif

‎numpy/core/src/multiarray/_multiarray_tests.c.src

Lines changed: 96 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "common.h"
1010
#include "mem_overlap.h"
1111
#include "npy_extint128.h"
12+
#include "array_method.h"
1213

1314
#if defined(MS_WIN32) || defined(__CYGWIN__)
1415
#define EXPORT(x) __declspec(dllexport) x
@@ -977,6 +978,79 @@ get_c_wrapping_array(PyObject* NPY_UNUSED(self), PyObject* arg)
977978
}
978979

979980

981+
static PyObject *
982+
get_all_cast_information(PyObject *NPY_UNUSED(mod), PyObject *NPY_UNUSED(args))
983+
{
984+
PyObject *result = PyList_New(0);
985+
if (result == NULL) {
986+
return NULL;
987+
}
988+
PyObject *classes = PyObject_CallMethod(
989+
(PyObject *)&PyArrayDescr_Type, "__subclasses__", "");
990+
if (classes == NULL) {
991+
return NULL;
992+
}
993+
Py_SETREF(classes, PySequence_Fast(classes, NULL));
994+
if (classes == NULL) {
995+
goto fail;
996+
}
997+
998+
Py_ssize_t nclass = PySequence_Length(classes);
999+
for (Py_ssize_t i = 0; i < nclass; i++) {
1000+
PyArray_DTypeMeta *from_dtype = (
1001+
(PyArray_DTypeMeta *)PySequence_Fast_GET_ITEM(classes, i));
1002+
if (from_dtype->abstract) {
1003+
/*
1004+
* TODO: In principle probably needs to recursively check this,
1005+
* also we may allow casts to abstract dtypes at some point.
1006+
*/
1007+
continue;
1008+
}
1009+
1010+
PyObject *to_dtype, *cast_obj;
1011+
Py_ssize_t pos = 0;
1012+
1013+
while (PyDict_Next(from_dtype->castingimpls, &pos, &to_dtype, &cast_obj)) {
1014+
if (cast_obj == Py_None) {
1015+
continue;
1016+
}
1017+
PyArrayMethodObject *cast = (PyArrayMethodObject *)cast_obj;
1018+
1019+
/* Pass some information about this cast out! */
1020+
PyObject *cast_info = Py_BuildValue("{sOsOsisisisisisssi}",
1021+
"from", from_dtype,
1022+
"to", to_dtype,
1023+
"legacy", (cast->name != NULL &&
1024+
strncmp(cast->name, "legacy_", 7) == 0),
1025+
"casting", cast->casting & ~_NPY_CAST_IS_VIEW,
1026+
"requires_pyapi", cast->flags & NPY_METH_REQUIRES_PYAPI,
1027+
"supports_unaligned",
1028+
cast->flags & NPY_METH_SUPPORTS_UNALIGNED,
1029+
"no_floatingpoint_errors",
1030+
cast->flags & NPY_METH_NO_FLOATINGPOINT_ERRORS,
1031+
"name", cast->name,
1032+
"cast_is_view",
1033+
cast->casting & _NPY_CAST_IS_VIEW);
1034+
if (cast_info == NULL) {
1035+
goto fail;
1036+
}
1037+
int res = PyList_Append(result, cast_info);
1038+
Py_DECREF(cast_info);
1039+
if (res < 0) {
1040+
goto fail;
1041+
}
1042+
}
1043+
}
1044+
Py_DECREF(classes);
1045+
return result;
1046+
1047+
fail:
1048+
Py_XDECREF(classes);
1049+
Py_XDECREF(result);
1050+
return NULL;
1051+
}
1052+
1053+
9801054
/*
9811055
* Test C-api level item getting.
9821056
*/
@@ -2010,6 +2084,18 @@ getset_numericops(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args))
20102084
return ret;
20112085
}
20122086

2087+
2088+
static PyObject *
2089+
uses_new_casts(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args))
2090+
{
2091+
#if NPY_USE_NEW_CASTINGIMPL
2092+
Py_RETURN_TRUE;
2093+
#else
2094+
Py_RETURN_FALSE;
2095+
#endif
2096+
}
2097+
2098+
20132099
static PyObject *
20142100
run_byteorder_converter(PyObject* NPY_UNUSED(self), PyObject *args)
20152101
{
@@ -2113,8 +2199,8 @@ run_casting_converter(PyObject* NPY_UNUSED(self), PyObject *args)
21132199
case NPY_SAFE_CASTING: return PyUnicode_FromString("NPY_SAFE_CASTING");
21142200
case NPY_SAME_KIND_CASTING: return PyUnicode_FromString("NPY_SAME_KIND_CASTING");
21152201
case NPY_UNSAFE_CASTING: return PyUnicode_FromString("NPY_UNSAFE_CASTING");
2202+
default: return PyLong_FromLong(casting);
21162203
}
2117-
return PyLong_FromLong(casting);
21182204
}
21192205

21202206
static PyObject *
@@ -2194,6 +2280,12 @@ static PyMethodDef Multiarray_TestsMethods[] = {
21942280
{"get_c_wrapping_array",
21952281
get_c_wrapping_array,
21962282
METH_O, NULL},
2283+
{"get_all_cast_information",
2284+
get_all_cast_information,
2285+
METH_NOARGS,
2286+
"Return a list with info on all available casts. Some of the info"
2287+
"may differ for an actual cast if it uses value-based casting "
2288+
"(flexible types)."},
21972289
{"array_indexing",
21982290
array_indexing,
21992291
METH_VARARGS, NULL},
@@ -2254,6 +2346,9 @@ static PyMethodDef Multiarray_TestsMethods[] = {
22542346
{"getset_numericops",
22552347
getset_numericops,
22562348
METH_NOARGS, NULL},
2349+
{"uses_new_casts",
2350+
uses_new_casts,
2351+
METH_NOARGS, NULL},
22572352
/**begin repeat
22582353
* #name = cabs, carg#
22592354
*/

0 commit comments

Comments
 (0)
0