8000 API,MAINT: Make ``NpyIter_GetTransferFlags`` public and avoid old use… · numpy/numpy@965fdf5 · GitHub
[go: up one dir, main page]
Skip to content

Commit 965fdf5

Browse files
sebergseberg
authored
API,MAINT: Make NpyIter_GetTransferFlags public and avoid old uses (#27998)
* API: Make ``NpyIter_IterationNeedsAPI`` public This makes NpyIter_IterationNeedsAPI public, since there is no reason not to now that the flags are public. * MAINT: Remove/move iterator "needs api" and avoid it in reductions * MAINT: Generally avoid IterationNeedsAPI or add a comment why it is there Also remove any duplicate calls which are now much slower. * DOC: Release note for new NpyIter_GetTransferFlags and needs API change * MAINT: Make `needs_api` declaration local in most cases as per review * MAINT: Return value is NPY_ARRAYMETHOD_FLAGS and remove internal define * fix typo in release note
1 parent 0428a34 commit 965fdf5

File tree

14 files changed

+151
-140
lines changed

14 files changed

+151
-140
lines changed

‎doc/release/upcoming_changes/27998.c_api.rst

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
New `NpyIter_GetTransferFlags` and ``NpyIter_IterationNeedsAPI`` change
2+
-----------------------------------------------------------------------
3+
NumPy now has the new `NpyIter_GetTransferFlags` function as a more precise
4+
way checking of iterator/buffering needs. I.e. whether the Python API/GIL is
5+
required or floating point errors may occur.
6+
This function is also faster if you already know your needs without buffering.
7+
8+
The ``NpyIter_IterationNeedsAPI`` function now performs all the checks that were
9+
previously performed at setup time. While it was never necessary to call it
10+
multiple times, doing so will now have a larger cost.

‎doc/source/reference/c-api/iterator.rst

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,9 @@ Construction and destruction
434434
is enabled, the caller must be sure to check whether
435435
``NpyIter_IterationNeedsAPI(iter)`` is true, in which case
436436
it may not release the GIL during iteration.
437+
If you are working with known dtypes `NpyIter_GetTransferFlags` is
438+
a faster and more precise way to check for whether the iterator needs
439+
the API due to buffering.
437440
438441
.. c:macro:: NPY_ITER_ZEROSIZE_OK
439442
@@ -823,6 +826,20 @@ Construction and destruction
823826
824827
Returns ``NPY_SUCCEED`` or ``NPY_FAIL``.
825828
829+
.. c:function:: NPY_ARRAYMETHOD_FLAGS NpyIter_GetTransferFlags(NpyIter *iter)
830+
831+
.. versionadded:: 2.3
832+
833+
Fetches the `NPY_METH_RUNTIME_FLAGS` which provide the information on
834+
whether buffering needs the Python GIL (`NPY_METH_REQUIRES_PYAPI`) or
835+
floating point errors may be set (`NPY_METH_NO_FLOATINGPOINT_ERRORS`).
836+
837+
Prior to NumPy 2.3, the public function available was
838+
``NpyIter_IterationNeedsAPI``, which is still available and additionally
839+
checks for object (or similar) dtypes and not exclusively for
840+
buffering/iteration needs itself.
841+
In general, this function should be preferred.
842+
826843
.. c:function:: int NpyIter_Reset(NpyIter* iter, char** errmsg)
827844
828845
Resets the iterator back to its initial state, at the beginning

‎numpy/_core/code_generators/cversions.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,5 +77,5 @@
7777
0x00000012 = 2b8f1f4da822491ff030b2b37dff07e3
7878
# Version 19 (NumPy 2.1.0) Only header additions
7979
# Version 19 (NumPy 2.2.0) No change
80-
# Version 19 (NumPy 2.3.0) No change
81-
0x00000013 = 2b8f1f4da822491ff030b2b37dff07e3
80+
# Version 19 (NumPy 2.3.0)
81+
0x00000013 = e56b74d32a934d085e7c3414cb9999b8,

‎numpy/_core/code_generators/numpy_api.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ def get_annotations():
106106
'__unused_indices__': (
107107
[1, 4, 40, 41, 66, 67, 68, 81, 82, 83,
108108
103, 115, 117, 122, 163, 164, 171, 173, 197,
109-
201, 202, 208, 219, 220, 221, 222, 223, 278,
109+
201, 202, 208, 219, 220, 221, 222, 278,
110110
291, 293, 294, 295, 301]
111111
# range/slots reserved DType classes (see _public_dtype_api_table.h):
112112
+ list(range(320, 361)) + [366, 367, 368]
@@ -293,8 +293,8 @@ def get_annotations():
293293
# Unused slot 220, was `PyArray_DatetimeToDatetimeStruct`
294294
# Unused slot 221, was `PyArray_TimedeltaToTimedeltaStruct`
295295
# Unused slot 222, was `PyArray_DatetimeStructToDatetime`
296-
# Unused slot 223, was `PyArray_TimedeltaStructToTimedelta`
297296
# NDIter API
297+
'NpyIter_GetTransferFlags': (223, MinVersion("2.3")),
298298
'NpyIter_New': (224,),
299299
'NpyIter_MultiNew': (225,),
300300
'NpyIter_AdvancedNew': (226,),
@@ -407,6 +407,8 @@ def get_annotations():
407407
# `PyDataType_GetArrFuncs` checks for the NumPy runtime version.
408408
'_PyDataType_GetArrFuncs': (365,),
409409
# End 2.0 API
410+
# NpyIterGetTransferFlags (slot 223) added.
411+
# End 2.3 API
410412
}
411413

412414
ufunc_types_api = {

‎numpy/_core/src/multiarray/common.h

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,6 @@ extern "C" {
1818

1919
#define error_converting(x) (((x) == -1) && PyErr_Occurred())
2020

21-
#ifdef NPY_ALLOW_THREADS
22-
#define NPY_BEGIN_THREADS_NDITER(iter) \
23-
do { \
24-
if (!NpyIter_IterationNeedsAPI(iter)) { \
25-
NPY_BEGIN_THREADS_THRESHOLDED(NpyIter_GetIterSize(iter)); \
26-
} \
27-
} while(0)
28-
#else
29-
#define NPY_BEGIN_THREADS_NDITER(iter)
30-
#endif
31-
3221

3322
NPY_NO_EXPORT PyArray_Descr *
3423
PyArray_DTypeFromObjectStringDiscovery(

‎numpy/_core/src/multiarray/ctors.c

Lines changed: 4 additions & 13 deletions
2773
Original file line numberDiff line numberDiff line change
@@ -2696,7 +2696,6 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order)
26962696

26972697
npy_intp dst_count, src_count, count;
26982698
npy_intp dst_size, src_size;
2699-
int needs_api;
27002699

27012700
NPY_BEGIN_THREADS_DEF;
27022701

@@ -2757,13 +2756,13 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order)
27572756
/* Get all the values needed for the inner loop */
27582757
dst_iternext = NpyIter_GetIterNext(dst_iter, NULL);
27592758
dst_dataptr = NpyIter_GetDataPtrArray(dst_iter);
2760-
/* Since buffering is disabled, we can cache the stride */
2759+
/* The inner stride is also the fixed stride for the whole iteration. */
27612760
dst_stride = NpyIter_GetInnerStrideArray(dst_iter)[0];
27622761
dst_countptr = NpyIter_GetInnerLoopSizePtr(dst_iter);
27632762

27642763
src_iternext = NpyIter_GetIterNext(src_iter, NULL);
27652764
src_dataptr = NpyIter_GetDataPtrArray(src_iter);
2766-
/* Since buffering is disabled, we can cache the stride */
2765+
/* The inner stride is also the fixed stride for the whole iteration. */
27672766
src_stride = NpyIter_GetInnerStrideArray(src_iter)[0];
27682767
src_countptr = NpyIter_GetInnerLoopSizePtr(src_iter);
27692768

@@ -2773,15 +2772,6 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order)
27732772
return -1;
2774
}
27752774

2776-
needs_api = NpyIter_IterationNeedsAPI(dst_iter) ||
2777-
NpyIter_IterationNeedsAPI(src_iter);
2778-
2779-
/*
2780-
* Because buffering is disabled in the iterator, the inner loop
2781-
* strides will be the same throughout the iteration loop. Thus,
2782-
* we can pass them to this function to take advantage of
2783-
* contiguous strides, etc.
2784-
*/
27852775
NPY_cast_info cast_info;
27862776
NPY_ARRAYMETHOD_FLAGS flags;
27872777
if (PyArray_GetDTypeTransferFunction(
@@ -2795,7 +2785,8 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order)
27952785
NpyIter_Deallocate(src_iter);
27962786
return -1;
27972787
}
2798-
needs_api |= (flags & NPY_METH_REQUIRES_PYAPI) != 0;
2788+
/* No need to worry about API use in unbuffered iterator */
2789+
int needs_api = (flags & NPY_METH_REQUIRES_PYAPI) != 0;
27992790
if (!(flags & NPY_METH_NO_FLOATINGPOINT_ERRORS)) {
28002791
npy_clear_floatstatus_barrier((char *)src_iter);
28012792
}

‎numpy/_core/src/multiarray/einsum.c.src

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -520,14 +520,16 @@ unbuffered_loop_nop1_ndim2(NpyIter *iter)
520520
return -1;
521521
}
522522

523-
/*
524-
* Since the iterator wasn't tracking coordinates, the
525-
* loop provided by the iterator is in Fortran-order.
526-
*/
523+
/* IterationNeedsAPI effectively only checks for object dtype here. */
527524
int needs_api = NpyIter_IterationNeedsAPI(iter);
528525
if (!needs_api) {
529526
NPY_BEGIN_THREADS_THRESHOLDED(shape[1] * shape[0]);
530527
}
528+
529+
/*
530+
* Since the iterator wasn't tracking coordinates, the
531+
* loop provided by the iterator is in Fortran-order.
532+
*/
531533
for (coord = shape[1]; coord > 0; --coord) {
532534
sop(1, ptrs[0], strides[0], shape[0]);
533535

@@ -581,14 +583,16 @@ unbuffered_loop_nop1_ndim3(NpyIter *iter)
581583
return -1;
582584
}
583585

584-
/*
585-
* Since the iterator wasn't tracking coordinates, the
586-
* loop provided by the iterator is in Fortran-order.
587-
*/
586+
/* IterationNeedsAPI effectively only checks for object dtype here. */
588587
int needs_api = NpyIter_IterationNeedsAPI(iter);
589588
if (!needs_api) {
590589
NPY_BEGIN_THREADS_THRESHOLDED(shape[2] * shape[1] * shape[0]);
591590
}
591+
592+
/*
593+
* Since the iterator wasn't tracking coordinates, the
594+
* loop provided by the iterator is in Fortran-order.
595+
*/
592596
for (coords[1] = shape[2]; coords[1] > 0; --coords[1]) {
593597
for (coords[0] = shape[1]; coords[0] > 0; --coords[0]) {
594598
sop(1, ptrs[0], strides[0], shape[0]);
@@ -645,14 +649,16 @@ unbuffered_loop_nop2_ndim2(NpyIter *iter)
645649
return -1;
646650
}
647651

648-
/*
649-
* Since the iterator wasn't tracking coordinates, the
650-
* loop provided by the iterator is in Fortran-order.
651-
*/
652+
/* IterationNeedsAPI effectively only checks for object dtype here. */
652653
int needs_api = NpyIter_IterationNeedsAPI(iter);
653654
if (!needs_api) {
654655
NPY_BEGIN_THREADS_THRESHOLDED(shape[1] * shape[0]);
655656
}
657+
658+
/*
659+
* Since the iterator wasn't tracking coordinates, the
660+
* loop provided by the iterator is in Fortran-order.
661+
*/
656662
for (coord = shape[1]; coord > 0; --coord) {
657663
sop(2, ptrs[0], strides[0], shape[0]);
658664

@@ -708,14 +714,16 @@ unbuffered_loop_nop2_ndim3(NpyIter *iter)
708714
return -1;
709715
}
710716

711-
/*
712-
* Since the iterator wasn't tracking coordinates, the
713-
* loop provided by the iterator is in Fortran-order.
714-
*/
717+
/* IterationNeedsAPI effectively only checks for object dtype here. */
715718
int needs_api = NpyIter_IterationNeedsAPI(iter);
716719
if (!needs_api) {
717720
NPY_BEGIN_THREADS_THRESHOLDED(shape[2] * shape[1] * shape[0]);
718721
}
722+
723+
/*
724+
* Since the iterator wasn't tracking coordinates, the
725+
* loop provided by the iterator is in Fortran-order.
726+
*/
719727
for (coords[1] = shape[2]; coords[1] > 0; --coords[1]) {
720728
for (coords[0] = shape[1]; coords[0] > 0; --coords[0]) {
721729
sop(2, ptrs[0], strides[0], shape[0]);
@@ -1120,7 +1128,6 @@ PyArray_EinsteinSum(char *subscripts, npy_intp nop,
11201128
char **dataptr;
11211129
npy_intp *stride;
11221130
npy_intp *countptr;
1123-
int needs_api;
11241131
NPY_BEGIN_THREADS_DEF;
11251132

11261133
iternext = NpyIter_GetIterNext(iter, NULL);
@@ -1130,17 +1137,20 @@ PyArray_EinsteinSum(char *subscripts, npy_intp nop,
11301137
dataptr = NpyIter_GetDataPtrArray(iter);
11311138
stride = NpyIter_GetInnerStrideArray(iter);
11321139
countptr = NpyIter_GetInnerLoopSizePtr(iter);
1133-
needs_api = NpyIter_IterationNeedsAPI(iter);
1140+
/* IterationNeedsAPI additionally checks for object dtype here. */
1141+
int needs_api = NpyIter_IterationNeedsAPI(iter);
1142+
if (!needs_api) {
1143+
NPY_BEGIN_THREADS_THRESHOLDED(NpyIter_GetIterSize(iter));
1144+
}
11341145

1135-
NPY_BEGIN_THREADS_NDITER(iter);
11361146
NPY_EINSUM_DBG_PRINT("Einsum loop\n");
11371147
do {
11381148
sop(nop, dataptr, stride, *countptr);
11391149
} while (!(needs_api && PyErr_Occurred()) && iternext(iter));
11401150
NPY_END_THREADS;
11411151

11421152
/* If the API was needed, it may have thrown an error */
1143-
if (NpyIter_IterationNeedsAPI(iter) && PyErr_Occurred()) {
1153+
if (needs_api && PyErr_Occurred()) {
11441154
goto fail;
11451155
}
11461156
}

‎numpy/_core/src/multiarray/item_selection.c

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2752,6 +2752,7 @@ PyArray_CountNonzero(PyArrayObject *self)
27522752
if (iter == NULL) {
27532753
return -1;
27542754
}
2755+
/* IterationNeedsAPI also checks dtype for whether `nonzero` may need it */
27552756
needs_api = NpyIter_IterationNeedsAPI(iter);
27562757

27572758
/* Get the pointers for inner loop iteration */
@@ -2761,7 +2762,9 @@ PyArray_CountNonzero(PyArrayObject *self)
27612762
return -1;
27622763
}
27632764

2764-
NPY_BEGIN_THREADS_NDITER(iter);
2765+
if (!needs_api) {
2766+
NPY_BEGIN_THREADS_THRESHOLDED(NpyIter_GetIterSize(iter));
2767+
}
27652768

27662769
dataptr = NpyIter_GetDataPtrArray(iter);
27672770
strideptr = NpyIter_GetInnerStrideArray(iter);
@@ -2982,9 +2985,12 @@ PyArray_Nonzero(PyArrayObject *self)
29822985
return NULL;
29832986
}
29842987

2988+
/* IterationNeedsAPI also checks dtype for whether `nonzero` may need it */
29852989
needs_api = NpyIter_IterationNeedsAPI(iter);
29862990

2987-
NPY_BEGIN_THREADS_NDITER(iter);
2991+
if (!needs_api) {
2992+
NPY_BEGIN_THREADS_THRESHOLDED(NpyIter_GetIterSize(iter));
2993+
}
29882994

29892995
dataptr = NpyIter_GetDataPtrArray(iter);
29902996

‎numpy/_core/src/multiarray/mapping.c

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -976,10 +976,7 @@ array_boolean_subscript(PyArrayObject *self,
976976
/* Get a dtype transfer function */
977977
NpyIter_GetInnerFixedStrideArray(iter, fixed_strides);
978978
NPY_cast_info cast_info;
979-
/*
980-
* TODO: Ignoring cast flags, since this is only ever a copy. In
981-
* principle that may not be quite right in some future?
982-
*/
979+
983980
NPY_ARRAYMETHOD_FLAGS cast_flags;
984981
if (PyArray_GetDTypeTransferFunction(
985982
IsUintAligned(self) && IsAligned(self),
@@ -992,6 +989,8 @@ array_boolean_subscript(PyArrayObject *self,
992989
NpyIter_Deallocate(iter);
993990
return NULL;
994991
}
992+
cast_flags = PyArrayMethod_COMBINED_FLAGS(
993+
cast_flags, NpyIter_GetTransferFlags(iter));
995994

996995
/* Get the values needed for the inner loop */
997996
iternext = NpyIter_GetIterNext(iter, NULL);
@@ -1002,7 +1001,10 @@ array_boolean_subscript(PyArrayObject *self,
10021001
return NULL;
10031002
}
10041003

1005-
NPY_BEGIN_THREADS_NDITER(iter);
1004+
/* NOTE: Don't worry about floating point errors as this is a copy. */
1005+
if (!(cast_flags & NPY_METH_REQUIRES_PYAPI)) {
1006+
NPY_BEGIN_THREADS_THRESHOLDED(NpyIter_GetIterSize(iter));
1007+
}
10061008

10071009
innerstrides = NpyIter_GetInnerStrideArray(iter);
10081010
dataptrs = NpyIter_GetDataPtrArray(iter);
@@ -1195,8 +1197,11 @@ array_assign_boolean_subscript(PyArrayObject *self,
11951197
return -1;
11961198
}
11971199

1200+
cast_flags = PyArrayMethod_COMBINED_FLAGS(
1201+
cast_flags, NpyIter_GetTransferFlags(iter));
1202+
11981203
if (!(cast_flags & NPY_METH_REQUIRES_PYAPI)) {
1199-
NPY_BEGIN_THREADS_NDITER(iter);
1204+
NPY_BEGIN_THREADS_THRESHOLDED(NpyIter_GetIterSize(iter));
12001205
}
12011206
if (!(flags & NPY_METH_NO_FLOATINGPOINT_ERRORS)) {
12021207
npy_clear_floatstatus_barrier((char *)self);
@@ -2662,7 +2667,9 @@ PyArray_MapIterCheckIndices(PyArrayMapIterObject *mit)
26622667
return -1;
26632668
}
26642669

2665-
NPY_BEGIN_THREADS_NDITER(op_iter);
2670+
if (!(NpyIter_GetTransferFlags(op_iter) & NPY_METH_REQUIRES_PYAPI)) {
2671+
NPY_BEGIN_THREADS_THRESHOLDED(NpyIter_GetIterSize(op_iter));
2672+
}
26662673
iterptr = NpyIter_GetDataPtrArray(op_iter);
26672674
iterstride = NpyIter_GetInnerStrideArray(op_iter);
26682675
do {

0 commit comments

Comments
 (0)
0