10000 ENH: create and use indexed inner loops by mattip · Pull Request #23136 · numpy/numpy · GitHub
[go: up one dir, main page]
Skip to content

ENH: create and use indexed inner loops #23136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 20 commits into from
Feb 7, 2023
Merged

ENH: create and use indexed inner loops #23136

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
90243f5
ENH: add indexed loops
mattip Jan 18, 2023
3ba259d
refactor ufunc_at to first get the inner loop, then create the iterators
mattip Jan 22, 2023
7910e2b
ENH: add indexed lower loops and connect them to ufuncs
mattip Jan 23, 2023
eb21b25
ENH: use an indexed loop if possible in ufunc_at
mattip Jan 30, 2023
888bba2
MAINT: fix linting
mattip Jan 31, 2023
be64173
ENH: update and sync the public API
mattip Jan 31, 2023
8bfaaf3
BENCH: fix benchmark
mattip Jan 31, 2023
28706af
MAINT, BUG: fixes from review and testing
mattip Feb 1, 2023
1db1ffe
MAINT: linting
mattip Feb 1, 2023
ec1a434
MAINT: update EXPERIMENTAL_DTYPE_API_VERSION too
mattip Feb 1, 2023
6541cf5
DOC: add a performance release note
mattip Feb 1, 2023
3802b16
MAINT: rework release note, changes from review
mattip Feb 5, 2023
f064552
BUG: Clean up reference handling in `ufunc.at` a little.
seberg Feb 6, 2023
6cf9cc9
BUG: Deal with casting and non-contig/1-D indices
seberg Feb 6, 2023
ff97723
BUG: fixes from review: use iter->nd and adapt to unary ufuncs
mattip Feb 6, 2023
163d3c5
BUG: improve check for 1d operands (from review)
mattip Feb 7, 2023
f79325e
TST: add test (from review)
mattip Feb 7, 2023
b80bc2d
ENH: fixes from review
mattip Feb 7, 2023
26978d7
DOC: improve release snippet (from review)
mattip Feb 7, 2023
790426e
TST, BUG: add a test for unary ufuncs, fix condition for indexed loops
mattip Feb 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
ENH: use an indexed loop if possible in ufunc_at
  • Loading branch information
@mattip
mattip committed Jan 30, 2023
commit eb21b25093c79ac0bc135aed3d192c06079c5bbf
4 changes: 2 additions & 2 deletions numpy/core/src/umath/dispatching.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -914,8 +914,8 @@ promote_and_get_ufuncimpl(PyUFuncObject *ufunc,
int nin = ufunc->nin, nargs = ufunc->nargs;

/*
* Get the actual DTypes we operate with by mixing the operand array
* ones with the passed signature.
* Get the actual DTypes we operate with by setting op_dtypes[i] from
* signature[i].
*/
for (int i = 0; i < nargs; i++) {
if (signature[i] != NULL) {
Expand Down
6 changes: 3 additions & 3 deletions numpy/core/src/umath/loops.c.src
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -547,7 +547,7 @@ NPY_NO_EXPORT NPY_GCC_OPT_3 @ATTR@ int
char *ip1 = args[0];
npy_intp *indx = (npy_intp *)args[1];
char *value = args[2];
npy_intp is1 = steps[0], is2 = steps[1], os1 = steps[2];
npy_intp is1 = steps[0], is2 = steps[1] / sizeof(npy_intp), os1 = steps[2];
npy_intp n = dimensions[0];
npy_intp i;
@type@ *indexed;
Expand Down Expand Up @@ -1559,7 +1559,7 @@ LONGDOUBLE_@kind@_indexed(char **args, npy_intp const *dimensions, npy_intp cons
char *ip1 = args[0];
npy_intp *indx = (npy_intp *)args[1];
char *value = args[2];
npy_intp is1 = steps[0], is2 = steps[1], os1 = steps[2];
npy_intp is1 = steps[0], is2 = steps[1] / sizeof(npy_intp), os1 = steps[2];
npy_intp n = dimensions[0];
npy_intp i;
npy_longdouble *indexed;
Expand Down Expand Up @@ -1681,7 +1681,7 @@ HALF_@kind@_indexed(void *NPY_UNUSED(context), char **args, npy_intp const *dime
char *ip1 = args[0];
npy_intp *indx = (npy_intp *)args[1];
char *value = args[2];
npy_intp is1 = steps[0], is2 = steps[1], os1 = steps[2];
npy_intp is1 = steps[0], is2 = steps[1] / sizeof(npy_intp), os1 = steps[2];
npy_intp n = dimensions[0];
npy_intp i;
npy_half *indexed;
Expand Down
2 changes: 1 addition & 1 deletion numpy/core/src/umath/loops_arithm_fp.dispatch.c.src
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(@TYPE@_@kind@_indexed)
char *ip1 = args[0];
npy_intp *indx = (npy_intp *)args[1];
char *value = args[2];
npy_intp is1 = steps[0], is2 = steps[1], os1 = steps[2];
npy_intp is1 = steps[0], is2 = steps[1] / sizeof(npy_intp), os1 = steps[2];
npy_intp n = dimensions[0];
npy_intp i;
@type@ *indexed;
Expand Down
4 changes: 2 additions & 2 deletions numpy/core/src/umath/loops_arithmetic.dispatch.c.src
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(@TYPE@_divide_indexed)
char *ip1 = args[0];
npy_intp *indx = (npy_intp *)args[1];
char *value = args[2];
npy_intp is1 = steps[0], is2 = steps[1], os1 = steps[2];
npy_intp is1 = steps[0], is2 = steps[1] / sizeof(npy_intp), os1 = steps[2];
npy_intp n = dimensions[0];
npy_intp i;
@type@ *indexed;
Expand Down Expand Up @@ -488,7 +488,7 @@ NPY_NO_EXPORT int NPY_CPU_DISPATCH_CURFX(@TYPE@_divide_indexed)
char *ip1 = args[0];
npy_intp *indx = (npy_intp *)args[1];
char *value = args[2];
npy_intp is1 = steps[0], is2 = steps[1], os1 = steps[2];
npy_intp is1 = steps[0], is2 = steps[1] / sizeof(npy_intp), os1 = steps[2];
npy_intp n = dimensions[0];
npy_intp i;
@type@ *indexed;
Expand Down
2 changes: 1 addition & 1 deletion numpy/core/src/umath/scalarmath.c.src
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,7 @@ static inline int
@name@_ctype_divide(@type@ a, @type@ b, @type@ *out)
{
char *args[3] = {(char *)&a, (char *)&b, (char *)out};
npy_intp steps[3];
npy_intp steps[3] = {0, 0, 0};
npy_intp size = 1;
@TYPE@_divide(args, &size, steps, NULL);
return 0;
Expand Down
61 changes: 54 additions & 7 deletions numpy/core/src/umath/ufunc_object.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5893,13 +5893,45 @@ new_array_op(PyArrayObject *op_array, char *data)
return (PyArrayObject *)r;
}

/*
* If operands are 1D we can use the indexed loop to do the work
* Returns 0 if successful, -1 otherwise
*/
static int
try_trivial_at_loop(PyArrayMethodObject *ufuncimpl, NPY_ARRAYMETHOD_FLAGS flags,
PyArrayMapIterObject *iter,
PyArrayObject *op1_array, PyArrayObject *op2_array,
PyArrayMethod_Context *context)
{
if (PyArray_NDIM(op1_array) != 1) {
return -1;
}
/* check that nop == 3 */
if (ufuncimpl->nin >1 && PyArray_NDIM(op2_array) != 1) {
return -1;
}
if (iter->numiter > 1) {
return -1;
}
char *args[3];
npy_intp dimensions = iter->size;
npy_intp steps[3];
args[0] = (char *) iter->baseoffset;
args[1] = (char *) iter->outer_ptrs[0];
args[2] = (char *)PyArray_DATA(op2_array);
steps[0] = iter->fancy_strides[0];
steps[1] = iter->outer_strides[0];
steps[2] = PyArray_STRIDE(op2_array, 0);
ufuncimpl->contiguous_indexed_loop(context, args, &dimensions, steps, NULL);
return 0;
}

static int
ufunc_at__fast_iter(PyUFuncObject *ufunc, NPY_ARRAYMETHOD_FLAGS flags,
PyArrayMapIterObject *iter, PyArrayIterObject *iter2,
PyArrayObject *op1_array, PyArrayObject *op2_array,
PyArrayMethod_StridedLoop *strided_loop,
PyArrayMethod_Context *context,
npy_intp strides[3],
NpyAuxData *auxdata
)
{
Expand All @@ -5921,6 +5953,7 @@ ufunc_at__fast_iter(PyUFuncObject *ufunc, NPY_ARRAYMETHOD_FLAGS flags,
NPY_BEGIN_THREADS;
}

npy_intp strides[3] = {0, 0, 0};
/*
* Iterate over first and second operands and call ufunc
* for each pair of inputs
Expand Down Expand Up @@ -5972,7 +6005,6 @@ ufunc_at__slow_iter(PyUFuncObject *ufunc, NPY_ARRAYMETHOD_FLAGS flags,
PyArray_Descr *operation_descrs[3],
PyArrayMethod_StridedLoop *strided_loop,
PyArrayMethod_Context *context,
npy_intp strides[3],
NpyAuxData *auxdata
)
{
Expand Down Expand Up @@ -6070,6 +6102,7 @@ ufunc_at__slow_iter(PyUFuncObject *ufunc, NPY_ARRAYMETHOD_FLAGS flags,
npy_clear_floatstatus_barrier((char*)&iter);
}

npy_intp strides[3] = {0, 0, 0};
if (!needs_api) {
NPY_BEGIN_THREADS;
}
Expand Down Expand Up @@ -6305,7 +6338,7 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args)
}
}

/*
/*
* Create a map iterator (there is no multi-mapiter, so we need at least 2
* iterators: one for the mapping, and another for the second operand)
*/
Expand Down Expand Up @@ -6383,11 +6416,25 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args)
}
}
if (fast_path == 1) {
res = ufunc_at__fast_iter(ufunc, flags, iter, iter2, op1_array, op2_array,
strided_loop, &context, strides, auxdata);
/*
* Try to use trivial loop (1d, no casting, aligned) if
* - the matching info has a indexed loop
* - all operands are 1d
*/
if (ufuncimpl->contiguous_indexed_loop != NULL) {
res = try_trivial_at_loop(ufuncimpl, flags, iter, op1_array,
op2_array, &context);

}
if (res == -1) {
/* Couldn't use the fastest path, use the faster path */
res = ufunc_at__fast_iter(ufunc, flags, iter, iter2, op1_array,
op2_array, strided_loop, &context, auxdata);
}
} else {
res = ufunc_at__slow_iter(ufunc, flags, iter, iter2, op1_array, op2_array,
operation_descrs, strided_loop, &context, strides, auxdata);
res = ufunc_at__slow_iter(ufunc, flags, iter, iter2, op1_array,
op2_array, operation_descrs, strided_loop, &context,
auxdata);
}

fail:
Expand Down
0