8000 Pull request #15: Reminder of np.array · mattip/numpy@4acf78a · GitHub
[go: up one dir, main page]
Skip to content

Commit 4acf78a

Browse files
steve-ssteve-s
committed
Pull request #15: Reminder of np.array
Merge in numpy-hpy from ss/array_array3 to labs-hpy-port * commit 'a197852c5099cec81ed8cdfde0a59a29316db2a3': hpy_raw_array_assign_scalar HPyArray_AssignRawScalar and HPyArray_EquivTypes
2 parents 620a54e + a197852 commit 4acf78a

File tree

9 files changed

+342
-11
lines changed

9 files changed

+342
-11
lines changed

‎numpy/core/include/numpy/ndarrayobject.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ HPyArray_CheckExact(HPyContext *ctx, HPy op)
100100
PyArray_NDIM(a1)))
101101

102102
#define PyArray_SIZE(m) PyArray_MultiplyList(PyArray_DIMS(m), PyArray_NDIM(m))
103+
#define HPyArray_SIZE(m) PyArray_MultiplyList(PyArray_DIMS(m), PyArray_NDIM(m))
103104
#define PyArray_NBYTES(m) (PyArray_ITEMSIZE(m) * PyArray_SIZE(m))
104105
#define PyArray_FROM_O(m) PyArray_FromAny(m, NULL, 0, 0, 0, NULL)
105106

‎numpy/core/include/numpy/ndarraytypes.h

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1569,15 +1569,18 @@ PyArrayNeighborhoodIter_Next2D(PyArrayNeighborhoodIterObject* iter);
15691569
NPY_ARRAY_F_CONTIGUOUS : 0))
15701570

15711571
static NPY_INLINE HPy
1572-
HPyArray_GetDescr(HPyContext *ctx, HPy arr)
1572+
HPyArray_DESCR(HPyContext *ctx, HPy arr, const PyArrayObject *arr_data)
15731573
{
1574-
return HPyField_Load(ctx, arr, HPyArray_AsFields(ctx, arr)->f_descr);
1574+
if (HPyField_IsNull(((PyArrayObject_fields *)arr_data)->f_descr)) {
1575+
return HPy_NULL;
1576+
}
1577+
return HPyField_Load(ctx, arr, ((PyArrayObject_fields *)arr_data)->f_descr);
15751578
}
15761579

15771580
static NPY_INLINE HPy
1578-
HPyArray_DESCR(HPyContext *ctx, HPy arr, const PyArrayObject *arr_data)
1581+
HPyArray_GetDescr(HPyContext *ctx, HPy arr)
15791582
{
1580-
return HPyField_Load(ctx, arr, ((PyArrayObject_fields *)arr_data)->f_descr);
1583+
return HPyArray_DESCR(ctx, arr, PyArrayObject_AsStruct(ctx, arr));
15811584
}
15821585

15831586
static NPY_INLINE NPY_RETURNS_BORROWED_REF PyArray_Descr *

‎numpy/core/src/common/array_assign.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ PyArray_AssignRawScalar(PyArrayObject *dst,
4242
PyArrayObject *wheremask,
4343
NPY_CASTING casting);
4444

45+
NPY_NO_EXPORT int
46+
HPyArray_AssignRawScalar(HPyContext *ctx, HPy h_dst,
47+
HPy h_src_dtype, char *src_data,
48+
HPy h_wheremask,
49+
NPY_CASTING casting);
50+
4551
/******** LOW-LEVEL SCALAR TO ARRAY ASSIGNMENT ********/
4652

4753
/*

‎numpy/core/src/multiarray/array_assign_array.c

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -440,8 +440,8 @@ PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src,
440440
}
441441

442442
NPY_NO_EXPORT int
443-
HPyArray_AssignArray(HPyContext *ctx, HPy h_dst, HPy h_src,
444-
HPy h_wheremask,
443+
HPyArray_AssignArray(HPyContext *ctx, /*PyArrayObject*/HPy h_dst, /*PyArrayObject*/HPy h_src,
444+
/*PyArrayObject*/HPy h_wheremask,
445445
NPY_CASTING casting)
446446
{
447447
PyArrayObject *src = PyArrayObject_AsStruct(ctx, h_src);
@@ -452,10 +452,9 @@ HPyArray_AssignArray(HPyContext *ctx, HPy h_dst, HPy h_src,
452452

453453
/* Use array_assign_scalar if 'src' NDIM is 0 */
454454
if (PyArray_NDIM(src) == 0) {
455-
capi_warn("HPyArray_AssignArray: PyArray_AssignRawScalar");
456-
return PyArray_AssignRawScalar(
457-
dst, PyArray_DESCR(src), PyArray_DATA(src),
458-
PyArrayObject_AsStruct(ctx, h_wheremask), casting);
455+
return HPyArray_AssignRawScalar(ctx,
456+
h_dst, HPyArray_DESCR(ctx, h_src, src), PyArray_DATA(src),
457+
h_wheremask, casting);
459458
}
460459

461460
HPy h_src_descr = HPyArray_DESCR(ctx, h_src, src);
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@
2525
#include "array_assign.h"
2626
#include "dtype_transfer.h"
2727

28+
// HPy includes
29+
#include "multiarraymodule.h"
30+
#include "arrayobject.h"
31+
#include "convert_datatype.h"
32+
2833
/*
2934
* Assigns the scalar value to every element of the destination raw array.
3035
*
@@ -99,6 +104,75 @@ raw_array_assign_scalar(int ndim, npy_intp const *shape,
99104
return -1;
100105
}
101106

107+
NPY_NO_EXPORT int
108+
hpy_raw_array_assign_scalar(HPyContext *ctx, int ndim, npy_intp const *shape,
109+
HPy h_dst_dtype, PyArray_Descr *dst_dtype, char *dst_data, npy_intp const *dst_strides,
110+
HPy h_src_dtype, PyArray_Descr *src_dtype, char *src_data)
111+
{
112+
int idim;
113+
npy_intp shape_it[NPY_MAXDIMS], dst_strides_it[NPY_MAXDIMS];
114+
npy_intp coord[NPY_MAXDIMS];
115+
116+
int aligned, needs_api = 0;
117+
118+
NPY_BEGIN_THREADS_DEF;
119+
120+
/* Check both uint and true alignment */
121+
aligned = raw_array_is_aligned(ndim, shape, dst_data, dst_strides,
122+
npy_uint_alignment(dst_dtype->elsize)) &&
123+
raw_array_is_aligned(ndim, shape, dst_data, dst_strides,
124+
dst_dtype->alignment) &&
125+
npy_is_aligned(src_data, npy_uint_alignment(src_dtype->elsize) &&
126+
npy_is_aligned(src_data, src_dtype->alignment));
127+
128+
/* Use raw iteration with no heap allocation */
129+
if (PyArray_PrepareOneRawArrayIter(
130+
ndim, shape,
131+
dst_data, dst_strides,
132+
&ndim, shape_it,
133+
&dst_data, dst_strides_it) < 0) {
134+
return -1;
135+
}
136+
137+
/* Get the function to do the casting */
138+
NPY_cast_info cast_info;
139+
if (HPyArray_GetDTypeTransferFunction(ctx, aligned,
140+
0, dst_strides_it[0],
141+
h_src_dtype, h_dst_dtype,
142+
0,
143+
&cast_info, &needs_api) != NPY_SUCCEED) {
144+
return -1;
145+
}
146+
147+
if (!needs_api) {
148+
npy_intp nitems = 1, i;
149+
for (i = 0; i < ndim; i++) {
150+
nitems *= shape_it[i];
151+
}
152+
NPY_BEGIN_THREADS_THRESHOLDED(nitems);
153+
}
154+
155+
npy_intp strides[2] = {0, dst_strides_it[0]};
156+
157+
NPY_RAW_ITER_START(idim, ndim, coord, shape_it) {
158+
/* Process the innermost dimension */
159+
char *args[2] = {src_data, dst_data};
160+
if (cast_info.func(&cast_info.context,
161+
args, &shape_it[0], strides, cast_info.auxdata) < 0) {
162+
goto fail;
163+
}
164+
} NPY_RAW_ITER_ONE_NEXT(idim, ndim, coord,
165+
shape_it, dst_data, dst_strides_it);
166+
167+
NPY_END_THREADS;
168+
NPY_cast_info_xfree(&cast_info);
169+
return 0;
170+
fail:
171+
NPY_END_THREADS;
172+
NPY_cast_info_xfree(&cast_info);
173+
return -1;
174+
}
175+
102176
/*
103177
* Assigns the scalar value to every element of the destination raw array
104178
* where the 'wheremask' value is True.
@@ -305,3 +379,123 @@ PyArray_AssignRawScalar(PyArrayObject *dst,
305379

306380
return -1;
307381
}
382+
383+
NPY_NO_EXPORT int
384+
HPyArray_AssignRawScalar(HPyContext *ctx, /*PyArrayObject*/HPy h_dst,
385+
/*PyArray_Descr*/HPy h_src_dtype, char *src_data,
386+
/*PyArrayObject*/HPy h_wheremask,
387+
NPY_CASTING casting)
388+
{
389+
int allocated_src_data = 0;
390+
npy_longlong scalarbuffer[4];
391+
392+
if (HPyArray_FailUnlessWriteable(ctx, h_dst, "assignment destination") < 0) {
393+
return -1;
394+
}
395+
396+
PyArray_Descr *src_dtype = PyArray_Descr_AsStruct(ctx, h_src_dtype);
397+
HPy h_dst_dtype = HPyArray_GetDescr(ctx, h_dst);
398+
PyArray_Descr *dst_dtype = PyArray_Descr_AsStruct(ctx, h_dst_dtype);
399+
400+
/* Check the casting rule */
401+
if (!hpy_can_cast_scalar_to(ctx, h_src_dtype, src_data, h_dst_dtype, casting)) {
402+
CAPI_WARN("npy_set_invalid_cast_error");
403+
npy_set_invalid_cast_error(
404+
src_dtype, dst_dtype, casting, NPY_TRUE);
405+
return -1;
406+
}
407+
408+
/*
409+
* Make a copy of the src data if it's a different dtype than 'dst'
410+
* or isn't aligned, and the destination we're copying to has
411+
* more than one element. To avoid having to manage object lifetimes,
412+
* we also skip this if 'dst' has an object dtype.
413+
*/
414+
PyArrayObject *dst = PyArrayObject_AsStruct(ctx, h_dst);
415+
if ((!HPyArray_EquivTypes(ctx, h_dst_dtype, h_src_dtype) ||
416+
!(npy_is_aligned(src_data, npy_uint_alignment(src_dtype->elsize)) &&
417+
npy_is_aligned(src_data, src_dtype->alignment))) &&
418+
HPyArray_SIZE(dst) > 1 &&
419+
!HPyDataType_REFCHK(ctx, h_dst_dtype)) {
420+
char *tmp_src_data;
421+
422+
/*
423+
* Use a static buffer to store the aligned/cast version,
424+
* or allocate some memory if more space is needed.
425+
*/
426+
PyArray_Descr *dst_dtype = PyArray_Descr_AsStruct(ctx, h_dst_dtype);
427+
if ((int)sizeof(scalarbuffer) >= dst_dtype->elsize) {
428+
tmp_src_data = (char *)&scalarbuffer[0];
429+
}
430+
else {
431+
CAPI_WARN("PyArray_malloc");
432+
tmp_src_data = PyArray_malloc(dst_dtype->elsize);
433+
if (tmp_src_data == NULL) {
434+
HPyErr_NoMemory(ctx);
435+
goto fail;
436+
}
437+
allocated_src_data = 1;
438+
}
439+
440+
if (PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT)) {
441+
memset(tmp_src_data, 0, dst_dtype->elsize);
442+
}
443+
444+
CAPI_WARN("PyArray_CastRawArrays");
445+
if (PyArray_CastRawArrays(1, src_data, tmp_src_data, 0, 0,
446+
src_dtype, PyArray_DESCR(dst), 0) != NPY_SUCCEED) {
447+
src_data = tmp_src_data;
448+
goto fail;
449+
}
450+
451+
/* Replace src_data/src_dtype */
452+
src_data = tmp_src_data;
453+
src_dtype = PyArray_DESCR(dst);
454+
}
455+
456+
PyArrayObject *wheremask = PyArrayObject_AsStruct(ctx, h_wheremask);
457+
if (HPy_IsNull(h_wheremask)) {
458+
/* A straightforward value assignment */
459+
/* Do the assignment with raw array iteration */
460+
if (hpy_raw_array_assign_scalar(ctx, PyArray_NDIM(dst), PyArray_DIMS(dst),
461+
h_dst_dtype, dst_dtype, PyArray_DATA(dst), PyArray_STRIDES(dst),
462+
h_src_dtype, src_dtype, src_data) < 0) {
463+
goto fail;
464+
}
465+
}
466+
else {
467+
CAPI_WARN("non-straightforward value assignment");
468+
npy_intp wheremask_strides[NPY_MAXDIMS];
469+
470+
/* Broadcast the wheremask to 'dst' for raw iteration */
471+
if (broadcast_strides(PyArray_NDIM(dst), PyArray_DIMS(dst),
472+
PyArray_NDIM(wheremask), PyArray_DIMS(wheremask),
473+
PyArray_STRIDES(wheremask), "where mask",
474+
wheremask_strides) < 0) {
475+
goto fail;
476+
}
477+
478+
/* Do the masked assignment with raw array iteration */
479+
if (raw_array_wheremasked_assign_scalar(
480+
PyArray_NDIM(dst), PyArray_DIMS(dst),
481+
PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst),
482+
src_dtype, src_data,
483+
PyArray_DESCR(wheremask), PyArray_DATA(wheremask),
484+
wheremask_strides) < 0) {
485+
goto fail;
486+
}
487+
}
488+
489+
if (allocated_src_data) {
490+
PyArray_free(src_data);
491+
}
492+
493+
return 0;
494+
495+
fail:
496+
if (allocated_src_data) {
497+
PyArray_free(src_data);
498+
}
499+
500+
return -1;
501+
}
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@
3131
#include "usertypes.h"
3232
#include "dtype_transfer.h"
3333

34+
// added by HPy porting:
35+
#include "arraytypes.h"
36+
3437

3538
/*
3639
* Required length of string when converting from unsigned integer type.
@@ -745,7 +748,7 @@ HPyArray_CheckCastSafety(HPyContext *ctx, NPY_CASTING casting,
745748
HPy_Close(ctx, meth);
746749
return -1;
747750
}
748-
751+
749752
PyArrayMethodObject *castingimpl = PyArrayMethodObject_AsStruct(ctx, meth);
750753
if (PyArray_MinCastSafety(castingimpl->casting, casting) == casting) {
751754
/* No need to check using `castingimpl.resolve_descriptors()` */
@@ -1053,6 +1056,84 @@ can_cast_scalar_to(PyArray_Descr *scal_type, char *scal_data,
10531056
return ret;
10541057
}
10551058

1059+
/*
1060+
* NOTE: This function uses value based casting logic for scalars. It will
1061+
* require updates when we phase out value-based-casting.
1062+
*/
1063+
NPY_NO_EXPORT npy_bool
1064+
hpy_can_cast_scalar_to(HPyContext *ctx, HPy scal_type, char *scal_data,
1065+
HPy to, NPY_CASTING casting)
1066+
{
1067+
/*
1068+
* If the two dtypes are actually references to the same object
1069+
* or if casting type is forced unsafe then always OK.
1070+
*
1071+
* TODO: Assuming that unsafe casting always works is not actually correct
1072+
*/
1073+
if (HPy_Is(ctx, scal_type, to) || casting == NPY_UNSAFE_CASTING ) {
1074+
return 1;
1075+
}
1076+
1077+
int valid = HPyArray_CheckCastSafety(ctx, casting, scal_type, to, HPyArray_DTYPE(ctx, to));
1078+
if (valid == 1) {
1079+
/* This is definitely a valid cast. */
1080+
return 1;
1081+
}
1082+
if (valid < 0) {
1083+
/* Probably must return 0, but just keep trying for now. */
1084+
HPyErr_Clear(ctx);
1085+
}
1086+
1087+
PyArray_Descr *scal_type_data = PyArray_Descr_AsStruct(ctx, scal_type);
1088+
/*
1089+
* If the scalar isn't a number, value-based casting cannot kick in and
1090+
* we must not attempt it.
1091+
* (Additional fast-checks would be possible, but probably unnecessary.)
1092+
*/
1093+
if (!PyTypeNum_ISNUMBER(scal_type_data->type_num)) {
1094+
return 0;
1095+
}
1096+
1097+
/*
1098+
* At this point we have to check value-based casting.
1099+
*/
1100+
int is_small_unsigned = 0, type_num;
1101+
/* An aligned memory buffer large enough to hold any builtin numeric type */
1102+
npy_longlong value[4];
1103+
1104+
int swap = !PyArray_ISNBO(scal_type_data->byteorder);
1105+
CAPI_WARN("Not clear what scal_type_data->f->copyswap may call...");
1106+
scal_type_data->f->copyswap(&value, scal_data, swap, NULL);
1107+
1108+
type_num = min_scalar_type_num((char *)&value, scal_type_data->type_num,
1109+
&is_small_unsigned);
1110+
1111+
/*
1112+
* If we've got a small unsigned scalar, and the 'to' type
1113+
* is not unsigned, then make it signed to allow the value
1114+
* to be cast more appropriately.
1115+
*/
1116+
PyArray_Descr *to_data = PyArray_Descr_AsStruct(ctx, to);
1117+
if (is_small_unsigned && !(PyTypeNum_ISUNSIGNED(to_data->type_num))) {
1118+
type_num = type_num_unsigned_to_signed(type_num);
1119+
}
1120+
1121+
HPy dtype = HPyArray_DescrFromType(ctx, type_num);
1122+
if (HPy_IsNull(dtype)) {
1123+
return 0;
1124+
}
1125+
#if 0
1126+
printf("min scalar cast ");
1127+
PyObject_Print(dtype, stdout, 0);
1128+
printf(" to ");
1129+
PyObject_Print(to, stdout, 0);
1130+
printf("\n");
1131+
#endif
1132+
npy_bool ret = HPyArray_CanCastTypeTo(ctx, dtype, to, casting);
1133+
HPy_Close(ctx, dtype);
1134+
return ret;
1135+
}
1136+
10561137
/*NUMPY_API
10571138
* Returns 1 if the array object may be cast to the given data type using
10581139
* the casting rule, 0 otherwise. This differs from PyArray_CanCastTo in
Original file line numberDiff line numberDiff line change
@@ -118,4 +118,8 @@ NPY_NO_EXPORT npy_bool
118118
HPyArray_CanCastTypeTo(HPyContext *ctx, HPy h_from, HPy h_to,
119119
NPY_CASTING casting);
120120

121+
NPY_NO_EXPORT npy_bool
122+
hpy_can_cast_scalar_to(HPyContext *ctx, HPy scal_type, char *scal_data,
123+
HPy to, NPY_CASTING casting);
124+
121125
#endif /* NUMPY_CORE_SRC_MULTIARRAY_CONVERT_DATATYPE_H_ */