From b0861c76e1e8ccd7524d0cdde73560023821c987 Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Sun, 4 Mar 2012 09:46:02 -0800 Subject: [PATCH 1/5] STY: cleanup hashdesc.c to follow our C conventions. --- numpy/core/src/multiarray/hashdescr.c | 36 ++++++++++++++++----------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/numpy/core/src/multiarray/hashdescr.c b/numpy/core/src/multiarray/hashdescr.c index a5ca2d3c5535..d9b09868a118 100644 --- a/numpy/core/src/multiarray/hashdescr.c +++ b/numpy/core/src/multiarray/hashdescr.c @@ -36,16 +36,17 @@ static int _array_descr_builtin(PyArray_Descr* descr, PyObject *l); */ static char _normalize_byteorder(char byteorder) { - switch(byteorder) { - case '=': - if (PyArray_GetEndianness() == NPY_CPU_BIG) { - return '>'; - } else { - return '<'; - } - default: - return byteorder; - } + switch(byteorder) { + case '=': + if (PyArray_GetEndianness() == NPY_CPU_BIG) { + return '>'; + } + else { + return '<'; + } + default: + return byteorder; + } } /* @@ -139,7 +140,8 @@ static int _array_descr_walk_fields(PyObject* fields, PyObject* l) PyErr_SetString(PyExc_SystemError, "(Hash) First item in compound dtype tuple not a descr ???"); return -1; - } else { + } + else { Py_INCREF(fdescr); st = _array_descr_walk((PyArray_Descr*)fdescr, l); Py_DECREF(fdescr); @@ -153,7 +155,8 @@ static int _array_descr_walk_fields(PyObject* fields, PyObject* l) PyErr_SetString(PyExc_SystemError, "(Hash) Second item in compound dtype tuple not an int ???"); return -1; - } else { + } + else { Py_INCREF(foffset); PyList_Append(l, foffset); } @@ -187,10 +190,12 @@ static int _array_descr_walk_subarray(PyArray_ArrayDescr* adescr, PyObject *l) Py_INCREF(item); PyList_Append(l, item); } - } else if (PyInt_Check(adescr->shape)) { + } + else if (PyInt_Check(adescr->shape)) { Py_INCREF(adescr->shape); PyList_Append(l, adescr->shape); - } else { + } + else { PyErr_SetString(PyExc_SystemError, "(Hash) Shape of subarray dtype neither a tuple or int ???"); return -1; @@ -212,7 +217,8 @@ static int _array_descr_walk(PyArray_Descr* descr, PyObject *l) if (_is_array_descr_builtin(descr)) { return _array_descr_builtin(descr, l); - } else { + } + else { if(descr->fields != NULL && descr->fields != Py_None) { if (!PyDict_Check(descr->fields)) { PyErr_SetString(PyExc_SystemError, From 40e439348611e5d9ffa2854f114c4df6cd9b4b58 Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Tue, 6 Mar 2012 22:12:32 -0500 Subject: [PATCH 2/5] BUG: fix inconsistencies in dtype flag type at the C level. --- numpy/core/src/multiarray/descriptor.c | 35 ++++++++++++++++++-------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 1bb439dcf93b..ea6117a99c79 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -323,7 +323,7 @@ _convert_from_array_descr(PyObject *obj, int align) PyObject *nameslist; PyArray_Descr *new; PyArray_Descr *conv; - int dtypeflags = 0; + char dtypeflags = 0; int maxalign = 0; n = PyList_GET_SIZE(obj); @@ -479,7 +479,7 @@ _convert_from_array_descr(PyObject *obj, int align) new->fields = fields; new->names = nameslist; new->elsize = totalsize; - new->flags=dtypeflags; + new->flags = dtypeflags; /* Structured arrays get a sticky aligned bit */ if (align) { @@ -512,7 +512,7 @@ _convert_from_list(PyObject *obj, int align) PyObject *nameslist = NULL; int ret; int maxalign = 0; - int dtypeflags = 0; + char dtypeflags = 0; n = PyList_GET_SIZE(obj); /* @@ -844,7 +844,7 @@ _convert_from_dict(PyObject *obj, int align) int n, i; int totalsize, itemsize; int maxalign = 0; - int dtypeflags = 0; + char dtypeflags = 0; int has_out_of_order_fields = 0; fields = PyDict_New(); @@ -2197,10 +2197,10 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args)) } /* - * returns 1 if this data-type has an object portion - * used when setting the state because hasobject is not stored. + * returns NPY_OBJECT_DTYPE_FLAGS if this data-type has an object portion used + * when setting the state because hasobject is not stored. */ -static int +static char _descr_find_object(PyArray_Descr *self) { if (self->flags @@ -2247,7 +2247,8 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) #endif PyObject *subarray, *fields, *names = NULL, *metadata=NULL; int incref_names = 1; - int dtypeflags = 0; + int int_dtypeflags = 0; + char dtypeflags; if (self->fields == Py_None) { Py_INCREF(Py_None); @@ -2267,7 +2268,7 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) #endif if (!PyArg_ParseTuple(args, _ARGSTR_, &version, &endian, &subarray, &names, &fields, &elsize, - &alignment, &dtypeflags, &metadata)) { + &alignment, &int_dtypeflags, &metadata)) { return NULL; #undef _ARGSTR_ } @@ -2280,7 +2281,7 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) #endif if (!PyArg_ParseTuple(args, _ARGSTR_, &version, &endian, &subarray, &names, &fields, &elsize, - &alignment, &dtypeflags)) { + &alignment, &int_dtypeflags)) { return NULL; #undef _ARGSTR_ } @@ -2448,7 +2449,19 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) self->alignment = alignment; } - self->flags = dtypeflags; + /* We use an integer converted to char for backward compatibility with + * pickled arrays. Pickled arrays created with previous versions encoded + * flags as an int even though it actually was a char in the PyArray_Descr + * structure */ + dtypeflags = int_dtypeflags; + if (dtypeflags != int_dtypeflags) { + PyErr_Format(PyExc_ValueError, + "incorrect value for flags variable (overflow)"); + return NULL; + } else { + self->flags = dtypeflags; + } + if (version < 3) { self->flags = _descr_find_object(self); } From 3d8da1540b51060097dea6eb48fe40f5e4a3811f Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Mon, 5 Mar 2012 22:43:56 -0500 Subject: [PATCH 3/5] BUG: fix #2017 by ignoring type_num in the hash input. Since type_num is not considered in PyArray_EquivTypes (for dtype equality comparison), seems reasonable to ignore it for hashing as well. --- numpy/core/src/multiarray/hashdescr.c | 5 ++--- numpy/core/tests/test_dtype.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/multiarray/hashdescr.c b/numpy/core/src/multiarray/hashdescr.c index d9b09868a118..8ec78037c4d6 100644 --- a/numpy/core/src/multiarray/hashdescr.c +++ b/numpy/core/src/multiarray/hashdescr.c @@ -76,9 +76,8 @@ static int _array_descr_builtin(PyArray_Descr* descr, PyObject *l) * For builtin type, hash relies on : kind + byteorder + flags + * type_num + elsize + alignment */ - t = Py_BuildValue("(cciiii)", descr->kind, nbyteorder, - descr->flags, descr->type_num, descr->elsize, - descr->alignment); + t = Py_BuildValue("(cccii)", descr->kind, nbyteorder, + descr->flags, descr->elsize, descr->alignment); for(i = 0; i < PyTuple_Size(t); ++i) { item = PyTuple_GetItem(t, i); diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index ba9de32ba137..d142e1af8167 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -33,6 +33,18 @@ def test_dtype(self): self.assertTrue(dt.byteorder != dt3.byteorder, "bogus test") assert_dtype_equal(dt, dt3) + def test_equivalent_dtype_hashing(self): + # Make sure equivalent dtypes with different type num hash equal + uintp = np.dtype(np.uintp) + if uintp.itemsize == 4: + left = uintp + right = np.dtype(np.uint32) + else: + left = uintp + right = np.dtype(np.ulonglong) + self.assertTrue(left == right) + self.assertTrue(hash(left) == hash(right)) + def test_invalid_types(self): # Make sure invalid type strings raise exceptions for typestr in ['O3', 'O5', 'O7', 'b3', 'h4', 'I5', 'l4', 'l8', From 27a1bb2909b571f13ea98094dfeba20748c5edbb Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Wed, 7 Mar 2012 08:47:20 -0500 Subject: [PATCH 4/5] BUG: fix flags type when exposed to python. --- numpy/core/src/multiarray/descriptor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index ea6117a99c79..0fc0146c3a3b 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -1522,7 +1522,7 @@ static PyMemberDef arraydescr_members[] = { {"alignment", T_INT, offsetof(PyArray_Descr, alignment), READONLY, NULL}, {"flags", - T_INT, offsetof(PyArray_Descr, flags), READONLY, NULL}, + T_BYTE, offsetof(PyArray_Descr, flags), READONLY, NULL}, {NULL, 0, 0, 0, NULL}, }; From b301dfd0b39e3f937035fb603b80323ee5778153 Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Sun, 11 Mar 2012 20:39:33 -0700 Subject: [PATCH 5/5] STY: more C-style tweaks. --- numpy/core/src/multiarray/descriptor.c | 9 ++++-- numpy/core/src/multiarray/hashdescr.c | 42 +++++++++++++------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 0fc0146c3a3b..cd9f887ee6d2 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -2449,16 +2449,19 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) self->alignment = alignment; } - /* We use an integer converted to char for backward compatibility with + /* + * We use an integer converted to char for backward compatibility with * pickled arrays. Pickled arrays created with previous versions encoded * flags as an int even though it actually was a char in the PyArray_Descr - * structure */ + * structure + */ dtypeflags = int_dtypeflags; if (dtypeflags != int_dtypeflags) { PyErr_Format(PyExc_ValueError, "incorrect value for flags variable (overflow)"); return NULL; - } else { + } + else { self->flags = dtypeflags; } diff --git a/numpy/core/src/multiarray/hashdescr.c b/numpy/core/src/multiarray/hashdescr.c index 8ec78037c4d6..9dd065197416 100644 --- a/numpy/core/src/multiarray/hashdescr.c +++ b/numpy/core/src/multiarray/hashdescr.c @@ -34,33 +34,33 @@ static int _array_descr_builtin(PyArray_Descr* descr, PyObject *l); /* * normalize endian character: always return 'I', '<' or '>' */ - static char _normalize_byteorder(char byteorder) - { - switch(byteorder) { - case '=': - if (PyArray_GetEndianness() == NPY_CPU_BIG) { - return '>'; - } - else { - return '<'; - } - default: - return byteorder; - } - } +static char _normalize_byteorder(char byteorder) +{ + switch(byteorder) { + case '=': + if (PyArray_GetEndianness() == NPY_CPU_BIG) { + return '>'; + } + else { + return '<'; + } + default: + return byteorder; + } +} /* * Return true if descr is a builtin type */ static int _is_array_descr_builtin(PyArray_Descr* descr) { - if (descr->fields != NULL && descr->fields != Py_None) { - return 0; - } - if (PyDataType_HASSUBARRAY(descr)) { - return 0; - } - return 1; + if (descr->fields != NULL && descr->fields != Py_None) { + return 0; + } + if (PyDataType_HASSUBARRAY(descr)) { + return 0; + } + return 1; } /*