From 86dbd45bfd2753000a90756cc947089657f3db54 Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Fri, 7 Dec 2012 11:12:38 -0600 Subject: [PATCH 1/6] Fix invalid typestring size Revert to pre numpy 1.7 behavior where invalid typestring size was ignored and isplay deprecate warning. This warning should eventually be changed to an error in future numpy versions. --- numpy/core/src/multiarray/conversion_utils.c | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 914bc274dd30..f577e06782d2 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -954,6 +954,7 @@ NPY_NO_EXPORT int PyArray_TypestrConvert(int itemsize, int gentype) { int newtype = NPY_NOTYPE; + PyArray_Descr *temp; switch (gentype) { case NPY_GENBOOLLTR: @@ -1105,7 +1106,28 @@ PyArray_TypestrConvert(int itemsize, int gentype) newtype = NPY_TIMEDELTA; } break; + default: + temp = PyArray_DescrFromType(gentype); + if (temp != NULL) { + if (temp->elsize != itemsize) { + + const char *msg = "Specified size is invalid for this data type.\n" + "Size will be ignored in NumPy 1.7 but may throw an exception in future versions."; + if (DEPRECATE_FUTUREWARNING(msg) < 0) { + return -1; + } + + newtype = gentype; + } + else { + newtype = gentype; + } + + Py_DECREF(temp); + } + break; } + return newtype; } From 4237c6e3e2396ef0b7e9130d0c0bf15c4cba9afa Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Tue, 11 Dec 2012 13:54:56 -0600 Subject: [PATCH 2/6] Refactor to cover more cases; modify unit tests --- numpy/core/src/multiarray/conversion_utils.c | 42 +++++++++++--------- numpy/core/tests/test_datetime.py | 11 +++-- numpy/core/tests/test_dtype.py | 15 ++++--- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index f577e06782d2..e27cd4ffde46 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -955,6 +955,8 @@ PyArray_TypestrConvert(int itemsize, int gentype) { int newtype = NPY_NOTYPE; PyArray_Descr *temp; + const char *msg = "Specified size is invalid for this data type.\n" + "Size will be ignored in NumPy 1.7 but may throw an exception in future versions."; switch (gentype) { case NPY_GENBOOLLTR: @@ -1106,28 +1108,32 @@ PyArray_TypestrConvert(int itemsize, int gentype) newtype = NPY_TIMEDELTA; } break; - default: - temp = PyArray_DescrFromType(gentype); - if (temp != NULL) { - if (temp->elsize != itemsize) { - - const char *msg = "Specified size is invalid for this data type.\n" - "Size will be ignored in NumPy 1.7 but may throw an exception in future versions."; - if (DEPRECATE_FUTUREWARNING(msg) < 0) { - return -1; - } - - newtype = gentype; - } - else { - newtype = gentype; + } + + /* + * Raise deprecate warning if new type hasn't been + * set yet and size char is invalid. + * This should eventually be changed to an error in + * future NumPy versions. + */ + if (newtype == NPY_NOTYPE) { + temp = PyArray_DescrFromType(gentype); + if (temp != NULL) { + if (temp->elsize != itemsize) { + if (DEPRECATE(msg) < 0) { + return -1; } - Py_DECREF(temp); + newtype = gentype; } - break; + else { + newtype = gentype; + } + + Py_DECREF(temp); + } } - + return newtype; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index fbde86b5779c..f4a1ee617e52 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -48,12 +48,11 @@ def test_datetime_dtype_creation(self): assert_raises(TypeError, np.dtype, 'm8[badunit]') assert_raises(TypeError, np.dtype, 'M8[YY]') assert_raises(TypeError, np.dtype, 'm8[YY]') - assert_raises(TypeError, np.dtype, 'M4') - assert_raises(TypeError, np.dtype, 'm4') - assert_raises(TypeError, np.dtype, 'M7') - assert_raises(TypeError, np.dtype, 'm7') - assert_raises(TypeError, np.dtype, 'M16') - assert_raises(TypeError, np.dtype, 'm16') + assert_warns(DeprecationWarning, np.dtype, 'm4') + assert_warns(DeprecationWarning, np.dtype, 'M7') + assert_warns(DeprecationWarning, np.dtype, 'm7') + assert_warns(DeprecationWarning, np.dtype, 'M16') + assert_warns(DeprecationWarning, np.dtype, 'm16') def test_datetime_casting_rules(self): # Cannot cast safely/same_kind between timedelta and datetime diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 5645a0824a2a..88cae4e577c0 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -47,13 +47,16 @@ def test_equivalent_dtype_hashing(self): self.assertTrue(hash(left) == hash(right)) def test_invalid_types(self): - # Make sure invalid type strings raise exceptions - for typestr in ['O3', 'O5', 'O7', 'b3', 'h4', 'I5', 'l4', 'l8', - 'L4', 'L8', 'q8', 'q16', 'Q8', 'Q16', 'e3', - 'f5', 'd8', 't8', 'g12', 'g16', - 'NA[u4,0xffffffff]']: + # Make sure invalid type strings raise exceptions. + # For now, display a deprecation warning for invalid + # type sizes. In the future this should be changed + # to an exception. + for typestr in ['O3', 'O5', 'O7', 'b3', 'h4', 'I5', 'l4', + 'L4', 'q16', 'Q16', 'e3', 'f5', 'g12']: #print typestr - assert_raises(TypeError, np.dtype, typestr) + assert_warns(DeprecationWarning, np.dtype, typestr) + + assert_raises(TypeError, np.dtype, 't8', 'NA[u4,0xffffffff]') def test_bad_param(self): # Can't give a size that's too small From 18e73ab359968875e7df6a0b45e788feee8fd716 Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Fri, 14 Dec 2012 14:09:29 -0600 Subject: [PATCH 3/6] Modify unit tests for 32 bit support --- numpy/core/tests/test_dtype.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 88cae4e577c0..542add3a2f03 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -51,11 +51,26 @@ def test_invalid_types(self): # For now, display a deprecation warning for invalid # type sizes. In the future this should be changed # to an exception. - for typestr in ['O3', 'O5', 'O7', 'b3', 'h4', 'I5', 'l4', - 'L4', 'q16', 'Q16', 'e3', 'f5', 'g12']: + + for typestr in ['O3', 'O5', 'O7', 'b3', 'h4', 'I5', + 'e3', 'f5', 'g12']: #print typestr assert_warns(DeprecationWarning, np.dtype, typestr) + if np.dtype('l').itemsize == 8: + assert_warns(DeprecationWarning, np.dtype, 'l4') + assert_warns(DeprecationWarning, np.dtype, 'L4') + else: + assert_warns(DeprecationWarning, np.dtype, 'l8') + assert_warns(DeprecationWarning, np.dtype, 'L8') + + if np.dtype('q').itemsize == 8: + assert_warns(DeprecationWarning, np.dtype, 'q4') + assert_warns(DeprecationWarning, np.dtype, 'Q4') + else: + assert_warns(DeprecationWarning, np.dtype, 'q8') + assert_warns(DeprecationWarning, np.dtype, 'Q8') + assert_raises(TypeError, np.dtype, 't8', 'NA[u4,0xffffffff]') def test_bad_param(self): From 83de7cd1810f38a845d7e281943d46c5e225633b Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Mon, 17 Dec 2012 11:42:36 -0600 Subject: [PATCH 4/6] Refactor unit tests and add DECREF - Refactor unit tests for invalid dtypes so that each test is on a separate line. This will make it easier to figure out which typestring is failing the unit test. - Add Py_DECREF for temp variable --- numpy/core/src/multiarray/conversion_utils.c | 1 + numpy/core/tests/test_dtype.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index e27cd4ffde46..b05314a0a697 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -1121,6 +1121,7 @@ PyArray_TypestrConvert(int itemsize, int gentype) if (temp != NULL) { if (temp->elsize != itemsize) { if (DEPRECATE(msg) < 0) { + Py_DECREF(temp); return -1; } diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 542add3a2f03..ad8e5a8b386f 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -47,15 +47,20 @@ def test_equivalent_dtype_hashing(self): self.assertTrue(hash(left) == hash(right)) def test_invalid_types(self): - # Make sure invalid type strings raise exceptions. + # Make sure invalid type strings raise a warning. # For now, display a deprecation warning for invalid # type sizes. In the future this should be changed # to an exception. - for typestr in ['O3', 'O5', 'O7', 'b3', 'h4', 'I5', - 'e3', 'f5', 'g12']: - #print typestr - assert_warns(DeprecationWarning, np.dtype, typestr) + assert_warns(DeprecationWarning, np.dtype, 'O3') + assert_warns(DeprecationWarning, np.dtype, 'O5') + assert_warns(DeprecationWarning, np.dtype, 'O7') + assert_warns(DeprecationWarning, np.dtype, 'b3') + assert_warns(DeprecationWarning, np.dtype, 'h4') + assert_warns(DeprecationWarning, np.dtype, 'I5') + assert_warns(DeprecationWarning, np.dtype, 'e3') + assert_warns(DeprecationWarning, np.dtype, 'f5') + assert_warns(DeprecationWarning, np.dtype, 'g12') if np.dtype('l').itemsize == 8: assert_warns(DeprecationWarning, np.dtype, 'l4') @@ -71,8 +76,6 @@ def test_invalid_types(self): assert_warns(DeprecationWarning, np.dtype, 'q8') assert_warns(DeprecationWarning, np.dtype, 'Q8') - assert_raises(TypeError, np.dtype, 't8', 'NA[u4,0xffffffff]') - def test_bad_param(self): # Can't give a size that's too small assert_raises(ValueError, np.dtype, From 0f5afde4b719d622f4530c707d9577eea6c131fd Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Fri, 21 Dec 2012 13:39:19 -0600 Subject: [PATCH 5/6] Add debug logging to figure out why unit test fails on Travis --- numpy/core/src/multiarray/conversion_utils.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index b05314a0a697..7adf6ae2063d 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -1116,9 +1116,19 @@ PyArray_TypestrConvert(int itemsize, int gentype) * This should eventually be changed to an error in * future NumPy versions. */ + + if (newtype == NPY_NOTYPE) + { + printf("XXX: gentype=%c\n", gentype); + } if (newtype == NPY_NOTYPE) { temp = PyArray_DescrFromType(gentype); + + if (gentype == 'g') + printf("XXX: temp=%x\n", temp); if (temp != NULL) { + if (gentype == 'g') + printf("XXX: elsize=%d itemsize=%d\n", temp->elsize, itemsize); if (temp->elsize != itemsize) { if (DEPRECATE(msg) < 0) { Py_DECREF(temp); From 1f1537eff87721f9d3829513eadb1fe85fc1d866 Mon Sep 17 00:00:00 2001 From: Jay Bourque Date: Fri, 21 Dec 2012 14:29:52 -0600 Subject: [PATCH 6/6] Add fix to unit test; remove debug logging Add check to see if 12 bytes is a valid size for a long double --- numpy/core/src/multiarray/conversion_utils.c | 10 ---------- numpy/core/tests/test_dtype.py | 6 +++++- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 7adf6ae2063d..b05314a0a697 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -1116,19 +1116,9 @@ PyArray_TypestrConvert(int itemsize, int gentype) * This should eventually be changed to an error in * future NumPy versions. */ - - if (newtype == NPY_NOTYPE) - { - printf("XXX: gentype=%c\n", gentype); - } if (newtype == NPY_NOTYPE) { temp = PyArray_DescrFromType(gentype); - - if (gentype == 'g') - printf("XXX: temp=%x\n", temp); if (temp != NULL) { - if (gentype == 'g') - printf("XXX: elsize=%d itemsize=%d\n", temp->elsize, itemsize); if (temp->elsize != itemsize) { if (DEPRECATE(msg) < 0) { Py_DECREF(temp); diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index ad8e5a8b386f..92e3351652bd 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -60,7 +60,11 @@ def test_invalid_types(self): assert_warns(DeprecationWarning, np.dtype, 'I5') assert_warns(DeprecationWarning, np.dtype, 'e3') assert_warns(DeprecationWarning, np.dtype, 'f5') - assert_warns(DeprecationWarning, np.dtype, 'g12') + + if np.dtype('g').itemsize == 8 or np.dtype('g').itemsize == 16: + assert_warns(DeprecationWarning, np.dtype, 'g12') + elif np.dtype('g').itemsize == 12: + assert_warns(DeprecationWarning, np.dtype, 'g16') if np.dtype('l').itemsize == 8: assert_warns(DeprecationWarning, np.dtype, 'l4')