From 61e926c416fe6f6ccac7848245f5ed09144f2d3b Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sat, 10 Sep 2022 22:57:21 +0900 Subject: [PATCH 1/8] gh-90751: memoryview supports half-float --- Lib/test/test_memoryview.py | 9 +++++++ ...2-09-11-00-37-50.gh-issue-90751.VE8-zf.rst | 1 + Objects/memoryobject.c | 25 ++++++++++++++++--- 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 9d1e1f3063ca55..af91790a689868 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -13,6 +13,7 @@ import io import copy import pickle +import struct from test.support import import_helper @@ -527,6 +528,14 @@ def test_ctypes_cast(self): m[2:] = memoryview(p6).cast(format)[2:] self.assertEqual(d.value, 0.6) + def test_half_float(self): + half_data = struct.pack('eee', 0.0, -1.5, 1.5) + float_data = struct.pack('fff', 0.0, -1.5, 1.5) + half_view = memoryview(half_data).cast('e') + float_view = memoryview(float_data).cast('f') + self.assertEqual(half_view.nbytes*2, float_view.nbytes) + self.assertListEqual(half_view.tolist(), float_view.tolist()) + def test_memoryview_hex(self): # Issue #9951: memoryview.hex() segfaults with non-contiguous buffers. x = b'0' * 200000 diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst new file mode 100644 index 00000000000000..7a54f9a71f04c7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst @@ -0,0 +1 @@ +:class:`memoryview` now supports half-floats. Patch by Dong-hee Na. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index d29e35c2bc34de..67d99c24152216 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1135,6 +1135,7 @@ get_native_fmtchar(char *result, const char *fmt) case 'n': case 'N': size = sizeof(Py_ssize_t); break; case 'f': size = sizeof(float); break; case 'd': size = sizeof(double); break; + case 'e': size = sizeof(float) / 2; break; case '?': size = sizeof(_Bool); break; case 'P': size = sizeof(void *); break; } @@ -1178,6 +1179,7 @@ get_native_fmtstr(const char *fmt) case 'N': RETURN("N"); case 'f': RETURN("f"); case 'd': RETURN("d"); + case 'e': RETURN("e"); case '?': RETURN("?"); case 'P': RETURN("P"); } @@ -1697,6 +1699,12 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) CHECK_RELEASED_AGAIN(self); +#if PY_LITTLE_ENDIAN + int endian = 1; +#else + int endian = 0; +#endif + switch (fmt[0]) { /* signed integers and fast path for 'B' */ @@ -1725,6 +1733,7 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) /* floats */ case 'f': UNPACK_SINGLE(d, ptr, float); goto convert_double; case 'd': UNPACK_SINGLE(d, ptr, double); goto convert_double; + case 'e': d = PyFloat_Unpack2(ptr, endian); goto convert_double; /* bytes object */ case 'c': goto convert_bytes; @@ -1786,6 +1795,11 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt double d; void *p; +#if PY_LITTLE_ENDIAN + int endian = 1; +#else + int endian = 0; +#endif switch (fmt[0]) { /* signed integers */ case 'b': case 'h': case 'i': case 'l': @@ -1862,7 +1876,7 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt break; /* floats */ - case 'f': case 'd': + case 'f': case 'd': case 'e': d = PyFloat_AsDouble(item); if (d == -1.0 && PyErr_Occurred()) goto err_occurred; @@ -1870,9 +1884,14 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt if (fmt[0] == 'f') { PACK_SINGLE(ptr, d, float); } - else { + else if (fmt[0] == 'd') { PACK_SINGLE(ptr, d, double); } + else { + if (PyFloat_Pack2(d, ptr, endian) < 0) { + goto err_occurred; + } + } break; /* bool */ @@ -1882,7 +1901,7 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt return -1; /* preserve original error */ CHECK_RELEASED_INT_AGAIN(self); PACK_SINGLE(ptr, ld, _Bool); - break; + break; /* bytes object */ case 'c': From 2fd17a86f2d8ffab91e55bb6c7e0f02f6c166c3e Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 11 Sep 2022 01:18:10 +0900 Subject: [PATCH 2/8] Address code review --- Lib/test/test_memoryview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index af91790a689868..0eb2a367603cfc 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -533,7 +533,7 @@ def test_half_float(self): float_data = struct.pack('fff', 0.0, -1.5, 1.5) half_view = memoryview(half_data).cast('e') float_view = memoryview(float_data).cast('f') - self.assertEqual(half_view.nbytes*2, float_view.nbytes) + self.assertEqual(half_view.nbytes * 2, float_view.nbytes) self.assertListEqual(half_view.tolist(), float_view.tolist()) def test_memoryview_hex(self): From 9e615fd36ef6264bd284eaaecabb648e9633e2d0 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 11 Sep 2022 01:49:04 +0900 Subject: [PATCH 3/8] Add test --- Lib/test/test_buffer.py | 12 +++++++++--- Objects/memoryobject.c | 10 ++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 468c6ea9def923..cee6ab615d4ef5 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -64,7 +64,7 @@ '?':0, 'c':0, 'b':0, 'B':0, 'h':0, 'H':0, 'i':0, 'I':0, 'l':0, 'L':0, 'n':0, 'N':0, - 'f':0, 'd':0, 'P':0 + 'e':0, 'f':0, 'd':0, 'P':0 } # NumPy does not have 'n' or 'N': @@ -89,7 +89,8 @@ 'i':(-(1<<31), 1<<31), 'I':(0, 1<<32), 'l':(-(1<<31), 1<<31), 'L':(0, 1<<32), 'q':(-(1<<63), 1<<63), 'Q':(0, 1<<64), - 'f':(-(1<<63), 1<<63), 'd':(-(1<<1023), 1<<1023) + 'e':(-65519, 65520), 'f':(-(1<<63), 1<<63), + 'd':(-(1<<1023), 1<<1023) } def native_type_range(fmt): @@ -98,6 +99,8 @@ def native_type_range(fmt): lh = (0, 256) elif fmt == '?': lh = (0, 2) + elif fmt == 'e': + lh = (-65519, 65520) elif fmt == 'f': lh = (-(1<<63), 1<<63) elif fmt == 'd': @@ -125,7 +128,10 @@ def native_type_range(fmt): for fmt in fmtdict['@']: fmtdict['@'][fmt] = native_type_range(fmt) +# Format codes suppported by the memoryview object MEMORYVIEW = NATIVE.copy() + +# Format codes suppported by array.array ARRAY = NATIVE.copy() for k in NATIVE: if not k in "bBhHiIlLfd": @@ -164,7 +170,7 @@ def randrange_fmt(mode, char, obj): x = b'\x01' if char == '?': x = bool(x) - if char == 'f' or char == 'd': + if char in 'efd': x = struct.pack(char, x) x = struct.unpack(char, x)[0] return x diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 67d99c24152216..c35e2b9d98a0af 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2732,6 +2732,15 @@ struct_unpack_cmp(const char *p, const char *q, equal = (x == y); \ } while (0) +#define CMP_HALF(p, q, type) \ + do { \ + type x; \ + type y; \ + memcpy((char *)&x, p, (sizeof x)/2); \ + memcpy((char *)&y, q, (sizeof y)/2); \ + equal = (x == y); \ + } while (0) + static inline int unpack_cmp(const char *p, const char *q, char fmt, struct unpacker *unpack_p, struct unpacker *unpack_q) @@ -2767,6 +2776,7 @@ unpack_cmp(const char *p, const char *q, char fmt, /* XXX DBL_EPSILON? */ case 'f': CMP_SINGLE(p, q, float); return equal; case 'd': CMP_SINGLE(p, q, double); return equal; + case 'e': CMP_HALF(p, q, float); return equal; /* bytes object */ case 'c': return *p == *q; From 8bc8754db15e87ec0250b74b994e5134f5a3bcdc Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 11 Sep 2022 01:51:07 +0900 Subject: [PATCH 4/8] Add Antoine Pitrou as author too. --- .../2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst index 7a54f9a71f04c7..0908f1cc066fd8 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst @@ -1 +1,2 @@ -:class:`memoryview` now supports half-floats. Patch by Dong-hee Na. +:class:`memoryview` now supports half-floats. +Patch by Dong-hee Na and Antoine Pitrou. From 107bdd27892bf1cee0f2159bd6ebfc5bdaf25977 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 11 Sep 2022 02:03:30 +0900 Subject: [PATCH 5/8] Update whatsnew --- Doc/whatsnew/3.12.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 70a1104127e9a8..7000ff62587d3d 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -94,6 +94,9 @@ Other Language Changes length limitation ` documentation. The default limit is 4300 digits in string form. +* :class:`memoryview` now supports the half-floats type (the "e" format code). + (Contributed by Dong-hee Na and Antoine Pitrou in :gh:`90751`.) + New Modules =========== From 343441de0740992a5febc5ea611cada586aa941d Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 10 Sep 2022 19:17:37 +0200 Subject: [PATCH 6/8] Fix half-float memoryview comparison --- Objects/memoryobject.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index c35e2b9d98a0af..c5ab0bf2dedbe4 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1703,7 +1703,7 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) int endian = 1; #else int endian = 0; -#endif +#endif switch (fmt[0]) { @@ -1799,7 +1799,7 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt int endian = 1; #else int endian = 0; -#endif +#endif switch (fmt[0]) { /* signed integers */ case 'b': case 'h': case 'i': case 'l': @@ -2732,15 +2732,6 @@ struct_unpack_cmp(const char *p, const char *q, equal = (x == y); \ } while (0) -#define CMP_HALF(p, q, type) \ - do { \ - type x; \ - type y; \ - memcpy((char *)&x, p, (sizeof x)/2); \ - memcpy((char *)&y, q, (sizeof y)/2); \ - equal = (x == y); \ - } while (0) - static inline int unpack_cmp(const char *p, const char *q, char fmt, struct unpacker *unpack_p, struct unpacker *unpack_q) @@ -2776,7 +2767,17 @@ unpack_cmp(const char *p, const char *q, char fmt, /* XXX DBL_EPSILON? */ case 'f': CMP_SINGLE(p, q, float); return equal; case 'd': CMP_SINGLE(p, q, double); return equal; - case 'e': CMP_HALF(p, q, float); return equal; + case 'e': { +#if PY_LITTLE_ENDIAN + int endian = 1; +#else + int endian = 0; +#endif + /* Note: PyFloat_Unpack2 should never fail */ + double u = PyFloat_Unpack2(p, endian); + double v = PyFloat_Unpack2(q, endian); + return (u == v); + } /* bytes object */ case 'c': return *p == *q; From a1ce9d41ee404c2ece0c16cf4eeebb18d7e9a18a Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 10 Sep 2022 19:18:57 +0200 Subject: [PATCH 7/8] Update Doc/whatsnew/3.12.rst --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 7000ff62587d3d..c211ea9a678bc7 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -94,7 +94,7 @@ Other Language Changes length limitation ` documentation. The default limit is 4300 digits in string form. -* :class:`memoryview` now supports the half-floats type (the "e" format code). +* :class:`memoryview` now supports the half-float type (the "e" format code). (Contributed by Dong-hee Na and Antoine Pitrou in :gh:`90751`.) From e8c1fed38ff93546d554c43a49e82bbd6fa910f4 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sat, 10 Sep 2022 17:31:28 +0000 Subject: [PATCH 8/8] Replace deprecated method for numpy test --- Lib/test/test_buffer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index cee6ab615d4ef5..8ac3b7e7eb29d1 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -2252,7 +2252,7 @@ def test_py_buffer_to_contiguous(self): ### ### Fortran output: ### --------------- - ### >>> fortran_buf = nd.tostring(order='F') + ### >>> fortran_buf = nd.tobytes(order='F') ### >>> fortran_buf ### b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b' ### @@ -2295,7 +2295,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='C')) + self.assertEqual(b, na.tobytes(order='C')) # 'F' request if f == 0: # 'C' to 'F' @@ -2318,7 +2318,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='F')) + self.assertEqual(b, na.tobytes(order='F')) # 'A' request if f == ND_FORTRAN: @@ -2342,7 +2342,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='A')) + self.assertEqual(b, na.tobytes(order='A')) # multi-dimensional, non-contiguous input nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE|ND_PIL)