8000 gh-90751: memoryview now supports half-float by corona10 · Pull Request #96738 · python/cpython · GitHub
[go: up one dir, main page]
Skip to content

gh-90751: memoryview now supports half-float #96738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Sep 10, 2022
Merged

gh-90751: memoryview now supports half-float #96738

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
61e926c
gh-90751: memoryview supports half-float
corona10 Sep 10, 2022
2fd17a8
Address code review
corona10 Sep 10, 2022
9e615fd
Add test
corona10 Sep 10, 2022
8bc8754
Add Antoine Pitrou as author too.
corona10 Sep 10, 2022
107bdd2
Update whatsnew
corona10 Sep 10, 2022
343441d
Fix half-float memoryview comparison
pitrou Sep 10, 2022
abf2050
Merge branch 'gh-90751' of https://github.com/corona10/cpython into g…
pitrou Sep 10, 2022
a1ce9d4
Update Doc/whatsnew/3.12.rst
pitrou Sep 10, 2022
e8c1fed
Replace deprecated method for numpy test
corona10 Sep 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Doc/whatsnew/3.12.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,9 @@ Other Language Changes
length limitation <int_max_str_digits>` documentation. The default limit
is 4300 digits in string form.

* :class:`memoryview` now supports the half-float type (the "e" format code).
(Contributed by Dong-hee Na and Antoine Pitrou in :gh:`90751`.)


New Modules
===========
Expand Down
20 changes: 13 additions & 7 deletions Lib/test/test_buffer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
'?':0, 'c':0, 'b':0, 'B':0,
'h':0, 'H':0, 'i':0, 'I':0,
'l':0, 'L':0, 'n':0, 'N':0,
'f':0, 'd':0, 'P':0
'e':0, 'f':0, 'd':0, 'P':0
}

# NumPy does not have 'n' or 'N':
Expand All @@ -89,7 +89,8 @@
'i':(-(1<<31), 1<<31), 'I':(0, 1<<32),
'l':(-(1<<31), 1<<31), 'L':(0, 1<<32),
'q':(-(1<<63), 1<<63), 'Q':(0, 1<<64),
'f':(-(1<<63), 1<<63), 'd':(-(1<<1023), 1<<1023)
'e':(-65519, 65520), 'f':(-(1<<63), 1<<63),
'd':(-(1<<1023), 1<<1023)
}

def native_type_range(fmt):
Expand All @@ -98,6 +99,8 @@ def native_type_range(fmt):
lh = (0, 256)
elif fmt == '?':
lh = (0, 2)
elif fmt == 'e':
lh = (-65519, 65520)
elif fmt == 'f':
lh = (-(1<<63), 1<<63)
elif fmt == 'd':
Expand Down Expand Up @@ -125,7 +128,10 @@ def native_type_range(fmt):
for fmt in fmtdict['@']:
fmtdict['@'][fmt] = native_type_range(fmt)

# Format codes suppported by the memoryview object
MEMORYVIEW = NATIVE.copy()

# Format codes suppported by array.array
ARRAY = NATIVE.copy()
for k in NATIVE:
if not k in "bBhHiIlLfd":
Expand Down Expand Up @@ -164,7 +170,7 @@ def randrange_fmt(mode, char, obj):
x = b'\x01'
if char == '?':
x = bool(x)
if char == 'f' or char == 'd':
if char in 'efd':
x = struct.pack(char, x)
x = struct.unpack(char, x)[0]
return x
Expand Down Expand Up @@ -2246,7 +2252,7 @@ def test_py_buffer_to_contiguous(self):
###
### Fortran output:
### ---------------
### >>> fortran_buf = nd.tostring(order='F')
### >>> fortran_buf = nd.tobytes(order='F')
### >>> fortran_buf
### b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b'
###
Expand Down Expand Up @@ -2289,7 +2295,7 @@ def test_py_buffer_to_contiguous(self):
self.assertEqual(memoryview(y), memoryview(nd))

if numpy_array:
self.assertEqual(b, na.tostring(order='C'))
self.assertEqual(b, na.tobytes(order='C'))

# 'F' request
if f == 0: # 'C' to 'F'
Expand All @@ -2312,7 +2318,7 @@ def test_py_buffer_to_contiguous(self):
self.assertEqual(memoryview(y), memoryview(nd))

if numpy_array:
self.assertEqual(b, na.tostring(order='F'))
self.assertEqual(b, na.tobytes(order='F'))

# 'A' request
if f == ND_FORTRAN:
Expand All @@ -2336,7 +2342,7 @@ def test_py_buffer_to_contiguous(self):
self.assertEqual(memoryview(y), memoryview(nd))

if numpy_array:
self.assertEqual(b, na.tostring(order='A'))
self.assertEqual(b, na.tobytes(order='A'))

# multi-dimensional, non-contiguous input
nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE|ND_PIL)
Expand Down
9 changes: 9 additions & 0 deletions Lib/test/test_memoryview.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import io
import copy
import pickle
import struct

from test.support import import_helper

Expand Down Expand Up @@ -527,6 +528,14 @@ def test_ctypes_cast(self):
m[2:] = memoryview(p6).cast(format)[2:]
self.assertEqual(d.value, 0.6)

def test_half_float(self):
half_data = struct.pack('eee', 0.0, -1.5, 1.5)
float_data = struct.pack('fff', 0.0, -1.5, 1.5)
half_view = memoryview(half_data).cast('e')
float_view = memoryview(float_data).cast('f')
self.assertEqual(half_view.nbytes * 2, float_view.nbytes)
self.assertListEqual(half_view.tolist(), float_view.tolist())

def test_memoryview_hex(self):
# Issue #9951: memoryview.hex() segfaults with non-contiguous buffers.
x = b'0' * 200000
Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
:class:`memoryview` now supports half-floats.
Patch by Dong-hee Na and Antoine Pitrou.
36 changes: 33 additions & 3 deletions Objects/memoryobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1135,6 +1135,7 @@ get_native_fmtchar(char *result, const char *fmt)
case 'n': case 'N': size = sizeof(Py_ssize_t); break;
case 'f': size = sizeof(float); break;
case 'd': size = sizeof(double); break;
case 'e': size = sizeof(float) / 2; break;
case '?': size = sizeof(_Bool); break;
case 'P': size = sizeof(void *); break;
}
Expand Down Expand Up @@ -1178,6 +1179,7 @@ get_native_fmtstr(const char *fmt)
case 'N': RETURN("N");
case 'f': RETURN("f");
case 'd': RETURN("d");
case 'e': RETURN("e");
case '?': RETURN("?");
case 'P': RETURN("P");
}
Expand Down Expand Up @@ -1697,6 +1699,12 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt)

CHECK_RELEASED_AGAIN(self);

#if PY_LITTLE_ENDIAN
int endian = 1;
#else
int endian = 0;
#endif

switch (fmt[0]) {

/* signed integers and fast path for 'B' */
Expand Down Expand Up @@ -1725,6 +1733,7 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt)
/* floats */
case 'f': UNPACK_SINGLE(d, ptr, float); goto convert_double;
case 'd': UNPACK_SINGLE(d, ptr, double); goto convert_double;
case 'e': d = PyFloat_Unpack2(ptr, endian); goto convert_double;

/* bytes object */
case 'c': goto convert_bytes;
Expand Down Expand Up @@ -1786,6 +1795,11 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt
double d;
void *p;

#if PY_LITTLE_ENDIAN
int endian = 1;
#else
int endian = 0;
#endif
switch (fmt[0]) {
/* signed integers */
case 'b': case 'h': case 'i': case 'l':
Expand Down Expand Up @@ -1862,17 +1876,22 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt
break;

/* floats */
case 'f': case 'd':
case 'f': case 'd': case 'e':
d = PyFloat_AsDouble(item);
if (d == -1.0 && PyErr_Occurred())
goto err_occurred;
CHECK_RELEASED_INT_AGAIN(self);
if (fmt[0] == 'f') {
PACK_SINGLE(ptr, d, float);
}
else {
else if (fmt[0] == 'd') {
PACK_SINGLE(ptr, d, double);
}
else {
if (PyFloat_Pack2(d, ptr, endian) < 0) {
goto err_occurred;
}
}
break;

/* bool */
Expand All @@ -1882,7 +1901,7 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt
return -1; /* preserve original error */
CHECK_RELEASED_INT_AGAIN(self);
PACK_SINGLE(ptr, ld, _Bool);
break;
break;

/* bytes object */
case 'c':
Expand Down Expand Up @@ -2748,6 +2767,17 @@ unpack_cmp(const char *p, const char *q, char fmt,
/* XXX DBL_EPSILON? */
case 'f': CMP_SINGLE(p, q, float); return equal;
case 'd': CMP_SINGLE(p, q, double); return equal;
case 'e': {
#if PY_LITTLE_ENDIAN
int endian = 1;
#else
int endian = 0;
#endif
/* Note: PyFloat_Unpack2 should never fail */
double u = PyFloat_Unpack2(p, endian);
double v = PyFloat_Unpack2(q, endian);
return (u == v);
}

/* bytes object */
case 'c': return *p == *q;
Expand Down
0