8000 BUG: Various fixes to _dtype_from_pep3118 by eric-wieser · Pull Request #9054 · numpy/numpy · GitHub
[go: up one dir, main page]
Skip to content

BUG: Various fixes to _dtype_from_pep3118 #9054

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 9, 2017
Merged

BUG: Various fixes to _dtype_from_pep3118 #9054

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7f6c95f
MAINT: refactor _dtype_from_pep3118 in terms of a stream
eric-wieser May 4, 2017
fed2e1a
ENH: Pad with itemsize, not padding fields
eric-wieser May 5, 2017
3c4545f
BUG: Fix non-determinism in order of fields created from pep3118 formats
eric-wieser May 5, 2017
a4f435c
BUG: Prevent autogenerated names clashing with given names
eric-wieser May 5, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
ENH: Pad with itemsize, not padding fields 
Adding padding fields means that dtypes with the same fields cannot necessarily
be cast to one another. Padding shouldn't prevent casting.

Partially fixes #9053
  • Loading branch information
@eric-wieser
eric-wieser committed May 5, 2017
commit fed2e1a4304123f8a0cc4313cae9ffe3d6614b78
45 changes: 18 additions & 27 deletions numpy/core/_internal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -619,33 +619,24 @@ def get_dummy_name():
def _add_trailing_padding(value, padding):
"""Inject the specified number of padding bytes at the end of a dtype"""
if value.fields is None:
vfields = {'f0': (value, 0)}
field_spec = dict(
names=['f0'],
formats=[value],
offsets=[0],
itemsize=value.itemsize
)
else:
vfields = dict(value.fields)

if (value.names and value.names[-1] == '' and
value[''].char == 'V'):
# A trailing padding field is already present
vfields[''] = ('V%d' % (vfields[''][0].itemsize + padding),
vfields[''][1])
value = dtype(vfields)
else:
# Get a free name for the padding field
j = 0
while True:
name = 'pad%d' % j
if name not in vfields:
vfields[name] = ('V%d' % padding, value.itemsize)
break
j += 1

value = dtype(vfields)
if '' not in vfields:
# Strip out the name of the padding field
names = list(value.names)
names[-1] = ''
value.names = tuple(names)
return value
fields = value.fields
names = value.names
field_spec = dict(
names=names,
formats=[fields[name][0] for name in names],
offsets=[fields[name][1] for name in names],
itemsize=value.itemsize
)

field_spec['itemsize'] += padding
return dtype(field_spec)

def _prod(a):
p = 1
Expand All @@ -660,7 +651,7 @@ def _gcd(a, b):
return a

def _lcm(a, b):
return a / _gcd(a, b) * b
return a // _gcd(a, b) * b

# Exception used in shares_memory()
class TooHardError(RuntimeError):
Expand Down
47 changes: 26 additions & 21 deletions numpy/core/tests/test_multiarray.py
View file
Open in desktop
9E95
Original file line number Diff line number Diff line change
Expand Up @@ -5831,12 +5831,8 @@ def test_object(self):
class TestPEP3118Dtype(object):
def _check(self, spec, wanted):
dt = np.dtype(wanted)
if isinstance(wanted, list) and isinstance(wanted[-1], tuple):
if wanted[-1][0] == '':
names = list(dt.names)
names[-1] = ''
dt.names = tuple(names)
assert_equal(_dtype_from_pep3118(spec), dt,
actual = _dtype_from_pep3118(spec)
assert_equal(actual, dt,
err_msg="spec %r != dtype %r" % (spec, wanted))

def test_native_padding(self):
Expand All @@ -5860,21 +5856,24 @@ def test_trailing_padding(self):
# Trailing padding should be included, *and*, the item size
# should match the alignment if in aligned mode
align = np.dtype('i').alignment
size = np.dtype('i').itemsize

def VV(n):
return 'V%d' % (align*(1 + (n-1)//align))
def aligned(n):
return align*(1 + (n-1)//align)

self._check('ix', [('f0', 'i'), ('', VV(1))])
self._check('ixx', [('f0', 'i'), ('', VV(2))])
self._check('ixxx', [('f0', 'i'), ('', VV(3))])
self._check('ixxxx', [('f0', 'i'), ('', VV(4))])
self._check('i7x', [('f0', 'i'), ('', VV(7))])
base = dict(formats=['i'], names=['f0'])

self._check('^ix', [('f0', 'i'), ('', 'V1')])
self._check('^ixx', [('f0', 'i'), ('', 'V2')])
self._check('^ixxx', [('f0', 'i'), ('', 'V3')])
self._check('^ixxxx', [('f0', 'i'), ('', 'V4')])
self._check('^i7x', [('f0', 'i'), ('', 'V7')])
self._check('ix', dict(itemsize=aligned(size + 1), **base))
self._check('ixx', dict(itemsize=aligned(size + 2), **base))
self._check('ixxx', dict(itemsize=aligned(size + 3), **base))
self._check('ixxxx', dict(itemsize=aligned(size + 4), **base))
self._check('i7x', dict(itemsize=aligned(size + 7), **base))

self._check('^ix', dict(itemsize=size + 1, **base))
self._check('^ixx', dict(itemsize=size + 2, **base))
self._check('^ixxx', dict(itemsize=size + 3, **base))
self._check('^ixxxx', dict(itemsize=size + 4, **base))
self._check('^i7x', dict(itemsize=size + 7, **base))

def test_native_padding_3(self):
dt = np.dtype(
Expand Down Expand Up @@ -5904,11 +5903,17 @@ def test_byteorder_inside_struct(self):
def test_intra_padding(self):
# Natively aligned sub-arrays may require some internal padding
align = np.dtype('i').alignment
size = np.dtype('i').itemsize

def VV(n):
return 'V%d' % (align*(1 + (n-1)//align))
def aligned(n):
return (align*(1 + (n-1)//align))

self._check('(3)T{ix}', ({'f0': ('i', 0), '': (VV(1), 4)}, (3,)))
self._check('(3)T{ix}', (dict(
names=['f0'],
formats=['i'],
offsets=[0],
itemsize=aligned(size + 1)
), (3,)))

def test_char_vs_string(self):
dt = np.dtype('c')
Expand Down
0