8000 MAINT: Large overhead in some random functions by przemb · Pull Request #15511 · numpy/numpy · GitHub
[go: up one dir, main page]
Skip to content

MAINT: Large overhead in some random functions #15511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 6, 2020
Merged

MAINT: Large overhead in some random functions #15511

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a15dc30
MAINT: Large overhead in some random functions #15460
przemb Feb 5, 2020
df27b88
Changes suggested in review
przemb Feb 5, 2020
deb8571
Documentation updated.
przemb Feb 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 0 additions & 10 deletions numpy/random/_bounded_integers.pyx.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,6 @@ cdef extern from "numpy/random/distributions.h":
np.npy_bool *out) nogil



_integers_types = {'bool': (0, 2),
'int8': (-2**7, 2**7),
'int16': (-2**15, 2**15),
'int32': (-2**31, 2**31),
'int64': (-2**63, 2**63),
'uint8': (0, 2**8),
'uint16': (0, 2**16),
'uint32': (0, 2**32),
'uint64': (0, 2**64)}
{{
py:
type_info = (('uint32', 'uint32', 'uint64', 'NPY_UINT64', 0, 0, 0, '0X100000000ULL'),
Expand Down
71 changes: 35 additions & 36 deletions numpy/random/_generator.pyx
View file
Open in desktop
8000
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ from libc.stdint cimport (uint8_t, uint16_t, uint32_t, uint64_t,
from ._bounded_integers cimport (_rand_bool, _rand_int32, _rand_int64,
_rand_int16, _rand_int8, _rand_uint64, _rand_uint32, _rand_uint16,
_rand_uint8, _gen_mask)
from ._bounded_integers import _integers_types
from ._pcg64 import PCG64
from numpy.random cimport bitgen_t
from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE,
Expand Down Expand Up @@ -312,13 +311,13 @@ cdef class Generator:

"""
cdef double temp
key = np.dtype(dtype).name
if key == 'float64':
_dtype = np.dtype(dtype)
Copy link
Member
@eric-wieser eric-wieser Feb 5, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would normally spell this

Suggested change
_dtype = np.dtype(dtype)
dtype = np.dtype(dtype)

Does cython not like that?

(same throughout)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I change it, test are failing... seems that it is not proper.

Copy link
Member
@eric-wieser eric-wieser Feb 5, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you have a link to the CI failure?

Edit: caused diagnosed here: #15511 (comment)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need the original value so we can raise an error with it later. What about descr or concrete_dtype?

if _dtype == np.float64:
return double_fill(&random_standard_uniform_fill, &self._bitgen, size, self.lock, out)
elif key == 'float32':
elif _dtype == np.float32:
return float_fill(&random_standard_uniform_fill_f, &self._bitgen, size, self.lock, out)
else:
raise TypeError('Unsupported dtype "%s" for random' % key)
raise TypeError('Unsupported dtype %r for random' % _dtype)

def beta(self, a, b, size=None):
"""
Expand Down Expand Up @@ -454,20 +453,20 @@ cdef class Generator:
>>> n = np.random.default_rng().standard_exponential((3, 8000))

"""
key = np.dtype(dtype).name
if key == 'float64':
_dtype = np.dtype(dtype)
if _dtype == np.float64:
if method == u'zig':
return double_fill(&random_standard_exponential_fill, &self._bitgen, size, self.lock, out)
else:
return double_fill(&random_standard_exponential_inv_fill, &self._bitgen, size, self.lock, out)
elif key == 'float32':
elif _dtype == np.float32:
if method == u'zig':
return float_fill(&random_standard_exponential_fill_f, &self._bitgen, size, self.lock, out)
else:
return float_fill(&random_standard_exponential_inv_fill_f, &self._bitgen, size, self.lock, out)
else:
raise TypeError('Unsupported dtype "%s" for standard_exponential'
% key)
raise TypeError('Unsupported dtype %r for standard_exponential'
% _dtype)

def integers(self, low, high=None, size=None, dtype=np.int64, endpoint=False):
"""
Expand Down Expand Up @@ -559,39 +558,39 @@ cdef class Generator:
high = low
low = 0

dt = np.dtype(dtype)
key = dt.name
if key not in _integers_types:
raise TypeError('Unsupported dtype "%s" for integers' % key)
if not dt.isnative:
raise ValueError('Providing a dtype with a non-native byteorder '
'is not supported. If you require '
'platform-independent byteorder, call byteswap '
'when required.')
_dtype = np.dtype(dtype)

# Implementation detail: the old API used a masked method to generate
# bounded uniform integers. Lemire's method is preferable since it is
# faster. randomgen allows a choice, we will always use the faster one.
cdef bint _masked = False

if key == 'int32':
if _dtype == np.int32:
ret = _rand_int32(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'int64':
elif _dtype == np.int64:
ret = _rand_int64(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'int16':
elif _dtype == np.int16:
ret = _rand_int16(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'int8':
elif _dtype == np.int8:
ret = _rand_int8(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'uint64':
elif _dtype == np.uint64:
ret = _rand_uint64(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'uint32':
elif _dtype == np.uint32:
ret = _rand_uint32(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'uint16':
elif _dtype == np.uint16:
ret = _rand_uint16(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'uint8':
elif _dtype == np.uint8:
ret = _rand_uint8(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'bool':
elif _dtype == np.bool_:
ret = _rand_bool(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif not _dtype.isnative:
raise ValueError('Providing a dtype with a non-native byteorder '
'is not supported. If you require '
'platform-independent byteorder, call byteswap '
'when required.')
else:
raise TypeError('Unsupported dtype %r for integers' % _dtype)


if size is None and dtype in (bool, int, np.compat.long):
if np.array(ret).shape == ():
Expand Down Expand Up @@ -1041,14 +1040,14 @@ cdef class Generator:
[ 0.39924804, 4.68456316, 4.99394529, 4.84057254]]) # random

"""
key = np.dtype(dtype).name
if key == 'float64':
_dtype = np.dtype(dtype)
if _dtype == np.float64:
return double_fill(&random_standard_normal_fill, &self._bitgen, size, self.lock, out)
elif key == 'float32':
elif _dtype == np.float32:
return float_fill(&random_standard_normal_fill_f, &self._bitgen, size, self.lock, out)

else:
raise TypeError('Unsupported dtype "%s" for standard_normal' % key)
raise TypeError('Unsupported dtype %r for standard_normal' % _dtype)

def normal(self, loc=0.0, scale=1.0, size=None):
"""
Expand Down Expand Up @@ -1230,19 +1229,19 @@ cdef class Generator:

"""
cdef void *func
key = np.dtype(dtype).name
if key == 'float64':
_dtype = np.dtype(dtype)
if _dtype == np.float64:
return cont(&random_standard_gamma, &self._bitgen, size, self.lock, 1,
shape, 'shape', CONS_NON_NEGATIVE,
0.0, '', CONS_NONE,
0.0, '', CONS_NONE,
out)
if key == 'float32':
if _dtype == np.float32:
return cont_f(&random_standard_gamma_f, &self._bitgen, size, self.lock,
shape, 'shape', CONS_NON_NEGATIVE,
out)
else:
raise TypeError('Unsupported dtype "%s" for standard_gamma' % key)
raise TypeError('Unsupported dtype %r for standard_gamma' % _dtype)

def gamma(self, shape, scale=1.0, size=None):
"""
Expand Down
30 changes: 15 additions & 15 deletions numpy/random/mtrand.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ from libc.stdint cimport int64_t, uint64_t
from ._bounded_integers cimport (_rand_bool, _rand_int32, _rand_int64,
_rand_int16, _rand_int8, _rand_uint64, _rand_uint32, _rand_uint16,
_rand_uint8,)
from ._bounded_integers import _integers_types
from ._mt19937 import MT19937 as _MT19937
from numpy.random cimport bitgen_t
from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE,
Expand Down Expand Up @@ -724,17 +723,16 @@ cdef class RandomState:
high = low
low = 0

dt = np.dtype(dtype)
key = dt.name
if key not in _integers_types:
raise TypeError('Unsupported dtype "%s" for randint' % key)
if not dt.isnative:
_dtype = np.dtype(dtype)

if not _dtype.isnative:
# numpy 1.17.0, 2019-05-28
warnings.warn('Providing a dtype with a non-native byteorder is '
'not supported. If you require platform-independent '
'byteorder, call byteswap when required.\nIn future '
'version, providing byteorder will raise a '
'ValueError', DeprecationWarning)
_dtype = _dtype.newbyteorder()

# Implementation detail: the use a masked method to generate
# bounded uniform integers. Lemire's method is preferable since it is
Expand All @@ -743,24 +741,26 @@ cdef class RandomState:
cdef bint _masked = True
cdef bint _endpoint = False

if key == 'int32':
if _dtype == np.int32:
ret = _rand_int32(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'int64':
elif _dtype == np.int64:
ret = _rand_int64(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'int16':
elif _dtype == np.int16:
ret = _rand_int16(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'int8':
elif _dtype == np.int8:
ret = _rand_int8(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'uint64':
elif _dtype == np.uint64:
ret = _rand_uint64(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'uint32':
elif _dtype == np.uint32:
ret = _rand_uint32(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'uint16':
elif _dtype == np.uint16:
ret = _rand_uint16(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'uint8':
elif _dtype == np.uint8:
ret = _rand_uint8(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'bool':
elif _dtype == np.bool_:
ret = _rand_bool(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
else:
raise TypeError('Unsupported dtype %r for randint' % _dtype)

if size is None and dtype in (bool, int, np.compat.long):
if np.array(ret).shape == ():
Expand Down
0