8000 MAINT: Large overhead in some random functions by przemb · Pull Request #15511 · numpy/numpy · GitHub
[go: up one dir, main page]
Skip to content

MAINT: Large overhead in some random functions #15511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 6, 2020
Merged

MAINT: Large overhead in some random functions #15511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a15dc30
MAINT: Large overhead in some random functions #15460
przemb Feb 5, 2020
df27b88
Changes suggested in review
przemb Feb 5, 2020
deb8571
Documentation updated.
przemb Feb 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 0 additions & 10 deletions numpy/random/_bounded_integers.pyx.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,6 @@ cdef extern from "numpy/random/distributions.h":
np.npy_bool *out) nogil



_integers_types = {'bool': (0, 2),
'int8': (-2**7, 2**7),
'int16': (-2**15, 2**15),
'int32': (-2**31, 2**31),
'int64': (-2**63, 2**63),
'uint8': (0, 2**8),
'uint16': (0, 2**16),
'uint32': (0, 2**32),
'uint64': (0, 2**64)}
{{
py:
type_info = (('uint32', 'uint32', 'uint64', 'NPY_UINT64', 0, 0, 0, '0X100000000ULL'),
Expand Down
116 changes: 54 additions & 62 deletions numpy/random/_generator.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ from libc.stdint cimport (uint8_t, uint16_t, uint32_t, uint64_t,
from ._bounded_integers cimport (_rand_bool, _rand_int32, _rand_int64,
_rand_int16, _rand_int8, _rand_uint64, _rand_uint32, _rand_uint16,
_rand_uint8, _gen_mask)
from ._bounded_integers import _integers_types
from ._pcg64 import PCG64
from numpy.random cimport bitgen_t
from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE,
Expand Down Expand Up @@ -262,7 +261,7 @@ cdef class Generator:

def random(self, size=None, dtype=np.float64, out=None):
"""
random(size=None, dtype='d', out=None)
random(size=None, dtype=np.float64, out=None)

Return random floats in the half-open interval [0.0, 1.0).

Expand All @@ -278,10 +277,9 @@ cdef class Generator:
Output shape. If the given shape is, e.g., ``(m, n, k)``, then
``m * n * k`` samples are drawn. Default is None, in which case a
single value is returned.
dtype : {str, dtype}, optional
Desired dtype of the result, either 'd' (or 'float64') or 'f'
(or 'float32'). All dtypes are determined by their name. The
default value is 'd'.
dtype : dtype, optional
Desired dtype of the result, only `float64` and `float32` are supported.
Byteorder must be native. The default value is np.float64.
out : ndarray, optional
Alternative output array in which to place the result. If size is not None,
it must have the same shape as the provided size and must match the type of
Expand Down Expand Up @@ -312,13 +310,13 @@ cdef class Generator:

"""
cdef double temp
key = np.dtype(dtype).name
if key == 'float64':
_dtype = np.dtype(dtype)
Copy link
Member
@eric-wieser eric-wieser Feb 5, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would normally spell this

Suggested change
_dtype = np.dtype(dtype)
dtype = np.dtype(dtype)

Does cython not like that?

(same throughout)

Copy link
Contributor Author

Choos 8000 e a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I change it, test are failing... seems that it is not proper.

Copy link
Member
@eric-wieser eric-wieser Feb 5, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you have a link to the CI failure?

Edit: caused diagnosed here: #15511 (comment)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need the original value so we can raise an error with it later. What about descr or concrete_dtype?

if _dtype == np.float64:
return double_fill(&random_standard_uniform_fill, &self._bitgen, size, self.lock, out)
elif key == 'float32':
elif _dtype == np.float32:
return float_fill(&random_standard_uniform_fill_f, &self._bitgen, size, self.lock, out)
else:
raise TypeError('Unsupported dtype "%s" for random' % key)
raise TypeError('Unsupported dtype %r for random' % _dtype)

def beta(self, a, b, size=None):
"""
Expand Down Expand Up @@ -417,7 +415,7 @@ cdef class Generator:

def standard_exponential(self, size=None, dtype=np.float64, method=u'zig', out=None):
"""
standard_exponential(size=None, dtype='d', method='zig', out=None)
standard_exponential(size=None, dtype=np.float64, method='zig', out=None)

Draw samples from the standard exponential distribution.

Expand All @@ -431,9 +429,8 @@ cdef class Generator:
``m * n * k`` samples are drawn. Default is None, in which case a
single value is returned.
dtype : dtype, optional
Desired dtype of the result, either 'd' (or 'float64') or 'f'
(or 'float32'). All dtypes are determined by their name. The
default value is 'd'.
Desired dtype of the result, only `float64` and `float32` are supported.
Byteorder must be native. The default value is np.float64.
method : str, optional
Either 'inv' or 'zig'. 'inv' uses the default inverse CDF method.
'zig' uses the much faster Ziggurat method of Marsaglia and Tsang.
Expand All @@ -454,24 +451,24 @@ cdef class Generator:
>>> n = np.random.default_rng().standard_exponential((3, 8000))

"""
key = np.dtype(dtype).name
if key == 'float64':
_dtype = np.dtype(dtype)
if _dtype == np.float64:
if method == u'zig':
return double_fill(&random_standard_exponential_fill, &self._bitgen, size, self.lock, out)
else:
return double_fill(&random_standard_exponential_inv_fill, &self._bitgen, size, self.lock, out)
elif key == 'float32':
elif _dtype == np.float32:
if method == u'zig':
return float_fill(&random_standard_exponential_fill_f, &self._bitgen, size, self.lock, out)
else:
return float_fill(&random_standard_exponential_inv_fill_f, &self._bitgen, size, self.lock, out)
else:
raise TypeError('Unsupported dtype "%s" for standard_exponential'
% key)
raise TypeError('Unsupported dtype %r for standard_exponential'
% _dtype)

def integers(self, low, high=None, size=None, dtype=np.int64, endpoint=False):
"""
integers(low, high=None, size=None, dtype='int64', endpoint=False)
integers(low, high=None, size=None, dtype=np.int64, endpoint=False)

Return random integers from `low` (inclusive) to `high` (exclusive), or
if endpoint=True, `low` (inclusive) to `high` (inclusive). Replaces
Expand All @@ -496,11 +493,9 @@ cdef class Generator:
Output shape. If the given shape is, e.g., ``(m, n, k)``, then
``m * n * k`` samples are drawn. Default is None, in which case a
single value is returned.
dtype : {str, dtype}, optional
Desired dtype of the result. All dtypes are determined by their
name, i.e., 'int64', 'int', etc, so byteorder is not available
and a specific precision may have different C types depending
on the platform. The default value is `np.int_`.
dtype : dtype, optional
Desired dtype of the result. Byteorder must be native.
The default value is np.int64.
endpoint : bool, optional
If true, sample from the interval [low, high] instead of the
default [low, high)
Expand Down Expand Up @@ -559,39 +554,39 @@ cdef class Generator:
high = low
low = 0

dt = np.dtype(dtype)
key = dt.name
if key not in _integers_types:
raise TypeError('Unsupported dtype "%s" for integers' % key)
if not dt.isnative:
raise ValueError('Providing a dtype with a non-native byteorder '
'is not supported. If you require '
'platform-independent byteorder, call byteswap '
'when required.')
_dtype = np.dtype(dtype)

# Implementation detail: the old API used a masked method to generate
# bounded uniform integers. Lemire's method is preferable since it is
# faster. randomgen allows a choice, we will always use the faster one.
cdef bint _masked = False

if key == 'int32':
if _dtype == np.int32:
ret = _rand_int32(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'int64':
elif _dtype == np.int64:
ret = _rand_int64(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'int16':
elif _dtype == np.int16:
ret = _rand_int16(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'int8':
elif _dtype == np.int8:
ret = _rand_int8(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'uint64':
elif _dtype == np.uint64:
ret = _rand_uint64(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'uint32':
elif _dtype == np.uint32:
ret = _rand_uint32(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'uint16':
elif _dtype == np.uint16:
ret = _rand_uint16(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'uint8':
elif _dtype == np.uint8:
ret = _rand_uint8(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif key == 'bool':
elif _dtype == np.bool_:
ret = _rand_bool(low, high, size, _masked, endpoint, &self._bitgen, self.lock)
elif not _dtype.isnative:
raise ValueError('Providing a dtype with a non-native byteorder '
'is not supported. If you require '
'platform-independent byteorder, call byteswap '
'when required.')
else:
raise TypeError('Unsupported dtype %r for integers' % _dtype)


if size is None and dtype in (bool, int, np.compat.long):
if np.array(ret).shape == ():
Expand Down Expand Up @@ -980,7 +975,7 @@ cdef class Generator:
# Complicated, continuous distributions:
def standard_normal(self, size=None, dtype=np.float64, out=None):
"""
standard_normal(size=None, dtype='d', out=None)
standard_normal(size=None, dtype=np.float64, out=None)

Draw samples from a standard Normal distribution (mean=0, stdev=1).

Expand All @@ -990,10 +985,9 @@ cdef class Generator:
Output shape. If the given shape is, e.g., ``(m, n, k)``, then
``m * n * k`` samples are drawn. Default is None, in which case a
single value is returned.
dtype : {str, dtype}, optional
Desired dtype of the result, either 'd' (or 'float64') or 'f'
(or 'float32'). All dtypes are determined by their name. The
default value is 'd'.
dtype : dtype, optional
Desired dtype of the result, only `float64` and `float32` are supported.
Byteorder must be native. The default value is np.float64.
out : ndarray, optional
Alternative output array in which to place the result. If size is not None,
it must have the same shape as the provided size and must match the type of
Expand Down Expand Up @@ -1041,14 +1035,13 @@ cdef class Generator:
[ 0.39924804, 4.68456316, 4.99394529, 4.84057254]]) # random

"""
key = np.dtype(dtype).name
if key == 'float64':
_dtype = np.dtype(dtype)
if _dtype == np.float64:
return double_fill(&random_standard_normal_fill, &self._bitgen, size, self.lock, out)
elif key == 'float32':
elif _dtype == np.float32:
return float_fill(&random_standard_normal_fill_f, &self._bitgen, size, self.lock, out)

else:
raise TypeError('Unsupported dtype "%s" for standard_normal' % key)
raise TypeError('Unsupported dtype %r for standard_normal' % _dtype)

def normal(self, loc=0.0, scale=1.0, size=None):
"""
Expand Down Expand Up @@ -1154,7 +1147,7 @@ cdef class Generator:

def standard_gamma(self, shape, size=None, dtype=np.float64, out=None):
"""
standard_gamma(shape, size=None, dtype='d', out=None)
standard_gamma(shape, size=None, dtype=np.float64, out=None)

Draw samples from a standard Gamma distribution.

Expand All @@ -1170,10 +1163,9 @@ cdef class Generator:
``m * n * k`` samples are drawn. If size is ``None`` (default),
a single value is returned if ``shape`` is a scalar. Otherwise,
``np.array(shape).size`` samples are drawn.
dtype : {str, dtype}, optional
Desired dtype of the result, either 'd' (or 'float64') or 'f'
(or 'float32'). All dtypes are determined by their name. The
default value is 'd'.
dtype : dtype, optional
Desired dtype of the result, only `float64` and `float32` are supported.
Byteorder must be native. The default value is np.float64.
out : ndarray, optional
Alternative output array in which to place the result. If size is
not None, it must have the same shape as the provided size and
Expand Down Expand Up @@ -1230,19 +1222,19 @@ cdef class Generator:

"""
cdef void *func
key = np.dtype(dtype).name
if key == 'float64':
_dtype = np.dtype(dtype)
if _dtype == np.float64:
return cont(&random_standard_gamma, &self._bitgen, size, self.lock, 1,
shape, 'shape', CONS_NON_NEGATIVE,
0.0, '', CONS_NONE,
0.0, '', CONS_NONE,
out)
if key == 'float32':
if _dtype == np.float32:
return cont_f(&random_standard_gamma_f, &self._bitgen, size, self.lock,
shape, 'shape', CONS_NON_NEGATIVE,
out)
else:
raise TypeError('Unsupported dtype "%s" for standard_gamma' % key)
raise TypeError('Unsupported dtype %r for standard_gamma' % _dtype)

def gamma(self, shape, scale=1.0, size=None):
"""
Expand Down
38 changes: 18 additions & 20 deletions numpy/random/mtrand.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ from libc.stdint cimport int64_t, uint64_t
from ._bounded_integers cimport (_rand_bool, _rand_int32, _rand_int64,
_rand_int16, _rand_int8, _rand_uint64, _rand_uint32, _rand_uint16,
_rand_uint8,)
from ._bounded_integers import _integers_types
from ._mt19937 import MT19937 as _MT19937
from numpy.random cimport bitgen_t
from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE,
Expand Down Expand Up @@ -643,7 +642,7 @@ cdef class RandomState:

def randint(self, low, high=None, size=None, dtype=int):
"""
randint(low, high=None, size=None, dtype='l')
randint(low, high=None, size=None, dtype=int)

Return random integers from `low` (inclusive) to `high` (exclusive).

Expand All @@ -670,10 +669,8 @@ cdef class RandomState:
``m * n * k`` samples are drawn. Default is None, in which case a
single value is returned.
dtype : dtype, optional
Desired dtype of the result. All dtypes are determined by their
name, i.e., 'int64', 'int', etc, so byteorder is not available
and a specific precision may have different C types depending
on the platform. The default value is `np.int_`.
Desired dtype of the result. Byteorder must be native.
The default value is int.

.. versionadded:: 1.11.0

Expand Down Expand Up @@ -724,17 +721,16 @@ cdef class RandomState:
high = low
low = 0

dt = np.dtype(dtype)
key = dt.name
if key not in _integers_types:
raise TypeError('Unsupported dtype "%s" for randint' % key)
if not dt.isnative:
_dtype = np.dtype(dtype)

if not _dtype.isnative:
# numpy 1.17.0, 2019-05-28
warnings.warn('Providing a dtype with a non-native byteorder is '
'not supported. If you require platform-independent '
'byteorder, call byteswap when required.\nIn future '
'version, providing byteorder will raise a '
'ValueError', DeprecationWarning)
_dtype = _dtype.newbyteorder()

# Implementation detail: the use a masked method to generate
# bounded uniform integers. Lemire's method is preferable since it is
Expand All @@ -743,24 +739,26 @@ cdef class RandomState:
cdef bint _masked = True
cdef bint _endpoint = False

if key == 'int32':
if _dtype == np.int32:
ret = _rand_int32(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'int64':
elif _dtype == np.int64:
ret = _rand_int64(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'int16':
elif _dtype == np.int16:
ret = _rand_int16(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'int8':
elif _dtype == np.int8:
ret = _rand_int8(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'uint64':
elif _dtype == np.uint64:
ret = _rand_uint64(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'uint32':
elif _dtype == np.uint32:
ret = _rand_uint32(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'uint16':
elif _dtype == np.uint16:
ret = _rand_uint16(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'uint8':
elif _dtype == np.uint8:
ret = _rand_uint8(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
elif key == 'bool':
elif _dtype == np.bool_:
ret = _rand_bool(low, high, size, _masked, _endpoint, &self._bitgen, self.lock)
else:
raise TypeError('Unsupported dtype %r for randint' % _dtype)

if size is None and dtype in (bool, int, np.compat.long):
if np.array(ret).shape == ():
Expand Down
0