8000 ENH: random: Add the method `permuted` to Generator. by WarrenWeckesser · Pull Request #15121 · numpy/numpy · GitHub
[go: up one dir, main page]
Skip to content

ENH: random: Add the method permuted to Generator. #15121

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Sep 2, 2020
Merged

ENH: random: Add the method permuted to Generator. #15121

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
a84cdf0
ENH: random: Make _shuffle_raw and _shuffle_int standalone functions.
WarrenWeckesser Dec 15, 2019
081298c
ENH: random: Add the method `permuted` to Generator.
WarrenWeckesser Dec 15, 2019
7fc1136
Use PyMem_Malloc/_Free instead of malloc/free
WarrenWeckesser Aug 13, 2020
bc49d4b
Raise TypeError instead ValueError when out is not an ndarray.
WarrenWeckesser Aug 13, 2020
3dbcbae
Use order='K' when copying x to out.
WarrenWeckesser Aug 13, 2020
50c7611
Require out.shape be the same as x.shape
WarrenWeckesser Aug 13, 2020
023114f
Handle axis=None more efficiently.
WarrenWeckesser Aug 14, 2020
91d208a
DOC: Overhaul section on shuffling methods.
WarrenWeckesser Aug 20, 2020
95eff00
Use np.copyto instead of out[...] = x
WarrenWeckesser Aug 20, 2020
677f862
Don't release the GIL when shuffling an object array.
WarrenWeckesser Aug 20, 2020
f5d6f0b
DOC: Fix examples and move method summary to the top.
WarrenWeckesser Aug 20, 2020
f016d3a
Fix C prototype declaration to fix compiler warning.
WarrenWeckesser Aug 20, 2020
ebed0e8
Refine the C prototype to eliminate another compiler warning.
WarrenWeckesser Aug 20, 2020
785e239
Remove cimport of unused malloc and free.
WarrenWeckesser Aug 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Don't release the GIL when shuffling an object array.
  • Loading branch information
@WarrenWeckesser
WarrenWeckesser committed Aug 20, 2020
commit 677f8626a4a1cd9eae9d312058b565f66d8e289e
56 changes: 41 additions & 15 deletions numpy/random/_generator.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,19 @@ cdef int64_t _safe_sum_nonneg_int64(size_t num_colors, int64_t *colors):
return sum


cdef inline void _shuffle_raw_wrap(bitgen_t *bitgen, np.npy_intp n,
np.npy_intp first, np.npy_intp itemsize,
np.npy_intp stride,
char* data, char* buf) nogil:
# We trick gcc into providing a specialized implementation for
# the most common case, yielding a ~33% performance improvement.
# Note that apparently, only one branch can ever be specialized.
if itemsize == sizeof(np.npy_intp):
_shuffle_raw(bitgen, n, first, sizeof(np.npy_intp), stride, data, buf)
else:
_shuffle_raw(bitgen, n, first, itemsize, stride, data, buf)


cdef inline void _shuffle_raw(bitgen_t *bitgen, np.npy_intp n,
np.npy_intp first, np.npy_intp itemsize,
np.npy_intp stride,
Expand All @@ -83,6 +96,7 @@ cdef inline void _shuffle_raw(bitgen_t *bitgen, np.npy_intp n,
Location of buffer (itemsize)
"""
cdef np.npy_intp i, j

for i in reversed(range(first, n)):
j = random_interval(bitgen, i)
string.memcpy(buf, data + j * stride, itemsize)
Expand Down Expand Up @@ -4311,11 +4325,24 @@ cdef class Generator:
if buf == NULL:
raise MemoryError('memory allocation failed in permuted')

with self.lock, nogil:
while np.PyArray_ITER_NOTDONE(it):
_shuffle_raw(&self._bitgen, axlen, 0, itemsize, axstride,
<char *>np.PyArray_ITER_DATA(it), <char *>buf)
np.PyArray_ITER_NEXT(it)
if out.dtype.hasobject:
# Keep the GIL when shuffling an object array.
with self.lock:
while np.PyArray_ITER_NOTDONE(it):
_shuffle_raw_wrap(&self._bitgen, axlen, 0, itemsize,
axstride,
<char *>np.PyArray_ITER_DATA(it),
<char *>buf)
np.PyArray_ITER_NEXT(it)
else:
# out is not an object array, so we can release the GIL.
with self.lock, nogil:
while np.PyArray_ITER_NOTDONE(it):
_shuffle_raw_wrap(&self._bitgen, axlen, 0, itemsize,
axstride,
<char *>np.PyArray_ITER_DATA(it),
<char *>buf)
np.PyArray_ITER_NEXT(it)

PyMem_Free(buf)
return out
Expand Down Expand Up @@ -4382,16 +4409,15 @@ cdef class Generator:
# when the function exits.
buf = np.empty(itemsize, dtype=np.int8) # GC'd at function exit
buf_ptr = <char*><size_t>np.PyArray_DATA(buf)
with self.lock, nogil:
# We trick gcc into providing a specialized implementation for
# the most common case, yielding a ~33% performance improvement.
# Note that apparently, only one branch can ever be specialized.
if itemsize == sizeof(np.npy_intp):
_shuffle_raw(&self._bitgen, n, 1, sizeof(np.npy_intp),
stride, x_ptr, buf_ptr)
else:
_shuffle_raw(&self._bitgen, n, 1, itemsize, stride, x_ptr,
buf_ptr)
if x.dtype.hasobject:
with self.lock:
_shuffle_raw_wrap(&self._bitgen, n, 1, itemsize, stride,
x_ptr, buf_ptr)
else:
# Same as above, but the GIL is released.
with self.lock, nogil:
_shuffle_raw_wrap(&self._bitgen, n, 1, itemsize, stride,
x_ptr, b 8000 uf_ptr)
elif isinstance(x, np.ndarray) and x.ndim and x.size:
x = np.swapaxes(x, 0, axis)
buf = np.empty_like(x[0, ...])
Expand Down
8 changes: 5 additions & 3 deletions numpy/random/tests/test_generator_mt19937.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1039,22 +1039,24 @@ def test_permutation_exceptions(self):
assert_raises(np.AxisError, random.permutation, arr, 3)
assert_raises(TypeError, random.permutation, arr, slice(1, 2, None))

@pytest.mark.parametrize("dtype", [int, object])
@pytest.mark.parametrize("axis, expected",
[(None, np.array([[3, 7, 0, 9, 10, 11],
[8, 4, 2, 5, 1, 6]])),
(0, np.array([[6, 1, 2, 9, 10, 11],
[0, 7, 8, 3, 4, 5]])),
(1, np.array([[ 5, 3, 4, 0, 2, 1],
[11, 9, 10, 6, 8, 7]]))])
def test_permuted(self, axis, expected):
def test_permuted(self, dtype, axis, expected):
random = Generator(MT19937(self.seed))
x = np.arange(12).reshape(2, 6)
x = np.arange(12).reshape(2, 6).astype(dtype)
random.permuted(x, axis=axis, out=x)
assert_array_equal(x, expected)

51CD random = Generator(MT19937(self.seed))
x = np.arange(12).reshape(2, 6)
x = np.arange(12).reshape(2, 6).astype(dtype)
y = random.permuted(x, axis=axis)
assert y.dtype == dtype
assert_array_equal(y, expected)

def test_permuted_with_strides(self):
Expand Down
0