diff --git a/doc/release/1.17.3-notes.rst b/doc/release/1.17.3-notes.rst index b85e86442923..9f20c2fdccbd 100644 --- a/doc/release/1.17.3-notes.rst +++ b/doc/release/1.17.3-notes.rst @@ -23,6 +23,10 @@ Compatibility notes =================== - The seldom used ``PyArray_DescrCheck`` macro has been changed/fixed. +- The use of the new ``numpy.random`` features from Cython and Numba + was not well documented and parts have been removed or refactored. + We plan to finish the refactor and fully document it in 1.18.0 + Contributors diff --git a/doc/source/reference/random/extending.rst b/doc/source/reference/random/extending.rst deleted file mode 100644 index 22f9cb7e47a1..000000000000 --- a/doc/source/reference/random/extending.rst +++ /dev/null @@ -1,165 +0,0 @@ -.. currentmodule:: numpy.random - -Extending ---------- -The BitGenerators have been designed to be extendable using standard tools for -high-performance Python -- numba and Cython. The `~Generator` object can also -be used with user-provided BitGenerators as long as these export a small set of -required functions. - -Numba -===== -Numba can be used with either CTypes or CFFI. The current iteration of the -BitGenerators all export a small set of functions through both interfaces. - -This example shows how numba can be used to produce Box-Muller normals using -a pure Python implementation which is then compiled. The random numbers are -provided by ``ctypes.next_double``. - -.. code-block:: python - - from numpy.random import PCG64 - import numpy as np - import numba as nb - - x = PCG64() - f = x.ctypes.next_double - s = x.ctypes.state - state_addr = x.ctypes.state_address - - def normals(n, state): - out = np.empty(n) - for i in range((n+1)//2): - x1 = 2.0*f(state) - 1.0 - x2 = 2.0*f(state) - 1.0 - r2 = x1*x1 + x2*x2 - while r2 >= 1.0 or r2 == 0.0: - x1 = 2.0*f(state) - 1.0 - x2 = 2.0*f(state) - 1.0 - r2 = x1*x1 + x2*x2 - g = np.sqrt(-2.0*np.log(r2)/r2) - out[2*i] = g*x1 - if 2*i+1 < n: - out[2*i+1] = g*x2 - return out - - # Compile using Numba - print(normals(10, s).var()) - # Warm up - normalsj = nb.jit(normals, nopython=True) - # Must use state address not state with numba - normalsj(1, state_addr) - %timeit normalsj(1000000, state_addr) - print('1,000,000 Box-Muller (numba/PCG64) randoms') - %timeit np.random.standard_normal(1000000) - print('1,000,000 Box-Muller (NumPy) randoms') - - -Both CTypes and CFFI allow the more complicated distributions to be used -directly in Numba after compiling the file distributions.c into a DLL or so. -An example showing the use of a more complicated distribution is in the -examples folder. - -.. _randomgen_cython: - -Cython -====== - -Cython can be used to unpack the ``PyCapsule`` provided by a BitGenerator. -This example uses `~pcg64.PCG64` and -``random_gauss_zig``, the Ziggurat-based generator for normals, to fill an -array. The usual caveats for writing high-performance code using Cython -- -removing bounds checks and wrap around, providing array alignment information --- still apply. - -.. code-block:: cython - - import numpy as np - cimport numpy as np - cimport cython - from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer - from numpy.random.common cimport * - from numpy.random.distributions cimport random_gauss_zig - from numpy.random import PCG64 - - - @cython.boundscheck(False) - @cython.wraparound(False) - def normals_zig(Py_ssize_t n): - cdef Py_ssize_t i - cdef bitgen_t *rng - cdef const char *capsule_name = "BitGenerator" - cdef double[::1] random_values - - x = PCG64() - capsule = x.capsule - if not PyCapsule_IsValid(capsule, capsule_name): - raise ValueError("Invalid pointer to anon_func_state") - rng = PyCapsule_GetPointer(capsule, capsule_name) - random_values = np.empty(n) - # Best practice is to release GIL and acquire the lock - with x.lock, nogil: - for i in range(n): - random_values[i] = random_gauss_zig(rng) - randoms = np.asarray(random_values) - return randoms - -The BitGenerator can also be directly accessed using the members of the basic -RNG structure. - -.. code-block:: cython - - @cython.boundscheck(False) - @cython.wraparound(False) - def uniforms(Py_ssize_t n): - cdef Py_ssize_t i - cdef bitgen_t *rng - cdef const char *capsule_name = "BitGenerator" - cdef double[::1] random_values - - x = PCG64() - capsule = x.capsule - # Optional check that the capsule if from a BitGenerator - if not PyCapsule_IsValid(capsule, capsule_name): - raise ValueError("Invalid pointer to anon_func_state") - # Cast the pointer - rng = PyCapsule_GetPointer(capsule, capsule_name) - random_values = np.empty(n) - with x.lock, nogil: - for i in range(n): - # Call the function - random_values[i] = rng.next_double(rng.state) - randoms = np.asarray(random_values) - return randoms - -These functions along with a minimal setup file are included in the -examples folder. - -New Basic RNGs -============== -`~Generator` can be used with other user-provided BitGenerators. The simplest -way to write a new BitGenerator is to examine the pyx file of one of the -existing BitGenerators. The key structure that must be provided is the -``capsule`` which contains a ``PyCapsule`` to a struct pointer of type -``bitgen_t``, - -.. code-block:: c - - typedef struct bitgen { - void *state; - uint64_t (*next_uint64)(void *st); - uint32_t (*next_uint32)(void *st); - double (*next_double)(void *st); - uint64_t (*next_raw)(void *st); - } bitgen_t; - -which provides 5 pointers. The first is an opaque pointer to the data structure -used by the BitGenerators. The next three are function pointers which return -the next 64- and 32-bit unsigned integers, the next random double and the next -raw value. This final function is used for testing and so can be set to -the next 64-bit unsigned integer function if not needed. Functions inside -``Generator`` use this structure as in - -.. code-block:: c - - bitgen_state->next_uint64(bitgen_state->state) diff --git a/doc/source/reference/random/index.rst b/doc/source/reference/random/index.rst index b0283f3a7d6d..641c2164a2e3 100644 --- a/doc/source/reference/random/index.rst +++ b/doc/source/reference/random/index.rst @@ -152,10 +152,6 @@ What's New or Different * Optional ``out`` argument that allows existing arrays to be filled for select distributions * All BitGenerators can produce doubles, uint64s and uint32s via CTypes - (`~.PCG64.ctypes`) and CFFI (`~.PCG64.cffi`). This allows the bit generators - to be used in numba. -* The bit generators can be used in downstream projects via - :ref:`Cython `. * `~.Generator.integers` is now the canonical way to generate integer random numbers from a discrete uniform distribution. The ``rand`` and ``randn`` methods are only available through the legacy `~.RandomState`. @@ -199,7 +195,6 @@ Features Multithreaded Generation new-or-different Comparing Performance - extending Original Source ~~~~~~~~~~~~~~~ diff --git a/doc/source/reference/random/new-or-different.rst b/doc/source/reference/random/new-or-different.rst index c8815f98f06c..402f24ab0ba9 100644 --- a/doc/source/reference/random/new-or-different.rst +++ b/doc/source/reference/random/new-or-different.rst @@ -56,13 +56,12 @@ And in more detail: ``randn`` methods are only available through the legacy `~.RandomState`. This replaces both ``randint`` and the deprecated ``random_integers``. * The Box-Muller method used to produce NumPy's normals is no longer available. -* All bit generators can produce doubles, uint64s and - uint32s via CTypes (`~PCG64.ctypes`) and CFFI (`~PCG64.cffi`). - This allows these bit generators to be used in numba. +* All bit generators can produce doubles, uint64s and uint32s via CTypes + (`~PCG64.ctypes`) and CFFI (`~PCG64.cffi`). This allows these bit generators + to be used in numba. * The bit generators can be used in downstream projects via Cython. - .. ipython:: python from numpy.random import Generator, PCG64 diff --git a/numpy/random/common.pxd b/numpy/random/common.pxd index 2f7baa06e0c6..ac0a94bb0514 100644 --- a/numpy/random/common.pxd +++ b/numpy/random/common.pxd @@ -5,7 +5,7 @@ from libc.stdint cimport (uint8_t, uint16_t, uint32_t, uint64_t, uintptr_t) from libc.math cimport sqrt -cdef extern from "numpy/random/bitgen.h": +cdef extern from "src/bitgen.h": struct bitgen: void *state uint64_t (*next_uint64)(void *st) nogil diff --git a/numpy/random/setup.py b/numpy/random/setup.py index f0ebe331fa77..ce7f0565f397 100644 --- a/numpy/random/setup.py +++ b/numpy/random/setup.py @@ -34,8 +34,6 @@ def generate_libraries(ext, build_dir): defs.append(('NPY_NO_DEPRECATED_API', 0)) config.add_data_dir('tests') - config.add_data_files('common.pxd') - config.add_data_files('bit_generator.pxd') EXTRA_LINK_ARGS = [] # Math lib diff --git a/numpy/core/include/numpy/random/bitgen.h b/numpy/random/src/bitgen.h similarity index 100% rename from numpy/core/include/numpy/random/bitgen.h rename to numpy/random/src/bitgen.h diff --git a/numpy/random/src/distributions/distributions.h b/numpy/random/src/distributions/distributions.h index f2c370c07e4c..b778968d7de1 100644 --- a/numpy/random/src/distributions/distributions.h +++ b/numpy/random/src/distributions/distributions.h @@ -9,7 +9,7 @@ #include "Python.h" #include "numpy/npy_common.h" #include "numpy/npy_math.h" -#include "numpy/random/bitgen.h" +#include "src/bitgen.h" /* * RAND_INT_TYPE is used to share integer generators with RandomState which