8000 ENH: implement nep 0015: merge multiarray and umath by mattip · Pull Request #10915 · numpy/numpy · GitHub
[go: up one dir, main page]

Skip to content

ENH: implement nep 0015: merge multiarray and umath #10915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Aug 31, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ recursive-include numpy/_build_utils *
recursive-include numpy/linalg/lapack_lite *.c *.h
include tox.ini
# Add sdist files whose use depends on local configuration.
include numpy/core/src/multiarray/cblasfuncs.c
include numpy/core/src/multiarray/python_xerbla.c
include numpy/core/src/common/cblasfuncs.c
include numpy/core/src/common/python_xerbla.c
# Adding scons build related files not found by distutils
recursive-include numpy/core/code_generators *.py *.txt
recursive-include numpy/core *.in *.h
Expand Down
9 changes: 9 additions & 0 deletions doc/release/1.16.0-notes.rst
Original file line number Diff line number Diff line change
Expand Up @@ -107,3 +107,12 @@ raise a ``DeprecationWarning`` if the array is not numerical (i.e.,
if ``np.positive(array)`` raises a ``TypeError``. For ``ndarray``
subclasses that override the default ``__array_ufunc__`` implementation,
the ``TypeError`` is passed on.
Umath and multiarray c-extension modules merged into a single module
--------------------------------------------------------------------

The two modules were merged, according to the first step in `NEP 15`_.
Previously `np.core.umath` and `np.core.multiarray` were the c-extension
modules, they are now python wrappers to the single `np.core/_multiarray_math`
c-extension module.

.. _`NEP 15` : http://www.numpy.org/neps/nep-0015-merge-multiarray-umath.html
4 changes: 2 additions & 2 deletions numpy/core/code_generators/generate_numpy_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@
_import_array(void)
{
int st;
PyObject *numpy = PyImport_ImportModule("numpy.core.multiarray");
PyObject *numpy = PyImport_ImportModule("numpy.core._multiarray_umath");
PyObject *c_api = NULL;

if (numpy == NULL) {
PyErr_SetString(PyExc_ImportError, "numpy.core.multiarray failed to import");
PyErr_SetString(PyExc_ImportError, "numpy.core._multiarray_umath failed to import");
return -1;
}
c_api = PyObject_GetAttrString(numpy, "_ARRAY_API");
Expand Down
5 changes: 3 additions & 2 deletions numpy/core/code_generators/generate_ufunc_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,12 @@
static NPY_INLINE int
_import_umath(void)
{
PyObject *numpy = PyImport_ImportModule("numpy.core.umath");
PyObject *numpy = PyImport_ImportModule("numpy.core._multiarray_umath");
PyObject *c_api = NULL;

if (numpy == NULL) {
PyErr_SetString(PyExc_ImportError, "numpy.core.umath failed to import");
PyErr_SetString(PyExc_ImportError,
"numpy.core._multiarray_umath failed to import");
return -1;
}
c_api = PyObject_GetAttrString(numpy, "_UFUNC_API");
Expand Down
2 changes: 2 additions & 0 deletions numpy/core/code_generators/generate_umath.py
Original file line number Diff line number Diff line change
Expand Up @@ -1080,6 +1080,8 @@ def make_code(funcdict, filename):
Please make changes to the code generator program (%s)
**/
#include "cpuid.h"
#include "ufunc_object.h"
#include "ufunc_type_resolution.h"
%s

static int
Expand Down
34 changes: 34 additions & 0 deletions numpy/core/multiarray.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
"""
Create the numpy.core.multiarray namespace for backward compatibility. In v1.16
the multiarray and umath c-extension modules were merged into a single
_multiarray_umath extension module. So we replicate the old namespace
by importing from the extension module.
"""

from . import _multiarray_umath
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this file could do with either a docstring or a comment, explaining why it exists and is just a bunch of imports

from numpy.core._multiarray_umath import *
from numpy.core._multiarray_umath import (_fastCopyAndTranspose, _flagdict, _insert,
_reconstruct, _vec_string, _ARRAY_API, _monotonicity)

__all__ = ['_ARRAY_API', 'ALLOW_THREADS 10000 9;, 'BUFSIZE', 'CLIP', 'DATETIMEUNITS',
'ITEM_HASOBJECT', 'ITEM_IS_POINTER', 'LIST_PICKLE', 'MAXDIMS',
'MAY_SHARE_BOUNDS', 'MAY_SHARE_EXACT', 'NEEDS_INIT', 'NEEDS_PYAPI',
'RAISE', 'USE_GETITEM', 'USE_SETITEM', 'WRAP',
'_fastCopyAndTranspose', '_flagdict', '_insert', '_reconstruct',
'_vec_string', '_monotonicity',
'add_docstring', 'arange', 'array', 'bincount', 'broadcast', 'busday_count',
'busday_offset', 'busdaycalendar', 'can_cast', 'compare_chararrays',
'concatenate', 'copyto', 'correlate', 'correlate2', 'count_nonzero',
'c_einsum', 'datetime_as_string', 'datetime_data', 'digitize', 'dot',
'dragon4_positional', 'dragon4_scientific', 'dtype', 'empty', 'empty_like',
'error', 'flagsobj', 'flatiter', 'format_longfloat', 'frombuffer',
'fromfile', 'fromiter', 'fromstring', 'getbuffer', 'inner', 'int_asbuffer',
'interp', 'interp_complex', 'is_busday', 'lexsort', 'matmul',
'may_share_memory', 'min_scalar_type', 'ndarray', 'nditer', 'nested_iters',
'newbuffer', 'normalize_axis_index', 'packbits', 'promote_types',
'putmask', 'ravel_multi_index', 'result_type', 'scalar',
'set_datetimeparse_function', 'set_legacy_print_mode', 'set_numeric_ops',
'set_string_function', 'set_typeDict', 'shares_memory', 'test_interrupt',
'tracemalloc_domain', 'typeinfo', 'unpackbits', 'unravel_index', 'vdot',
'where', 'zeros']

5 changes: 1 addition & 4 deletions numpy/core/numeric.py
Original file line number Diff line number Diff line change
Expand Up @@ -2919,10 +2919,7 @@ def _setdef():

def extend_all(module):
existing = set(__all__)
try:
mall = getattr(module, '__all__')
except AttributeError:
mall = [k for k in module.__dict__.keys() if not k.startswith('_')]
mall = getattr(module, '__all__')
for a in mall:
if a not in existing:
__all__.append(a)
Expand Down
138 changes: 69 additions & 69 deletions numpy/core/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -514,9 +514,9 @@ def generate_config_h(ext, build_dir):

def generate_numpyconfig_h(ext, build_dir):
"""Depends on config.h: generate_config_h has to be called before !"""
# put private include directory in build_dir on search path
# put common include directory in build_dir on search path
# allows using code generation in headers headers
config.add_include_dirs(join(build_dir, "src", "private"))
config.add_include_dirs(join(build_dir, "src", "common"))
config.add_include_dirs(join(build_dir, "src", "npymath"))

target = join(build_dir, header_dir, '_numpyconfig.h')
Expand Down Expand Up @@ -603,7 +603,7 @@ def generate_api(ext, build_dir):
generate_numpy_api = generate_api_func('generate_numpy_api')
generate_ufunc_api = generate_api_func('generate_ufunc_api')

config.add_include_dirs(join(local_dir, "src", "private"))
config.add_include_dirs(join(local_dir, "src", "common"))
config.add_include_dirs(join(local_dir, "src"))
config.add_include_dirs(join(local_dir))

Expand Down Expand Up @@ -700,26 +700,65 @@ def get_mathlib_info(*args):
npysort_sources = [join('src', 'npysort', 'quicksort.c.src'),
join('src', 'npysort', 'mergesort.c.src'),
join('src', 'npysort', 'heapsort.c.src'),
join('src', 'private', 'npy_partition.h.src'),
join('src', 'common', 'npy_partition.h.src'),
join('src', 'npysort', 'selection.c.src'),
join('src', 'private', 'npy_binsearch.h.src'),
join('src', 'common', 'npy_binsearch.h.src'),
join('src', 'npysort', 'binsearch.c.src'),
]
config.add_library('npysort',
sources=npysort_sources,
include_dirs=[])

#######################################################################
# multiarray module #
# _multiarray_umath module - common part #
#######################################################################

common_deps = [
join('src', 'common', 'array_assign.h'),
join('src', 'common', 'binop_override.h'),
join('src', 'common', 'cblasfuncs.h'),
join('src', 'common', 'lowlevel_strided_loops.h'),
join('src', 'common', 'mem_overlap.h'),
join('src', 'common', 'npy_config.h'),
join('src', 'common', 'npy_extint128.h'),
join('src', 'common', 'npy_longdouble.h'),
join('src', 'common', 'templ_common.h.src'),
join('src', 'common', 'ucsnarrow.h'),
join('src', 'common', 'ufunc_override.h'),
join('src', 'common', 'umathmodule.h'),
]

common_src = [
join('src', 'common', 'array_assign.c'),
join('src', 'common', 'mem_overlap.c'),
join('src', 'common', 'npy_longdouble.c'),
join('src', 'common', 'templ_common.h.src'),
join('src', 'common', 'ucsnarrow.c'),
join('src', 'common', 'ufunc_override.c'),
]

blas_info = get_info('blas_opt', 0)
if blas_info and ('HAVE_CBLAS', None) in blas_info.get('define_macros', []):
extra_info = blas_info
# These files are also in MANIFEST.in so that they are always in
# the source distribution independently of HAVE_CBLAS.
common_src.extend([join('src', 'common', 'cblasfuncs.c'),
join('src', 'common', 'python_xerbla.c'),
])
if uses_accelerate_framework(blas_info):
common_src.extend(get_sgemv_fix())
else:
extra_info = {}

#######################################################################
# _multiarray_umath module - multiarray part #
#######################################################################

multiarray_deps = [
join('src', 'multiarray', 'arrayobject.h'),
join('src', 'multiarray', 'arraytypes.h'),
join('src', 'multiarray', 'array_assign.h'),
join('src', 'multiarray', 'buffer.h'),
join('src', 'multiarray', 'calculation.h'),
join('src', 'multiarray', 'cblasfuncs.h'),
join('src', 'multiarray', 'common.h'),
join('src', 'multiarray', 'convert_datatype.h'),
join('src', 'multiarray', 'convert.h'),
Expand All @@ -742,17 +781,8 @@ def get_mathlib_info(*args):
join('src', 'multiarray', 'shape.h'),
join('src', 'multiarray', 'strfuncs.h'),
join('src', 'multiarray', 'typeinfo.h'),
join('src', 'multiarray', 'ucsnarrow.h'),
join('src', 'multiarray', 'usertypes.h'),
join('src', 'multiarray', 'vdot.h'),
join('src', 'private', 'npy_config.h'),
join('src', 'private', 'templ_common.h.src'),
join('src', 'private', 'lowlevel_strided_loops.h'),
join('src', 'private', 'mem_overlap.h'),
join('src', 'private', 'npy_longdouble.h'),
join('src', 'private', 'ufunc_override.h'),
join('src', 'private', 'binop_override.h'),
join('src', 'private', 'npy_extint128.h'),
join('include', 'numpy', 'arrayobject.h'),
join('include', 'numpy', '_neighborhood_iterator_imp.h'),
join('include', 'numpy', 'npy_endian.h'),
Expand All @@ -778,7 +808,6 @@ def get_mathlib_info(*args):
join('src', 'multiarray', 'alloc.c'),
join('src', 'multiarray', 'arrayobject.c'),
join('src', 'multiarray', 'arraytypes.c.src'),
join('src', 'multiarray', 'array_assign.c'),
join('src', 'multiarray', 'array_assign_scalar.c'),
join('src', 'multiarray', 'array_assign_array.c'),
join('src', 'multiarray', 'buffer.c'),
Expand Down Expand Up @@ -821,40 +850,11 @@ def get_mathlib_info(*args):
join('src', 'multiarray', 'temp_elide.c'),
join('src', 'multiarray', 'typeinfo.c'),
join('src', 'multiarray', 'usertypes.c'),
join('src', 'multiarray', 'ucsnarrow.c'),
join('src', 'multiarray', 'vdot.c'),
join('src', 'private', 'templ_common.h.src'),
join('src', 'private', 'mem_overlap.c'),
join('src', 'private', 'npy_longdouble.c'),
join('src', 'private', 'ufunc_override.c'),
]

blas_info = get_info('blas_opt', 0)
if blas_info and ('HAVE_CBLAS', None) in blas_info.get('define_macros', []):
extra_info = blas_info
# These files are also in MANIFEST.in so that they are always in
# the source distribution independently of HAVE_CBLAS.
multiarray_src.extend([join('src', 'multiarray', 'cblasfuncs.c'),
join('src', 'multiarray', 'python_xerbla.c'),
])
if uses_accelerate_framework(blas_info):
multiarray_src.extend(get_sgemv_fix())
else:
extra_info = {}

config.add_extension('multiarray',
sources=multiarray_src +
[generate_config_h,
generate_numpyconfig_h,
generate_numpy_api,
join(codegen_dir, 'generate_numpy_api.py'),
join('*.py')],
depends=deps + multiarray_deps,
libraries=['npymath', 'npysort'],
extra_info=extra_info)

#######################################################################
# umath module #
# _multiarray_umath module - umath part #
#######################################################################

def generate_umath_c(ext, build_dir):
Expand Down Expand Up @@ -883,34 +883,34 @@ def generate_umath_c(ext, build_dir):
join('src', 'umath', 'scalarmath.c.src'),
join('src', 'umath', 'ufunc_type_resolution.c'),
join('src', 'umath', 'override.c'),
join('src', 'private', 'mem_overlap.c'),
join('src', 'private', 'npy_longdouble.c'),
join('src', 'private', 'ufunc_override.c')]
]

umath_deps = [
generate_umath_py,
join('include', 'numpy', 'npy_math.h'),
join('include', 'numpy', 'halffloat.h'),
join('src', 'multiarray', 'common.h'),
join('src', 'private', 'templ_common.h.src'),
join('src', 'common', 'templ_common.h.src'),
join('src', 'umath', 'simd.inc.src'),
join('src', 'umath', 'override.h'),
join(codegen_dir, 'generate_ufunc_api.py'),
join('src', 'private', 'lowlevel_strided_loops.h'),
join('src', 'private', 'mem_overlap.h'),
join('src', 'private', 'npy_longdouble.h'),
join('src', 'private', 'ufunc_override.h'),
join('src', 'private', 'binop_override.h')] + npymath_sources

config.add_extension('umath',
sources=umath_src +
]

config.add_extension('_multiarray_umath',
sources=multiarray_src + umath_src +
npymath_sources + common_src +
[generate_config_h,
generate_numpyconfig_h,
generate_umath_c,
generate_ufunc_api],
depends=deps + umath_deps,
libraries=['npymath'],
)
generate_numpyconfig_h,
generate_numpy_api,
join(codegen_dir, 'generate_numpy_api.py'),
join('*.py'),
generate_umath_c,
generate_ufunc_api,
],
depends=deps + multiarray_deps + umath_deps +
common_deps,
libraries=['npymath', 'npysort'],
extra_info=extra_info)

#######################################################################
# umath_tests module #
Expand Down Expand Up @@ -939,9 +939,9 @@ def generate_umath_c(ext, build_dir):

config.add_extension('_multiarray_tests',
sources=[join('src', 'multiarray', '_multiarray_tests.c.src'),
join('src', 'private', 'mem_overlap.c')],
depends=[join('src', 'private', 'mem_overlap.h'),
join('src', 'private', 'npy_extint128.h')],
join('src', 'common', 'mem_overlap.c')],
depends=[join('src', 'common', 'mem_overlap.h'),
join('src', 'common', 'npy_extint128.h')],
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm able to reproduce the gcov issue locally on linux & moving this extension addition block up above _multiarray_umath module prevents mem_overlap.c from being excluded from the C coverage report.

The diff is below and build / unit tests still seem fine locally for me:

diff --git a/numpy/core/setup.py b/numpy/core/setup.py
index b306aa4..1588a26 100644
--- a/numpy/core/setup.py
+++ b/numpy/core/setup.py
@@ -710,6 +710,17 @@ def configuration(parent_package='',top_path=None):
                        include_dirs=[])

     #######################################################################
+    #                     multiarray_tests module                         #
+    #######################################################################
+
+    config.add_extension('_multiarray_tests',
+                    sources=[join('src', 'multiarray', '_multiarray_tests.c.src'),
+                             join('src', 'common', 'mem_overlap.c')],
+                    depends=[join('src', 'common', 'mem_overlap.h'),
+                             join('src', 'common', 'npy_extint128.h')],
+                    libraries=['npymath'])
+
+    #######################################################################
     #             _multiarray_umath module - common part                  #
     #######################################################################

@@ -933,16 +944,6 @@ def configuration(parent_package='',top_path=None):
     config.add_extension('_struct_ufunc_tests',
                     sources=[join('src', 'umath', '_struct_ufunc_tests.c.src')])

-    #######################################################################
-    #                     multiarray_tests module                         #
-    #######################################################################
-
-    config.add_extension('_multiarray_tests',
-                    sources=[join('src', 'multiarray', '_multiarray_tests.c.src'),
-                             join('src', 'common', 'mem_overlap.c')],
-                    depends=[join('src', 'common', 'mem_overlap.h'),
-                             join('src', 'common', 'npy_extint128.h')],
-                    libraries=['npymath'])

I could open a PR to do this on master, but since @mattip has been working on this code for a while I'll let him judge if this might be suitable & avoids merged conflicts if we change here, etc.

libraries=['npymath'])

#######################################################################
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
8 changes: 8 additions & 0 deletions numpy/core/src/common/umathmodule.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#include "__umath_generated.c"
#include "__ufunc_api.c"

PyObject * add_newdoc_ufunc(PyObject *NPY_UNUSED(dummy), PyObject *args);
PyObject * ufunc_frompyfunc(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *NPY_UNUSED(kwds));
int initumath(PyObject *m);


2 changes: 1 addition & 1 deletion numpy/core/src/multiarray/descriptor.c
Original file line number Diff line number Diff line change
Expand Up @@ -2401,7 +2401,7 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args))
if (ret == NULL) {
return NULL;
}
mod = PyImport_ImportModule("numpy.core.multiarray");
mod = PyImport_ImportModule("numpy.core._multiarray_umath");
if (mod == NULL) {
Py_DECREF(ret);
return NULL;
Expand Down
2 changes: 1 addition & 1 deletion numpy/core/src/multiarray/methods.c
Original file line number Diff line number Diff line change
Expand Up @@ -1566,7 +1566,7 @@ array_reduce(PyArrayObject *self, PyObject *NPY_UNUSED(args))
if (ret == NULL) {
return NULL;
}
mod = PyImport_ImportModule("numpy.core.multiarray");
mod = PyImport_ImportModule("numpy.core._multiarray_umath");
if (mod == NULL) {
Py_DECREF(ret);
return NULL;
Expand Down
Loading
0