diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ef0a122baf9c..a36f65faabed 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,7 @@ jobs: python3 -m pip install --user -r test_requirements.txt && \ python3 -m pip install . && \ F77=gfortran-5 F90=gfortran-5 \ - CFLAGS='-UNDEBUG -std=c99' python3 runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml && \ + CFLAGS='-UNDEBUG -std=c99' python3 runtests.py -n --debug-configure --mode=full -- -rsx --junitxml=junit/test-results.xml && \ python3 tools/openblas_support.py --check_version $(OpenBLAS_version)" displayName: 'Run 32-bit Ubuntu Docker Build / Tests' - task: PublishTestResults@2 @@ -94,7 +94,7 @@ jobs: displayName: 'Check for unreachable code paths in Python modules' # prefer usage of clang over gcc proper # to match likely scenario on many user mac machines - - script: python setup.py build -j 4 install + - script: python setup.py build -j 4 build_src -v install displayName: 'Build NumPy' env: BLAS: None diff --git a/doc/DISTUTILS.rst.txt b/doc/DISTUTILS.rst.txt index eadde63f8f71..bcef82500ba4 100644 --- a/doc/DISTUTILS.rst.txt +++ b/doc/DISTUTILS.rst.txt @@ -243,7 +243,7 @@ in writing setup scripts: after processing all source generators, no extension module will be built. This is the recommended way to conditionally define extension modules. Source generator functions are called by the - ``build_src`` command of ``numpy.distutils``. + ``build_src`` sub-command of ``numpy.distutils``. For example, here is a typical source generator function:: diff --git a/doc/source/dev/development_environment.rst b/doc/source/dev/development_environment.rst index ce571926e2c2..9d618cc9f910 100644 --- a/doc/source/dev/development_environment.rst +++ b/doc/source/dev/development_environment.rst @@ -96,6 +96,11 @@ installs a ``.egg-link`` file into your site-packages as well as adjusts the Other build options ------------------- +Build options can be discovered by running any of:: + + $ python setup.py --help + $ python setup.py --help-commands + It's possible to do a parallel build with ``numpy.distutils`` with the ``-j`` option; see :ref:`parallel-builds` for more details. @@ -106,6 +111,16 @@ source tree is to use:: $ export PYTHONPATH=/some/owned/folder/lib/python3.4/site-packages +NumPy uses a series of tests to probe the compiler and libc libraries for +funtions. The results are stored in ``_numpyconfig.h`` and ``config.h`` files +using ``HAVE_XXX`` definitions. These tests are run during the ``build_src`` +phase of the ``_multiarray_umath`` module in the ``generate_config_h`` and +``generate_numpyconfig_h`` functions. Since the output of these calls includes +many compiler warnings and errors, by default it is run quietly. If you wish +to see this output, you can run the ``build_src`` stage verbosely:: + + $ python build build_src -v + Using virtualenvs ----------------- diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 63b515b18563..5f2f4a7b2187 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -497,10 +497,10 @@ def generate_config_h(ext, build_dir): #endif """)) - print('File:', target) + log.info('File: %s' % target) with open(target) as target_f: - print(target_f.read()) - print('EOF') + log.info(target_f.read()) + log.info('EOF') else: mathlibs = [] with open(target) as target_f: @@ -587,10 +587,10 @@ def generate_numpyconfig_h(ext, build_dir): """)) # Dump the numpyconfig.h header to stdout - print('File: %s' % target) + log.info('File: %s' % target) with open(target) as target_f: - print(target_f.read()) - print('EOF') + log.info(target_f.read()) + log.info('EOF') config.add_data_files((header_dir, target)) return target @@ -638,23 +638,6 @@ def generate_api(ext, build_dir): join(codegen_dir, 'genapi.py'), ] - ####################################################################### - # dummy module # - ####################################################################### - - # npymath needs the config.h and numpyconfig.h files to be generated, but - # build_clib cannot handle generate_config_h and generate_numpyconfig_h - # (don't ask). Because clib are generated before extensions, we have to - # explicitly add an extension which has generate_config_h and - # generate_numpyconfig_h as sources *before* adding npymath. - - config.add_extension('_dummy', - sources=[join('src', 'dummymodule.c'), - generate_config_h, - generate_numpyconfig_h, - generate_numpy_api] - ) - ####################################################################### # npymath library # ####################################################################### diff --git a/numpy/distutils/__init__.py b/numpy/distutils/__init__.py index 55514750e26d..a6f804bdcbf7 100644 --- a/numpy/distutils/__init__.py +++ b/numpy/distutils/__init__.py @@ -28,7 +28,7 @@ def customized_fcompiler(plat=None, compiler=None): c.customize() return c -def customized_ccompiler(plat=None, compiler=None): - c = ccompiler.new_compiler(plat=plat, compiler=compiler) +def customized_ccompiler(plat=None, compiler=None, verbose=1): + c = ccompiler.new_compiler(plat=plat, compiler=compiler, verbose=verbose) c.customize('') return c diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py index 14451fa668ea..6438790234b7 100644 --- a/numpy/distutils/ccompiler.py +++ b/numpy/distutils/ccompiler.py @@ -140,7 +140,10 @@ def CCompiler_spawn(self, cmd, display=None): display = ' '.join(list(display)) log.info(display) try: - subprocess.check_output(cmd) + if self.verbose: + subprocess.check_output(cmd) + else: + subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: o = exc.output s = exc.returncode @@ -162,7 +165,8 @@ def CCompiler_spawn(self, cmd, display=None): if is_sequence(cmd): cmd = ' '.join(list(cmd)) - forward_bytes_to_stdout(o) + if self.verbose: + forward_bytes_to_stdout(o) if re.search(b'Too many open files', o): msg = '\nTry rerunning setup command until build succeeds.' @@ -727,10 +731,12 @@ def CCompiler_cxx_compiler(self): _distutils_new_compiler = new_compiler def new_compiler (plat=None, compiler=None, - verbose=0, + verbose=None, dry_run=0, force=0): # Try first C compilers from numpy.distutils. + if verbose is None: + verbose = log.get_threshold() <= log.INFO if plat is None: plat = os.name try: @@ -763,6 +769,7 @@ def new_compiler (plat=None, raise DistutilsModuleError(("can't compile C/C++ code: unable to find class '%s' " + "in module '%s'") % (class_name, module_name)) compiler = klass(None, dry_run, force) + compiler.verbose = verbose log.debug('new_compiler returns %s' % (klass)) return compiler diff --git a/numpy/distutils/command/build.py b/numpy/distutils/command/build.py index b3e18b2049ef..afc1d14774a1 100644 --- a/numpy/distutils/command/build.py +++ b/numpy/distutils/command/build.py @@ -16,8 +16,6 @@ class build(old_build): user_options = old_build.user_options + [ ('fcompiler=', None, "specify the Fortran compiler type"), - ('parallel=', 'j', - "number of parallel jobs"), ] help_options = old_build.help_options + [ @@ -28,14 +26,8 @@ class build(old_build): def initialize_options(self): old_build.initialize_options(self) self.fcompiler = None - self.parallel = None def finalize_options(self): - if self.parallel: - try: - self.parallel = int(self.parallel) - except ValueError: - raise ValueError("--parallel/-j argument must be an integer") build_scripts = self.build_scripts old_build.finalize_options(self) plat_specifier = ".{}-{}.{}".format(get_platform(), *sys.version_info[:2]) diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index e183b2090419..664b52e37684 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -53,9 +53,12 @@ class build_src(build_ext.build_ext): ('inplace', 'i', "ignore build-lib and put compiled extensions into the source " + "directory alongside your pure Python modules"), + ('verbose', 'v', + "change logging level from WARN to INFO which will show all " + + "compiler output") ] - boolean_options = ['force', 'inplace'] + boolean_options = ['force', 'inplace', 'verbose'] help_options = [] @@ -76,6 +79,7 @@ def initialize_options(self): self.swig_opts = None self.swig_cpp = None self.swig = None + self.verbose = False def finalize_options(self): self.set_undefined_options('build', @@ -365,6 +369,13 @@ def generate_sources(self, sources, extension): build_dir = os.path.join(*([self.build_src] +name.split('.')[:-1])) self.mkpath(build_dir) + + if self.verbose: + new_level = log.INFO + else: + new_level = log.WARN + old_level = log.set_threshold(new_level) + for func in func_sources: source = func(extension, build_dir) if not source: @@ -375,7 +386,7 @@ def generate_sources(self, sources, extension): else: log.info(" adding '%s' to sources." % (source,)) new_sources.append(source) - + log.set_threshold(old_level) return new_sources def filter_py_files(self, sources): diff --git a/numpy/distutils/log.py b/numpy/distutils/log.py index 37f9fe5dd0ef..ff7de86b1147 100644 --- a/numpy/distutils/log.py +++ b/numpy/distutils/log.py @@ -67,6 +67,8 @@ def set_threshold(level, force=False): ' %s to %s' % (prev_level, level)) return prev_level +def get_threshold(): + return _global_log.threshold def set_verbosity(v, force=False): prev_level = _global_log.threshold diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index 6cfce3b1cad0..5fd1003ab419 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -146,7 +146,7 @@ from distutils.errors import DistutilsError from distutils.dist import Distribution import distutils.sysconfig -from distutils import log +from numpy.distutils import log from distutils.util import get_platform from numpy.distutils.exec_command import ( @@ -550,7 +550,6 @@ class system_info(object): dir_env_var = None search_static_first = 0 # XXX: disabled by default, may disappear in # future unless it is proved to be useful. - verbosity = 1 saved_results = {} notfounderror = NotFoundError @@ -558,7 +557,6 @@ class system_info(object): def __init__(self, default_lib_dirs=default_lib_dirs, default_include_dirs=default_include_dirs, - verbosity=1, ): self.__class__.info = {} self.local_prefixes = [] @@ -704,7 +702,7 @@ def get_info(self, notfound_action=0): log.info(' FOUND:') res = self.saved_results.get(self.__class__.__name__) - if self.verbosity > 0 and flag: + if log.get_threshold() <= log.INFO and flag: for k, v in res.items(): v = str(v) if k in ['sources', 'libraries'] and len(v) > 270: @@ -914,7 +912,7 @@ def combine_paths(self, *args): """Return a list of existing paths composed by all combinations of items from the arguments. """ - return combine_paths(*args, **{'verbosity': self.verbosity}) + return combine_paths(*args) class fft_opt_info(system_info): @@ -1531,12 +1529,12 @@ def get_atlas_version(**config): try: s, o = c.get_output(atlas_version_c_text, libraries=libraries, library_dirs=library_dirs, - use_tee=(system_info.verbosity > 0)) + ) if s and re.search(r'undefined reference to `_gfortran', o, re.M): s, o = c.get_output(atlas_version_c_text, libraries=libraries + ['gfortran'], library_dirs=library_dirs, - use_tee=(system_info.verbosity > 0)) + ) if not s: warnings.warn(textwrap.dedent(""" ***************************************************** diff --git a/runtests.py b/runtests.py index 23245aeac365..6639937cd5cb 100755 --- a/runtests.py +++ b/runtests.py @@ -67,6 +67,10 @@ def main(argv): parser = ArgumentParser(usage=__doc__.lstrip()) parser.add_argument("--verbose", "-v", action="count", default=1, help="more verbosity") + parser.add_argument("--debug-configure", action="store_true", + help=("add -v to build_src to show compiler " + "configuration output while creating " + "_numpyconfig.h and config.h")) parser.add_argument("--no-build", "-n", action="store_true", default=False, help="do not build the project (use system installed version)") parser.add_argument("--build-only", "-b", action="store_true", default=False, @@ -366,6 +370,8 @@ def build_project(args): cmd += ["build"] if args.parallel > 1: cmd += ["-j", str(args.parallel)] + if args.debug_configure: + cmd += ["build_src", "--verbose"] # Install; avoid producing eggs so numpy can be imported from dst_dir. cmd += ['install', '--prefix=' + dst_dir, '--single-version-externally-managed', diff --git a/setup.py b/setup.py index 640105ed0e88..f778d7409b79 100755 --- a/setup.py +++ b/setup.py @@ -215,6 +215,7 @@ def __exit__(self, exception_type, exception_value, traceback): from distutils.command.sdist import sdist +from numpy.distutils.command.build_src import build_src class sdist_checked(sdist): """ check submodules on sdist to prevent incomplete tarballs """ def run(self): @@ -263,7 +264,7 @@ def parse_setuppy_commands(): # below and not standalone. Hence they're not added to good_commands. good_commands = ('develop', 'sdist', 'build', 'build_ext', 'build_py', 'build_clib', 'build_scripts', 'bdist_wheel', 'bdist_rpm', - 'bdist_wininst', 'bdist_msi', 'bdist_mpkg') + 'bdist_wininst', 'bdist_msi', 'bdist_mpkg', 'build_src') for command in good_commands: if command in args: @@ -403,7 +404,9 @@ def setup_package(): classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f], platforms = ["Windows", "Linux", "Solaris", "Mac OS-X", "Unix"], test_suite='nose.collector', - cmdclass={"sdist": sdist_checked}, + cmdclass={"sdist": sdist_checked, + "build_src": build_src, + }, python_requires='>=3.5', zip_safe=False, entry_points={ diff --git a/shippable.yml b/shippable.yml index 2abd8a84348b..91323ceb6664 100644 --- a/shippable.yml +++ b/shippable.yml @@ -48,7 +48,7 @@ build: # check OpenBLAS version - python tools/openblas_support.py --check_version 0.3.7 # run the test suite - - python runtests.py -- -rsx --junit-xml=$SHIPPABLE_REPO_DIR/shippable/testresults/tests.xml -n 2 --durations=10 + - python runtests.py --debug-configure --show-build-log -- -rsx --junit-xml=$SHIPPABLE_REPO_DIR/shippable/testresults/tests.xml -n 2 --durations=10 cache: true cache_dir_list: diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh index 5940cc75a657..b02d18778f15 100755 --- a/tools/pypy-test.sh +++ b/tools/pypy-test.sh @@ -39,7 +39,7 @@ echo pypy3 version pypy3/bin/pypy3 -c "import sys; print(sys.version)" echo -pypy3/bin/pypy3 runtests.py --show-build-log -v -- -rsx \ +pypy3/bin/pypy3 runtests.py --debug-configure --show-build-log -v -- -rsx \ --junitxml=junit/test-results.xml --durations 10 echo Make sure the correct openblas has been linked in diff --git a/tools/travis-test.sh b/tools/travis-test.sh index 8fbae4b0980c..1eda43c310af 100755 --- a/tools/travis-test.sh +++ b/tools/travis-test.sh @@ -52,7 +52,7 @@ setup_base() else # Python3.5-dbg on travis seems to need this export CFLAGS=$CFLAGS" -Wno-maybe-uninitialized" - $PYTHON setup.py build_ext --inplace 2>&1 | tee log + $PYTHON setup.py build build_src -v build_ext --inplace 2>&1 | tee log fi grep -v "_configtest" log \ | grep -vE "ld returned 1|no previously-included files matching|manifest_maker: standard file '-c'" \ @@ -151,7 +151,7 @@ if [ -n "$USE_WHEEL" ] && [ $# -eq 0 ]; then export F90='gfortran --coverage' export LDFLAGS='--coverage' fi - $PYTHON setup.py bdist_wheel + $PYTHON setup.py build build_src -v bdist_wheel # Make another virtualenv to install into virtualenv --python=`which $PYTHON` venv-for-wheel . venv-for-wheel/bin/activate