numpy · rgommers · Sep 20, 2019 · Sep 15, 2019 · Sep 15, 2019 · Sep 15, 2019
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -29,7 +29,7 @@ jobs:
            python3 -m pip install --user -r test_requirements.txt && \
            python3 -m pip install . && \
            F77=gfortran-5 F90=gfortran-5 \
-           CFLAGS='-UNDEBUG -std=c99' python3 runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml && \
+           CFLAGS='-UNDEBUG -std=c99' python3 runtests.py -n --debug-configure --mode=full -- -rsx --junitxml=junit/test-results.xml && \
            python3 tools/openblas_support.py --check_version $(OpenBLAS_version)"
     displayName: 'Run 32-bit Ubuntu Docker Build / Tests'
   - task: PublishTestResults@2
@@ -94,7 +94,7 @@ jobs:
     displayName: 'Check for unreachable code paths in Python modules'
   # prefer usage of clang over gcc proper
   # to match likely scenario on many user mac machines
-  - script: python setup.py build -j 4 install
+  - script: python setup.py build -j 4 build_src -v install
     displayName: 'Build NumPy'
     env:
       BLAS: None

diff --git a/doc/DISTUTILS.rst.txt b/doc/DISTUTILS.rst.txt
@@ -243,7 +243,7 @@ in writing setup scripts:
   after processing all source generators, no extension module will
   be built. This is the recommended way to conditionally define
   extension modules. Source generator functions are called by the
-  ``build_src`` command of ``numpy.distutils``.
+  ``build_src`` sub-command of ``numpy.distutils``.
 
   For example, here is a typical source generator function::
 

diff --git a/doc/source/dev/development_environment.rst b/doc/source/dev/development_environment.rst
@@ -96,6 +96,11 @@ installs a ``.egg-link`` file into your site-packages as well as adjusts the
 Other build options
 -------------------
 
+Build options can be discovered by running any of::
+
+    $ python setup.py --help
+    $ python setup.py --help-commands
+
 It's possible to do a parallel build with ``numpy.distutils`` with the ``-j`` option;
 see :ref:`parallel-builds` for more details.
 
@@ -106,6 +111,16 @@ source tree is to use::
     $ export PYTHONPATH=/some/owned/folder/lib/python3.4/site-packages
 
 
+NumPy uses a series of tests to probe the compiler and libc libraries for
+funtions. The results are stored in ``_numpyconfig.h`` and ``config.h`` files
+using ``HAVE_XXX`` definitions. These tests are run during the ``build_src``
+phase of the ``_multiarray_umath`` module in the ``generate_config_h`` and
+``generate_numpyconfig_h`` functions. Since the output of these calls includes
+many compiler warnings and errors, by default it is run quietly. If you wish
+to see this output, you can run the ``build_src`` stage verbosely::
+
+    $ python build build_src -v
+
 Using virtualenvs
 -----------------
 

diff --git a/numpy/core/setup.py b/numpy/core/setup.py
@@ -497,10 +497,10 @@ def generate_config_h(ext, build_dir):
                     #endif
                     """))
 
-            print('File:', target)
+            log.info('File: %s' % target)
             with open(target) as target_f:
-                print(target_f.read())
-            print('EOF')
+                log.info(target_f.read())
+            log.info('EOF')
         else:
             mathlibs = []
             with open(target) as target_f:
@@ -587,10 +587,10 @@ def generate_numpyconfig_h(ext, build_dir):
                     """))
 
             # Dump the numpyconfig.h header to stdout
-            print('File: %s' % target)
+            log.info('File: %s' % target)
             with open(target) as target_f:
-                print(target_f.read())
-            print('EOF')
+                log.info(target_f.read())
+            log.info('EOF')
         config.add_data_files((header_dir, target))
         return target
 
@@ -638,23 +638,6 @@ def generate_api(ext, build_dir):
             join(codegen_dir, 'genapi.py'),
             ]
 
-    #######################################################################
-    #                            dummy module                             #
-    #######################################################################
-
-    # npymath needs the config.h and numpyconfig.h files to be generated, but
-    # build_clib cannot handle generate_config_h and generate_numpyconfig_h
-    # (don't ask). Because clib are generated before extensions, we have to
-    # explicitly add an extension which has generate_config_h and
-    # generate_numpyconfig_h as sources *before* adding npymath.
-
-    config.add_extension('_dummy',
-                         sources=[join('src', 'dummymodule.c'),
-                                  generate_config_h,
-                                  generate_numpyconfig_h,
-                                  generate_numpy_api]
-                         )
-
     #######################################################################
     #                          npymath library                            #
     #######################################################################

diff --git a/numpy/distutils/__init__.py b/numpy/distutils/__init__.py
@@ -28,7 +28,7 @@ def customized_fcompiler(plat=None, compiler=None):
     c.customize()
     return c
 
-def customized_ccompiler(plat=None, compiler=None):
-    c = ccompiler.new_compiler(plat=plat, compiler=compiler)
+def customized_ccompiler(plat=None, compiler=None, verbose=1):
+    c = ccompiler.new_compiler(plat=plat, compiler=compiler, verbose=verbose)
     c.customize('')
     return c
diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py
@@ -140,7 +140,10 @@ def CCompiler_spawn(self, cmd, display=None):
             display = ' '.join(list(display))
     log.info(display)
     try:
-        subprocess.check_output(cmd)
+        if self.verbose:
+            subprocess.check_output(cmd)
+        else:
+            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as exc:
         o = exc.output
         s = exc.returncode
@@ -162,7 +165,8 @@ def CCompiler_spawn(self, cmd, display=None):
     if is_sequence(cmd):
         cmd = ' '.join(list(cmd))
 
-    forward_bytes_to_stdout(o)
+    if self.verbose:
+        forward_bytes_to_stdout(o)
 
     if re.search(b'Too many open files', o):
         msg = '\nTry rerunning setup command until build succeeds.'
@@ -727,10 +731,12 @@ def CCompiler_cxx_compiler(self):
 _distutils_new_compiler = new_compiler
 def new_compiler (plat=None,
                   compiler=None,
-                  verbose=0,
+                  verbose=None,
                   dry_run=0,
                   force=0):
     # Try first C compilers from numpy.distutils.
+    if verbose is None:
+        verbose = log.get_threshold() <= log.INFO
     if plat is None:
         plat = os.name
     try:
@@ -763,6 +769,7 @@ def new_compiler (plat=None,
         raise DistutilsModuleError(("can't compile C/C++ code: unable to find class '%s' " +
                "in module '%s'") % (class_name, module_name))
     compiler = klass(None, dry_run, force)
+    compiler.verbose = verbose
     log.debug('new_compiler returns %s' % (klass))
     return compiler
 

diff --git a/numpy/distutils/command/build.py b/numpy/distutils/command/build.py
@@ -16,8 +16,6 @@ class build(old_build):
     user_options = old_build.user_options + [
         ('fcompiler=', None,
          "specify the Fortran compiler type"),
-        ('parallel=', 'j',
-         "number of parallel jobs"),
         ]
 
     help_options = old_build.help_options + [
@@ -28,14 +26,8 @@ class build(old_build):
     def initialize_options(self):
         old_build.initialize_options(self)
         self.fcompiler = None
-        self.parallel = None
 
     def finalize_options(self):
-        if self.parallel:
-            try:
-                self.parallel = int(self.parallel)
-            except ValueError:
-                raise ValueError("--parallel/-j argument must be an integer")
         build_scripts = self.build_scripts
         old_build.finalize_options(self)
         plat_specifier = ".{}-{}.{}".format(get_platform(), *sys.version_info[:2])

diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py
@@ -53,9 +53,12 @@ class build_src(build_ext.build_ext):
         ('inplace', 'i',
          "ignore build-lib and put compiled extensions into the source " +
          "directory alongside your pure Python modules"),
+        ('verbose', 'v',
+         "change logging level from WARN to INFO which will show all " +
+         "compiler output")
         ]
 
-    boolean_options = ['force', 'inplace']
+    boolean_options = ['force', 'inplace', 'verbose']
 
     help_options = []
 
@@ 
F987
-76,6 +79,7 @@ def initialize_options(self):
         self.swig_opts = None
         self.swig_cpp = None
         self.swig = None
+        self.verbose = False
 
     def finalize_options(self):
         self.set_undefined_options('build',
@@ -365,6 +369,13 @@ def generate_sources(self, sources, extension):
             build_dir = os.path.join(*([self.build_src]
                                        +name.split('.')[:-1]))
         self.mkpath(build_dir)
+
+        if self.verbose:
+            new_level = log.INFO
+        else:
+            new_level = log.WARN
+        old_level = log.set_threshold(new_level)
+
         for func in func_sources:
             source = func(extension, build_dir)
             if not source:
@@ -375,7 +386,7 @@ def generate_sources(self, sources, extension):
             else:
                 log.info("  adding '%s' to sources." % (source,))
                 new_sources.append(source)
-
+        log.set_threshold(old_level)
         return new_sources
 
     def filter_py_files(self, sources):

diff --git a/numpy/distutils/log.py b/numpy/distutils/log.py
@@ -67,6 +67,8 @@ def set_threshold(level, force=False):
                 ' %s to %s' % (prev_level, level))
     return prev_level
 
+def get_threshold():
+	return _global_log.threshold
 
 def set_verbosity(v, force=False):
     prev_level = _global_log.threshold

diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py
@@ -146,7 +146,7 @@
 from distutils.errors import DistutilsError
 from distutils.dist import Distrib
1241
ution
 import distutils.sysconfig
-from distutils import log
+from numpy.distutils import log
 from distutils.util import get_platform
 
 from numpy.distutils.exec_command import (
@@ -550,15 +550,13 @@ class system_info(object):
     dir_env_var = None
     search_static_first = 0  # XXX: disabled by default, may disappear in
                             # future unless it is proved to be useful.
-    verbosity = 1
     saved_results = {}
 
     notfounderror = NotFoundError
 
     def __init__(self,
                   default_lib_dirs=default_lib_dirs,
                   default_include_dirs=default_include_dirs,
-                  verbosity=1,
                   ):
         self.__class__.info = {}
         self.local_prefixes = []
@@ -704,7 +702,7 @@ def get_info(self, notfound_action=0):
                 log.info('  FOUND:')
 
         res = self.saved_results.get(self.__class__.__name__)
-        if self.verbosity > 0 and flag:
+        if log.get_threshold() <= log.INFO and flag:
             for k, v in res.items():
                 v = str(v)
                 if k in ['sources', 'libraries'] and len(v) > 270:
@@ -914,7 +912,7 @@ def combine_paths(self, *args):
         """Return a list of existing paths composed by all combinations
         of items from the arguments.
         """
-        return combine_paths(*args, **{'verbosity': self.verbosity})
+        return combine_paths(*args)
 
 
 class fft_opt_info(system_info):
@@ -1531,12 +1529,12 @@ def get_atlas_version(**config):
     try:
         s, o = c.get_output(atlas_version_c_text,
                             libraries=libraries, library_dirs=library_dirs,
-                            use_tee=(system_info.verbosity > 0))
+                           )
         if s and re.search(r'undefined reference to `_gfortran', o, re.M):
             s, o = c.get_output(atlas_version_c_text,
                                 libraries=libraries + ['gfortran'],
                                 library_dirs=library_dirs,
-                                use_tee=(system_info.verbosity > 0))
+                               )
             if not s:
                 warnings.warn(textwrap.dedent("""
                     *****************************************************

diff --git a/runtests.py b/runtests.py
@@ -67,6 +67,10 @@ def main(argv):
     parser = ArgumentParser(usage=__doc__.lstrip())
     parser.add_argument("--verbose", "-v", action="count", default=1,
                         help="more verbosity")
+    parser.add_argument("--debug-configure", action="store_true",
+                        help=("add -v to build_src to show compiler "
+                              "configuration output while creating "
+                              "_numpyconfig.h and config.h"))
     parser.add_argument("--no-build", "-n", action="store_true", default=False,
                         help="do not build the project (use system installed version)")
     parser.add_argument("--build-only", "-b", action="store_true", default=False,
@@ -366,6 +370,8 @@ def build_project(args):
     cmd += ["build"]
     if args.parallel > 1:
         cmd += ["-j", str(args.parallel)]
+    if args.debug_configure:
+        cmd += ["build_src", "--verbose"]
     # Install; avoid producing eggs so numpy can be imported from dst_dir.
     cmd += ['install', '--prefix=' + dst_dir,
             '--single-version-externally-managed',

diff --git a/setup.py b/setup.py
@@ -215,6 +215,7 @@ def __exit__(self, exception_type, exception_value, traceback):
 
 
 from distutils.command.sdist import sdist
+from numpy.distutils.command.build_src import build_src
 class sdist_checked(sdist):
     """ check submodules on sdist to prevent incomplete tarballs """
     def run(self):
@@ -263,7 +264,7 @@ def parse_setuppy_commands():
     # below and not standalone.  Hence they're not added to good_commands.
     good_commands = ('develop', 'sdist', 'build', 'build_ext', 'build_py',
                      'build_clib', 'build_scripts', 'bdist_wheel', 'bdist_rpm',
-                     'bdist_wininst', 'bdist_msi', 'bdist_mpkg')
+                     'bdist_wininst', 'bdist_msi', 'bdist_mpkg', 'build_src')
 
     for command in good_commands:
         if command in args:
@@ -403,7 +404,9 @@ def setup_package():
         classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f],
         platforms = ["Windows", "Linux", "Solaris", "Mac OS-X", "Unix"],
         test_suite='nose.collector',
-        cmdclass={"sdist": sdist_checked},
+        cmdclass={"sdist": sdist_checked,
+                  "build_src": build_src,
+                 },
         python_requires='>=3.5',
         zip_safe=False,
         entry_points={

diff --git a/shippable.yml b/shippable.yml
@@ -48,7 +48,7 @@ build:
     # check OpenBLAS version
     - python tools/openblas_support.py --check_version 0.3.7
     # run the test suite
-    - python runtests.py -- -rsx --junit-xml=$SHIPPABLE_REPO_DIR/shippable/testresults/tests.xml -n 2 --durations=10
+    - python runtests.py --debug-configure --show-build-log -- -rsx --junit-xml=$SHIPPABLE_REPO_DIR/shippable/testresults/tests.xml -n 2 --durations=10
 
     cache: true
     cache_dir_list:

diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh
@@ -39,7 +39,7 @@ echo pypy3 version
 pypy3/bin/pypy3 -c "import sys; print(sys.version)"
 echo
 
-pypy3/bin/pypy3 runtests.py --show-build-log -v -- -rsx \
+pypy3/bin/pypy3 runtests.py --debug-configure --show-build-log -v -- -rsx \
       --junitxml=junit/test-results.xml --durations 10
 
 echo Make sure the correct openblas has been linked in

diff --git a/tools/travis-test.sh b/tools/travis-test.sh
@@ -52,7 +52,7 @@ setup_base()
   else
     # Python3.5-dbg on travis seems to need this
     export CFLAGS=$CFLAGS" -Wno-maybe-uninitialized"
-    $PYTHON setup.py build_ext --inplace 2>&1 | tee log
+    $PYTHON setup.py build build_src -v build_ext --inplace 2>&1 | tee log
   fi
   grep -v "_configtest" log \
     | grep -vE "ld returned 1|no previously-included files matching|manifest_maker: standard file '-c'" \
@@ -151,7 +151,7 @@ if [ -n "$USE_WHEEL" ] && [ $# -eq 0 ]; then
      export F90='gfortran --coverage'
      export LDFLAGS='--coverage'
   fi
-  $PYTHON setup.py bdist_wheel
+  $PYTHON setup.py build build_src -v bdist_wheel
   # Make another virtualenv to install into
   virtualenv --python=`which $PYTHON` venv-for-wheel
   . venv-for-wheel/bin/activate