diff --git a/.circleci/config.yml b/.circleci/config.yml index 04c8979fa0d6a..a1b186fee1a57 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,12 @@ jobs: - NUMPY_VERSION: 1.11.0 - SCIPY_VERSION: 0.17.0 - MATPLOTLIB_VERSION: 1.5.1 + # on conda, this is the latest for python 3.5 + # The following places need to be in sync with regard to Cython version: + # - pyproject.toml + # - .circleci config file + # - sklearn/_build_utils/__init__.py + # - advanced installation guide - CYTHON_VERSION: 0.28.5 - SCIKIT_IMAGE_VERSION: 0.12.3 steps: diff --git a/MANIFEST.in b/MANIFEST.in index e36adcae38b0e..89634452812e4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,4 +5,4 @@ recursive-include sklearn *.c *.h *.pyx *.pxd *.pxi *.tp recursive-include sklearn/datasets *.csv *.csv.gz *.rst *.jpg *.txt *.arff.gz *.json.gz include COPYING include README.rst - +include pyproject.toml diff --git a/build_tools/azure/install.cmd b/build_tools/azure/install.cmd index 2566ba4f4f3aa..735ba0f49218c 100644 --- a/build_tools/azure/install.cmd +++ b/build_tools/azure/install.cmd @@ -14,7 +14,6 @@ IF "%PYTHON_ARCH%"=="64" ( conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION% numpy scipy cython matplotlib wheel pillow joblib call activate %VIRTUALENV% - IF "%PYTEST_VERSION%"=="*" ( pip install pytest ) else ( diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index bd9fbe06edd50..e4db817cf40a3 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -74,6 +74,7 @@ if [[ "$DISTRIB" == "conda" ]]; then elif [[ "$DISTRIB" == "ubuntu" ]]; then sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test + sudo apt-get update sudo apt-get install python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate @@ -88,8 +89,12 @@ elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then # Since conda main channel usually lacks behind on the latest releases, # we use pypi to test against the latest releases of the dependencies. # conda is still used as a convenient way to install Python and pip. + # We let pip install the latest version of the build dependencies from + # pyproject.toml and the runtime dependencies from the dist-info metadata + # that results from the scikit-learn setup.py file. + # The optional test are installed separately: make_conda "python=$PYTHON_VERSION" - python -m pip install numpy scipy joblib cython + python -m pip install -U pip python -m pip install pytest==$PYTEST_VERSION pytest-cov pytest-xdist python -m pip install pandas matplotlib pyamg fi @@ -117,5 +122,14 @@ except ImportError: print('pandas not installed') " python -m pip list -python setup.py build_ext --inplace -j 3 -python setup.py develop + +if [[ "$DISTRIB" == "conda-pip-latest" ]]; then + # Check that pip can automatically install the build dependencies from + # pyproject.toml using an isolated build environment: + pip install --verbose --editable . +else + # Use the pre-installed build dependencies and build directly in the + # current environment. + python setup.py build_ext --inplace -j 3 + python setup.py develop +fi diff --git a/doc/developers/advanced_installation.rst b/doc/developers/advanced_installation.rst index e341268fc04d2..cc6f4849c5800 100644 --- a/doc/developers/advanced_installation.rst +++ b/doc/developers/advanced_installation.rst @@ -101,6 +101,13 @@ Build dependencies Building Scikit-learn also requires: +.. + # The following places need to be in sync with regard to Cython version: + # - pyproject.toml + # - .circleci config file + # - sklearn/_build_utils/__init__.py + # - advanced installation guide + - Cython >= 0.28.5 - A C/C++ compiler and a matching OpenMP_ runtime library. See the :ref:`platform system specific instructions diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000..ef9d82c71ef46 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +# Minimum requirements for the build system to execute. +requires = [ + "setuptools", + "wheel", + "numpy>=1.11", + "scipy>=0.17.0", + # on conda, this is the latest for python 3.5 + # The following places need to be in sync with regard to Cython version: + # - pyproject.toml + # - .circleci config file + # - sklearn/_build_utils/__init__.py + # - advanced installation guide + "Cython>=0.28.5", +] diff --git a/sklearn/_build_utils/__init__.py b/sklearn/_build_utils/__init__.py index 1d6c1eaf607a6..876381d035f0c 100644 --- a/sklearn/_build_utils/__init__.py +++ b/sklearn/_build_utils/__init__.py @@ -6,7 +6,6 @@ import os - from distutils.version import LooseVersion import contextlib @@ -14,67 +13,47 @@ DEFAULT_ROOT = 'sklearn' -# on conda, this is the latest for python 3.5 + +# The following places need to be in sync with regard to Cython version: +# - pyproject.toml +# - .circleci config file +# - sklearn/_build_utils/__init__.py +# - advanced installation guide CYTHON_MIN_VERSION = '0.28.5' -def build_from_c_and_cpp_files(extensions): - """Modify the extensions to build from the .c and .cpp files. - - This is useful for releases, this way cython is not required to - run python setup.py install. - """ - for extension in extensions: - sources = [] - for sfile in extension.sources: - path, ext = os.path.splitext(sfile) - if ext in ('.pyx', '.py'): - if extension.language == 'c++': - ext = '.cpp' - else: - ext = '.c' - sfile = path + ext - sources.append(sfile) - extension.sources = sources - - -def maybe_cythonize_extensions(top_path, config): - """Tweaks for building extensions between release and development mode.""" +def cythonize_extensions(top_path, config): + """Check that a recent Cython is available and cythonize extensions""" with_openmp = check_openmp_support() - is_release = os.path.exists(os.path.join(top_path, 'PKG-INFO')) - - if is_release: - build_from_c_and_cpp_files(config.ext_modules) - else: - message = ('Please install cython with a version >= {0} in order ' - 'to build a scikit-learn development version.').format( - CYTHON_MIN_VERSION) - try: - import Cython - if LooseVersion(Cython.__version__) < CYTHON_MIN_VERSION: - message += ' Your version of Cython was {0}.'.format( - Cython.__version__) - raise ValueError(message) - from Cython.Build import cythonize - except ImportError as exc: - exc.args += (message,) - raise - - n_jobs = 1 - with contextlib.suppress(ImportError): - import joblib - if LooseVersion(joblib.__version__) > LooseVersion("0.13.0"): - # earlier joblib versions don't account for CPU affinity - # constraints, and may over-estimate the number of available - # CPU particularly in CI (cf loky#114) - n_jobs = joblib.effective_n_jobs() - - config.ext_modules = cythonize( - config.ext_modules, - nthreads=n_jobs, - compile_time_env={'SKLEARN_OPENMP_SUPPORTED': with_openmp}, - compiler_directives={'language_level': 3}) + message = ('Please install cython with a version >= {0} in order ' + 'to build a scikit-learn from source.').format( + CYTHON_MIN_VERSION) + try: + import Cython + if LooseVersion(Cython.__version__) < CYTHON_MIN_VERSION: + message += (' The current version of Cython is {} installed in {}.' + .format(Cython.__version__, Cython.__path__)) + raise ValueError(message) + from Cython.Build import cythonize + except ImportError as exc: + exc.args += (message,) + raise + + n_jobs = 1 + with contextlib.suppress(ImportError): + import joblib + if LooseVersion(joblib.__version__) > LooseVersion("0.13.0"): + # earlier joblib versions don't account for CPU affinity + # constraints, and may over-estimate the number of available + # CPU particularly in CI (cf loky#114) + n_jobs = joblib.cpu_count() + + config.ext_modules = cythonize( + config.ext_modules, + nthreads=n_jobs, + compile_time_env={'SKLEARN_OPENMP_SUPPORTED': with_openmp}, + compiler_directives={'language_level': 3}) def gen_from_templates(templates, top_path): diff --git a/sklearn/setup.py b/sklearn/setup.py index 0c7f19f23d39c..892f36d305e7f 100644 --- a/sklearn/setup.py +++ b/sklearn/setup.py @@ -1,6 +1,7 @@ +import sys import os -from sklearn._build_utils import maybe_cythonize_extensions +from sklearn._build_utils import cythonize_extensions from sklearn._build_utils.deprecated_modules import ( _create_deprecated_modules_files ) @@ -78,7 +79,8 @@ def configuration(parent_package='', top_path=None): # add the test directory config.add_subpackage('tests') - maybe_cythonize_extensions(top_path, config) + if 'sdist' not in sys.argv: + cythonize_extensions(top_path, config) return config diff --git a/sklearn/tests/test_common.py b/sklearn/tests/test_common.py index c2938b9f6fc08..33eee4ae554a5 100644 --- a/sklearn/tests/test_common.py +++ b/sklearn/tests/test_common.py @@ -125,6 +125,10 @@ def test_check_estimator_generate_only(): def test_configure(): # Smoke test the 'configure' step of setup, this tests all the # 'configure' functions in the setup.pys in scikit-learn + # This test requires Cython which is not necessarily there when running + # the tests of an installed version of scikit-learn or when scikit-learn + # is installed in editable mode by pip build isolation enabled. + pytest.importorskip("Cython") cwd = os.getcwd() setup_path = os.path.abspath(os.path.join(sklearn.__path__[0], '..')) setup_filename = os.path.join(setup_path, 'setup.py')