From 8d03a7c1b424ddd164c4b3db6a9bbfe4d9afe225 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Sat, 28 Sep 2019 12:09:08 +0200 Subject: [PATCH 01/24] dropping python-3.5 --- azure-pipelines.yml | 38 ++++++++++++++++++------------------ build_tools/azure/install.sh | 9 --------- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9eca835865868..f86e3400cc5d4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,30 +3,30 @@ jobs: - template: build_tools/azure/posix.yml parameters: name: Linux - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 matrix: # Linux environment to test that scikit-learn can be built against - # versions of numpy, scipy with ATLAS that comes with Ubuntu Xenial 16.04 - # i.e. numpy 1.11 and scipy 0.17 - py35_ubuntu_atlas: + # versions of numpy, scipy with ATLAS that comes with Ubuntu Bionic 18.04 + # i.e. numpy 1.13 and scipy 0.19 + py36_ubuntu_atlas: DISTRIB: 'ubuntu' - PYTHON_VERSION: '3.5' + PYTHON_VERSION: '3.6' JOBLIB_VERSION: '0.11' SKLEARN_NO_OPENMP: 'True' - # Linux + Python 3.5 build with OpenBLAS and without SITE_JOBLIB - py35_conda_openblas: + # Linux + Python 3.6 build with OpenBLAS and without SITE_JOBLIB + py36_conda_openblas: DISTRIB: 'conda' - PYTHON_VERSION: '3.5' + PYTHON_VERSION: '3.6' INSTALL_MKL: 'false' - NUMPY_VERSION: '1.11.0' - SCIPY_VERSION: '0.17.0' + NUMPY_VERSION: '1.13.0' + SCIPY_VERSION: '0.19.0' PANDAS_VERSION: '*' CYTHON_VERSION: '*' PYTEST_VERSION: '*' PILLOW_VERSION: '4.0.0' - MATPLOTLIB_VERSION: '1.5.1' - # later version of joblib are not packaged in conda for Python 3.5 - JOBLIB_VERSION: '0.12.3' + MATPLOTLIB_VERSION: '2.1.1' + # latest version of joblib available in conda for Python 3.6 + JOBLIB_VERSION: '0.13.2' COVERAGE: 'true' # Linux environment to test the latest available dependencies and MKL. # It runs tests requiring pandas and PyAMG. @@ -53,12 +53,12 @@ jobs: - template: build_tools/azure/posix-32.yml parameters: name: Linux32 - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 matrix: - py35_ubuntu_atlas_32bit: + py36_ubuntu_atlas_32bit: DISTRIB: 'ubuntu-32' - PYTHON_VERSION: '3.5' - JOBLIB_VERSION: '0.11' + PYTHON_VERSION: '3.6' + JOBLIB_VERSION: '0.13' SKLEARN_NO_OPENMP: 'True' - template: build_tools/azure/posix.yml @@ -89,6 +89,6 @@ jobs: PYTHON_ARCH: '64' PYTEST_VERSION: '*' COVERAGE: 'true' - py35_pip_openblas_32bit: - PYTHON_VERSION: '3.5' + py36_pip_openblas_32bit: + PYTHON_VERSION: '3.6' PYTHON_ARCH: '32' diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index 4c0622396ad80..59971f8facb45 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -60,15 +60,6 @@ if [[ "$DISTRIB" == "conda" ]]; then TO_INSTALL="$TO_INSTALL matplotlib=$MATPLOTLIB_VERSION" fi - # Old packages coming from the 'free' conda channel have been removed but - # we are using them for testing Python 3.5. See - # https://www.anaconda.com/why-we-removed-the-free-channel-in-conda-4-7/ - # for more details. restore_free_channel is defined starting from conda 4.7 - conda_version=$(conda -V | awk '{print $2}') - if version_ge "$conda_version" "4.7.0" && [[ "$PYTHON_VERSION" == "3.5" ]]; then - conda config --set restore_free_channel true - fi - make_conda $TO_INSTALL if [[ "$PYTHON_VERSION" == "*" ]]; then pip install pytest-xdist From b13ba084d2df68507dcfa8ab283785961d1b7c19 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Sat, 28 Sep 2019 12:22:30 +0200 Subject: [PATCH 02/24] fix install guide --- doc/install.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/install.rst b/doc/install.rst index 8431f0beab371..2056c75ef9729 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -17,20 +17,21 @@ Installing the latest release Scikit-learn requires: -- Python (>= 3.5) -- NumPy (>= 1.11.0) -- SciPy (>= 0.17.0) +- Python (>= 3.6) +- NumPy (>= 1.13.0) +- SciPy (>= 0.19.0) - joblib (>= 0.11) Scikit-learn plotting capabilities (i.e., functions start with "plot\_" -and classes end with "Display") require Matplotlib (>= 1.5.1). For running the -examples Matplotlib >= 1.5.1 is required. A few examples require +and classes end with "Display") require Matplotlib (>= 2.1.1). For running the +examples Matplotlib >= 2.1.1 is required. A few examples require scikit-image >= 0.12.3, a few examples require pandas >= 0.18.0. .. warning:: Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4. - Scikit-learn now requires Python 3.5 or newer. + Scikit-learn 0.21 supported Python 3.5-3.7. + Scikit-learn now requires Python 3.6 or newer. If you already have a working installation of numpy and scipy, the easiest way to install scikit-learn is using ``pip`` :: From 1caac21df9ccb55850baa8a13afe0ac20b3d34af Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Sat, 28 Sep 2019 12:25:18 +0200 Subject: [PATCH 03/24] fix setup.py --- setup.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index d3b2494432f81..4e7b886d4c2ba 100755 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ try: import builtins except ImportError: - # Python 2 compat: just to be able to declare that Python >=3.5 is needed. + # Python 2 compat: just to be able to declare that Python >=3.6 is needed. import __builtin__ as builtins # This is a bit (!) hackish: we are setting a global variable so that the @@ -51,8 +51,8 @@ SCIPY_MIN_VERSION = '1.1.0' NUMPY_MIN_VERSION = '1.14.0' else: - SCIPY_MIN_VERSION = '0.17.0' - NUMPY_MIN_VERSION = '1.11.0' + SCIPY_MIN_VERSION = '0.19.0' + NUMPY_MIN_VERSION = '1.13.0' JOBLIB_MIN_VERSION = '0.11' @@ -138,7 +138,7 @@ def build_extensions(self): except ImportError: # Numpy should not be a dependency just to be able to introspect - # that python 3.5 is required. + # that python 3.6 is required. pass @@ -219,7 +219,6 @@ def setup_package(): 'Operating System :: Unix', 'Operating System :: MacOS', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', ('Programming Language :: Python :: ' @@ -228,7 +227,7 @@ def setup_package(): 'Implementation :: PyPy') ], cmdclass=cmdclass, - python_requires=">=3.5", + python_requires=">=3.6", install_requires=[ 'numpy>={}'.format(NUMPY_MIN_VERSION), 'scipy>={}'.format(SCIPY_MIN_VERSION), @@ -254,9 +253,9 @@ def setup_package(): metadata['version'] = VERSION else: - if sys.version_info < (3, 5): + if sys.version_info < (3, 6): raise RuntimeError( - "Scikit-learn requires Python 3.5 or later. The current" + "Scikit-learn requires Python 3.6 or later. The current" " Python version is %s installed in %s." % (platform.python_version(), sys.executable)) From 08a64a61542af6f285b2c428b1395d677855f545 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Sat, 28 Sep 2019 12:26:38 +0200 Subject: [PATCH 04/24] fix circleci --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index de08f2d5622f5..62cd3ae74092b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,10 +9,10 @@ jobs: - MKL_NUM_THREADS: 2 - MINICONDA_PATH: ~/miniconda - CONDA_ENV_NAME: testenv - - PYTHON_VERSION: 3.5 - - NUMPY_VERSION: 1.11.0 - - SCIPY_VERSION: 0.17.0 - - MATPLOTLIB_VERSION: 1.5.1 + - PYTHON_VERSION: 3.6 + - NUMPY_VERSION: 1.13.0 + - SCIPY_VERSION: 0.19.0 + - MATPLOTLIB_VERSION: 2.1.1 - CYTHON_VERSION: 0.28.5 - SCIKIT_IMAGE_VERSION: 0.12.3 steps: From e9d7550bd2972d3f8e4001e7c6d23160c1628b2b Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Sat, 28 Sep 2019 12:29:26 +0200 Subject: [PATCH 05/24] advanced_installation --- doc/developers/advanced_installation.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/developers/advanced_installation.rst b/doc/developers/advanced_installation.rst index 0eaac27699d37..e04882f273f65 100644 --- a/doc/developers/advanced_installation.rst +++ b/doc/developers/advanced_installation.rst @@ -60,9 +60,9 @@ Dependencies Scikit-learn requires: -- Python (>= 3.5), -- NumPy (>= 1.11), -- SciPy (>= 0.17), +- Python (>= 3.6), +- NumPy (>= 1.13), +- SciPy (>= 0.19), - Joblib (>= 0.11). .. note:: From 5383ef926544aeb6a1d3ceb678a580d5fd1c1adf Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Sat, 28 Sep 2019 12:30:01 +0200 Subject: [PATCH 06/24] index.html --- doc/templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/templates/index.html b/doc/templates/index.html index 7c8dc111bc91c..eefdc96707b57 100644 --- a/doc/templates/index.html +++ b/doc/templates/index.html @@ -156,7 +156,7 @@

News

  • On-going development: What's new (Changelog)
    • -
  • Scikit-learn from 0.21 requires Python 3.5 or greater. +
  • Scikit-learn from 0.22 requires Python 3.6 or greater.
  • July 2019. scikit-learn 0.21.3 (Changelog) and 0.20.4 (Changelog) are available for download.
    • From ad7707749f3c7ce23a3b35b337ff8083e6138832 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Sat, 28 Sep 2019 12:36:53 +0200 Subject: [PATCH 07/24] readme --- README.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 12dccbecd6802..1e4ea76448353 100644 --- a/README.rst +++ b/README.rst @@ -14,8 +14,8 @@ .. |CircleCI| image:: https://circleci.com/gh/scikit-learn/scikit-learn/tree/master.svg?style=shield&circle-token=:circle-token .. _CircleCI: https://circleci.com/gh/scikit-learn/scikit-learn -.. |Python35| image:: https://img.shields.io/badge/python-3.5-blue.svg -.. _Python35: https://badge.fury.io/py/scikit-learn +.. |Python36| image:: https://img.shields.io/badge/python-3.6-blue.svg +.. _Python36: https://badge.fury.io/py/scikit-learn .. |PyPi| image:: https://badge.fury.io/py/scikit-learn.svg .. _PyPi: https://badge.fury.io/py/scikit-learn @@ -47,17 +47,17 @@ Dependencies scikit-learn requires: -- Python (>= 3.5) -- NumPy (>= 1.11.0) -- SciPy (>= 0.17.0) +- Python (>= 3.6) +- NumPy (>= 1.13.0) +- SciPy (>= 0.19.0) - joblib (>= 0.11) **Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4.** -scikit-learn 0.21 and later require Python 3.5 or newer. +scikit-learn 0.22 and later require Python 3.6 or newer. Scikit-learn plotting capabilities (i.e., functions start with "plot_" -and classes end with "Display") require Matplotlib (>= 1.5.1). For running the -examples Matplotlib >= 1.5.1 is required. A few examples require +and classes end with "Display") require Matplotlib (>= 2.1.1). For running the +examples Matplotlib >= 2.1.1 is required. A few examples require scikit-image >= 0.12.3, a few examples require pandas >= 0.18.0. User installation From 455bd6a9a8bdc40dda77f482ab0446449b398dec Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Sat, 28 Sep 2019 12:37:03 +0200 Subject: [PATCH 08/24] pyparsing.py --- .../machine_learning_map/pyparsing.py | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/doc/tutorial/machine_learning_map/pyparsing.py b/doc/tutorial/machine_learning_map/pyparsing.py index 20690df7aec47..0c5fca5cf891d 100644 --- a/doc/tutorial/machine_learning_map/pyparsing.py +++ b/doc/tutorial/machine_learning_map/pyparsing.py @@ -1020,20 +1020,14 @@ def _trim_arity(func, maxargs=2): limit = [0] foundArity = [False] - # traceback return data structure changed in Py3.5 - normalize back to plain tuples - if system_version[:2] >= (3,5): - def extract_stack(limit=0): - # special handling for Python 3.5.0 - extra deep call stack by 1 - offset = -3 if system_version == (3,5,0) else -2 - frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] - return [(frame_summary.filename, frame_summary.lineno)] - def extract_tb(tb, limit=0): - frames = traceback.extract_tb(tb, limit=limit) - frame_summary = frames[-1] - return [(frame_summary.filename, frame_summary.lineno)] - else: - extract_stack = traceback.extract_stack - extract_tb = traceback.extract_tb + def extract_stack(limit=0): + offset = -2 + frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] + return [(frame_summary.filename, frame_summary.lineno)] + def extract_tb(tb, limit=0): + frames = traceback.extract_tb(tb, limit=limit) + frame_summary = frames[-1] + return [(frame_summary.filename, frame_summary.lineno)] # synthesize what would be returned by traceback.extract_stack at the call to # user's parse action 'func', so that we don't incur call penalty at parse time From b208612f5135ebb4799fe1705530b23198a6aad9 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Sat, 28 Sep 2019 12:39:19 +0200 Subject: [PATCH 09/24] remove clean_warning_registry --- sklearn/feature_extraction/tests/test_text.py | 2 -- sklearn/utils/testing.py | 15 --------------- 2 files changed, 17 deletions(-) diff --git a/sklearn/feature_extraction/tests/test_text.py b/sklearn/feature_extraction/tests/test_text.py index 7b7697ff47fff..d624cf07dd8d2 100644 --- a/sklearn/feature_extraction/tests/test_text.py +++ b/sklearn/feature_extraction/tests/test_text.py @@ -32,7 +32,6 @@ from sklearn.exceptions import ChangedBehaviorWarning from sklearn.utils.testing import (assert_almost_equal, assert_warns_message, assert_raise_message, - clean_warning_registry, SkipTest, assert_no_warnings, fails_if_pypy, assert_allclose_dense_sparse, skip_if_32bit) @@ -373,7 +372,6 @@ def test_tfidf_no_smoothing(): [1, 0, 0]] tr = TfidfTransformer(smooth_idf=False, norm='l2') - clean_warning_registry() with warnings.catch_warnings(record=True) as w: 1. / np.array([0.]) numpy_provides_div0_warning = len(w) == 1 diff --git a/sklearn/utils/testing.py b/sklearn/utils/testing.py index 4645e455d04c6..8156548d43e77 100644 --- a/sklearn/utils/testing.py +++ b/sklearn/utils/testing.py @@ -611,21 +611,6 @@ def set_random_state(estimator, random_state=0): pass -def clean_warning_registry(): - """Clean Python warning registry for easier testing of warning messages. - - When changing warning filters this function is not necessary with - Python3.5+, as __warningregistry__ will be re-set internally. - See https://bugs.python.org/issue4180 and - https://bugs.python.org/issue21724 for more details. - - """ - for mod in sys.modules.values(): - registry = getattr(mod, "__warningregistry__", None) - if registry is not None: - registry.clear() - - def check_skip_network(): if int(os.environ.get('SKLEARN_SKIP_NETWORK_TESTS', 0)): raise SkipTest("Text tutorial requires large dataset download") From 5ea9f0c81cebcf9f8f1d6dd24e3bcfe143a92caf Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Sat, 28 Sep 2019 13:00:53 +0200 Subject: [PATCH 10/24] 1.13.1 and 1.19.1 --- .circleci/config.yml | 4 ++-- README.rst | 4 ++-- azure-pipelines.yml | 4 ++-- doc/install.rst | 4 ++-- setup.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62cd3ae74092b..964bac86976f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,8 +10,8 @@ jobs: - MINICONDA_PATH: ~/miniconda - CONDA_ENV_NAME: testenv - PYTHON_VERSION: 3.6 - - NUMPY_VERSION: 1.13.0 - - SCIPY_VERSION: 0.19.0 + - NUMPY_VERSION: 1.13.1 + - SCIPY_VERSION: 0.19.1 - MATPLOTLIB_VERSION: 2.1.1 - CYTHON_VERSION: 0.28.5 - SCIKIT_IMAGE_VERSION: 0.12.3 diff --git a/README.rst b/README.rst index 1e4ea76448353..e6c9d6a09ff2a 100644 --- a/README.rst +++ b/README.rst @@ -48,8 +48,8 @@ Dependencies scikit-learn requires: - Python (>= 3.6) -- NumPy (>= 1.13.0) -- SciPy (>= 0.19.0) +- NumPy (>= 1.13.1) +- SciPy (>= 0.19.1) - joblib (>= 0.11) **Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4.** diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f86e3400cc5d4..7cb93bbf785b8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,8 +18,8 @@ jobs: DISTRIB: 'conda' PYTHON_VERSION: '3.6' INSTALL_MKL: 'false' - NUMPY_VERSION: '1.13.0' - SCIPY_VERSION: '0.19.0' + NUMPY_VERSION: '1.13.1' + SCIPY_VERSION: '0.19.1' PANDAS_VERSION: '*' CYTHON_VERSION: '*' PYTEST_VERSION: '*' diff --git a/doc/install.rst b/doc/install.rst index 2056c75ef9729..66963ce565763 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -18,8 +18,8 @@ Installing the latest release Scikit-learn requires: - Python (>= 3.6) -- NumPy (>= 1.13.0) -- SciPy (>= 0.19.0) +- NumPy (>= 1.13.1) +- SciPy (>= 0.19.1) - joblib (>= 0.11) Scikit-learn plotting capabilities (i.e., functions start with "plot\_" diff --git a/setup.py b/setup.py index 4e7b886d4c2ba..ad5a831bf3a17 100755 --- a/setup.py +++ b/setup.py @@ -51,8 +51,8 @@ SCIPY_MIN_VERSION = '1.1.0' NUMPY_MIN_VERSION = '1.14.0' else: - SCIPY_MIN_VERSION = '0.19.0' - NUMPY_MIN_VERSION = '1.13.0' + SCIPY_MIN_VERSION = '0.19.1' + NUMPY_MIN_VERSION = '1.13.1' JOBLIB_MIN_VERSION = '0.11' From 234583b67f23b0a782a8b5fdec07b048492c4cb6 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Fri, 22 Nov 2019 13:31:31 +0100 Subject: [PATCH 11/24] don't use 16.04 --- azure-pipelines.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 495fc5a640a90..7f8507d831fe6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,7 +3,7 @@ jobs: - job: linting displayName: Linting pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 steps: - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH @@ -21,7 +21,7 @@ jobs: - template: build_tools/azure/posix.yml parameters: name: Linux_Runs - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 matrix: pylatest_conda_mkl: DISTRIB: 'conda' @@ -79,23 +79,13 @@ jobs: - template: build_tools/azure/posix-32.yml parameters: name: Linux32 -<<<<<<< HEAD vmImage: ubuntu-18.04 -======= - vmImage: ubuntu-16.04 dependsOn: [linting] ->>>>>>> upstream/master matrix: py36_ubuntu_atlas_32bit: DISTRIB: 'ubuntu-32' -<<<<<<< HEAD PYTHON_VERSION: '3.6' JOBLIB_VERSION: '0.13' - SKLEARN_NO_OPENMP: 'True' -======= - PYTHON_VERSION: '3.5' - JOBLIB_VERSION: '0.11' ->>>>>>> upstream/master - template: build_tools/azure/posix.yml parameters: From 4286d5b5f0b1295b20bd9c8617260c75b18b2b0a Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Fri, 22 Nov 2019 13:36:11 +0100 Subject: [PATCH 12/24] 18.04 libatlas-dev -> libatlas-base-dev --- build_tools/azure/install.sh | 4 ++-- build_tools/circle/build_test_pypy.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index 61ee6bac7116f..a6a476fa5c4ae 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -78,13 +78,13 @@ if [[ "$DISTRIB" == "conda" ]]; then elif [[ "$DISTRIB" == "ubuntu" ]]; then sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test sudo apt-get update - sudo apt-get install python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv + sudo apt-get install python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate python -m pip install pytest==$PYTEST_VERSION pytest-cov cython joblib==$JOBLIB_VERSION elif [[ "$DISTRIB" == "ubuntu-32" ]]; then apt-get update - apt-get install -y python3-dev python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv + apt-get install -y python3-dev python3-scipy python3-matplotlib libatlas3-base libatlas-base-dev python3-virtualenv python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate python -m pip install pytest==$PYTEST_VERSION pytest-cov cython joblib==$JOBLIB_VERSION diff --git a/build_tools/circle/build_test_pypy.sh b/build_tools/circle/build_test_pypy.sh index 60b81e60709f0..2371917100b26 100755 --- a/build_tools/circle/build_test_pypy.sh +++ b/build_tools/circle/build_test_pypy.sh @@ -3,7 +3,7 @@ set -x set -e apt-get -yq update -apt-get -yq install libatlas-dev libatlas-base-dev liblapack-dev gfortran ccache libopenblas-dev +apt-get -yq install libatlas-base-dev liblapack-dev gfortran ccache libopenblas-dev pip install virtualenv From bc2debb4a25b9716312ec6bd02fa3ecaa22f38b1 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Fri, 22 Nov 2019 13:47:17 +0100 Subject: [PATCH 13/24] min pillow version for 3.6 is 4.2.1 --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7f8507d831fe6..6edf20f42c35f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -59,7 +59,7 @@ jobs: CYTHON_VERSION: '*' # temporary pin pytest due to unknown failure with pytest 5.3 PYTEST_VERSION: '5.2' - PILLOW_VERSION: '4.0.0' + PILLOW_VERSION: '4.2.1' MATPLOTLIB_VERSION: '2.1.1' # latest version of joblib available in conda for Python 3.6 JOBLIB_VERSION: '0.13.2' From 923b5375a7d70f1947000b46778efe28aba320f3 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Thu, 2 Jan 2020 23:05:44 +0100 Subject: [PATCH 14/24] echo commands --- build_tools/azure/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index a8cea594da20d..2419638baa841 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e +set -x UNAMESTR=`uname` From 1406677c658a3f936df2d78decdb5bb845ad3509 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Thu, 2 Jan 2020 23:43:27 +0100 Subject: [PATCH 15/24] fix conflict, and 32 bit --- azure-pipelines.yml | 2 +- build_tools/azure/posix-32.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7e2c941396ee1..8cfe536ebc979 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -59,7 +59,7 @@ jobs: DISTRIB: 'conda' PYTHON_VERSION: '3.6' BLAS: 'openblas' - NUMPY_VERSION: '1.13.1' + NUMPY_VERSION: '1.13' SCIPY_VERSION: '0.19.1' PANDAS_VERSION: '*' CYTHON_VERSION: '*' diff --git a/build_tools/azure/posix-32.yml b/build_tools/azure/posix-32.yml index 68e05e347f307..b4c9e4ebb287e 100644 --- a/build_tools/azure/posix-32.yml +++ b/build_tools/azure/posix-32.yml @@ -40,7 +40,7 @@ jobs: -e OMP_NUM_THREADS=$OMP_NUM_THREADS -e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS -e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS - i386/ubuntu:16.04 + i386/ubuntu:18.04 sleep 1000000 displayName: 'Start container' - script: > From f57cacb0b809bb946b04eabc763ccf30441e3c78 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Thu, 2 Jan 2020 23:55:50 +0100 Subject: [PATCH 16/24] fix conflict for circle --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ec9d0f7e0ef4d..3e3c85745f21a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - MINICONDA_PATH: ~/miniconda - CONDA_ENV_NAME: testenv - PYTHON_VERSION: 3.6 - - NUMPY_VERSION: 1.13.1 + - NUMPY_VERSION: 1.13 - SCIPY_VERSION: 0.19.1 - MATPLOTLIB_VERSION: 2.1.1 # on conda, this is the latest for python 3.5 From 23d5f97d64543fac629e016e4de0b9ee7a51267d Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Fri, 3 Jan 2020 11:22:19 +0100 Subject: [PATCH 17/24] fix scikit-image version dep --- .circleci/config.yml | 2 +- README.rst | 2 +- doc/install.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e3c85745f21a..5e1227d987bc8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,7 +19,7 @@ jobs: # - sklearn/_build_utils/__init__.py # - advanced installation guide - CYTHON_VERSION: 0.28.5 - - SCIKIT_IMAGE_VERSION: 0.12.3 + - SCIKIT_IMAGE_VERSION: 0.13 steps: - checkout - run: ./build_tools/circle/checkout_merge_commit.sh diff --git a/README.rst b/README.rst index 0e97126762841..ff5bef40b99d0 100644 --- a/README.rst +++ b/README.rst @@ -58,7 +58,7 @@ scikit-learn 0.22 and later require Python 3.6 or newer. Scikit-learn plotting capabilities (i.e., functions start with ``plot_`` and classes end with "Display") require Matplotlib (>= 2.1.1). For running the examples Matplotlib >= 2.1.1 is required. A few examples require -scikit-image >= 0.12.3, a few examples require pandas >= 0.18.0. +scikit-image >= 0.13, a few examples require pandas >= 0.18.0. User installation ~~~~~~~~~~~~~~~~~ diff --git a/doc/install.rst b/doc/install.rst index 527551f41fd9c..ec8f708a73fb4 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -134,7 +134,7 @@ it as ``scikit-learn[alldeps]``. Scikit-learn plotting capabilities (i.e., functions start with "plot\_" and classes end with "Display") require Matplotlib (>= 2.1.1). For running the examples Matplotlib >= 2.1.1 is required. A few examples require -scikit-image >= 0.12.3, a few examples require pandas >= 0.18.0. +scikit-image >= 0.13, a few examples require pandas >= 0.18.0. .. warning:: From a1eead647571b711ac657d93b0d56d2b08081a43 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Mon, 13 Jan 2020 16:40:02 +0100 Subject: [PATCH 18/24] mark tests as xfail on 32bit py3.6 --- sklearn/gaussian_process/tests/test_gpr.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sklearn/gaussian_process/tests/test_gpr.py b/sklearn/gaussian_process/tests/test_gpr.py index eb4bc6dec1761..4ed105db04246 100644 --- a/sklearn/gaussian_process/tests/test_gpr.py +++ b/sklearn/gaussian_process/tests/test_gpr.py @@ -3,6 +3,7 @@ # Author: Jan Hendrik Metzen # License: BSD 3 clause +import sys import numpy as np from scipy.optimize import approx_fprime @@ -46,6 +47,9 @@ def f(x): @pytest.mark.parametrize('kernel', kernels) def test_gpr_interpolation(kernel): + if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6): + pytest.xfail("This test may fail on 32bit Py3.6") + # Test the interpolating property for different kernels. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) y_pred, y_cov = gpr.predict(X, return_cov=True) @@ -70,6 +74,9 @@ def test_gpr_interpolation_structured(): @pytest.mark.parametrize('kernel', non_fixed_kernels) def test_lml_improving(kernel): + if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6): + pytest.xfail("This test may fail on 32bit Py3.6") + # Test that hyperparameter-tuning improves log-marginal likelihood. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) assert (gpr.log_marginal_likelihood(gpr.kernel_.theta) > @@ -175,6 +182,9 @@ def test_no_optimizer(): @pytest.mark.parametrize('kernel', kernels) def test_predict_cov_vs_std(kernel): + if sys.maxsize <= 2 ** 32 and sys.version_info[:2] == (3, 6): + pytest.xfail("This test may fail on 32bit Py3.6") + # Test that predicted std.-dev. is consistent with cov's diagonal. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) y_mean, y_cov = gpr.predict(X2, return_cov=True) From 6408b736ae87f55400e13493b4766d1a46b63c94 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Fri, 7 Feb 2020 14:42:52 +0100 Subject: [PATCH 19/24] move to 1.13.3 min numpy version, and simplify old code --- .circleci/config.yml | 2 +- README.rst | 2 +- azure-pipelines.yml | 4 ++-- doc/developers/advanced_installation.rst | 2 +- setup.py | 2 +- sklearn/utils/_mask.py | 19 ++++--------------- sklearn/utils/fixes.py | 12 ------------ 7 files changed, 10 insertions(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e1227d987bc8..5480831fd962e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - MINICONDA_PATH: ~/miniconda - CONDA_ENV_NAME: testenv - PYTHON_VERSION: 3.6 - - NUMPY_VERSION: 1.13 + - NUMPY_VERSION: 1.13.3 - SCIPY_VERSION: 0.19.1 - MATPLOTLIB_VERSION: 2.1.1 # on conda, this is the latest for python 3.5 diff --git a/README.rst b/README.rst index ff5bef40b99d0..2c74b8e3000e9 100644 --- a/README.rst +++ b/README.rst @@ -48,7 +48,7 @@ Dependencies scikit-learn requires: - Python (>= 3.6) -- NumPy (>= 1.13.1) +- NumPy (>= 1.13.3) - SciPy (>= 0.19.1) - joblib (>= 0.11) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8cfe536ebc979..6a5126d3a9ba0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -49,7 +49,7 @@ jobs: matrix: # Linux environment to test that scikit-learn can be built against # versions of numpy, scipy with ATLAS that comes with Ubuntu Bionic 18.04 - # i.e. numpy 1.13 and scipy 0.19 + # i.e. numpy 1.13.3 and scipy 0.19 py36_ubuntu_atlas: DISTRIB: 'ubuntu' PYTHON_VERSION: '3.6' @@ -59,7 +59,7 @@ jobs: DISTRIB: 'conda' PYTHON_VERSION: '3.6' BLAS: 'openblas' - NUMPY_VERSION: '1.13' + NUMPY_VERSION: '1.13.3' SCIPY_VERSION: '0.19.1' PANDAS_VERSION: '*' CYTHON_VERSION: '*' diff --git a/doc/developers/advanced_installation.rst b/doc/developers/advanced_installation.rst index ae33fbbbb1cd1..345fff733eb88 100644 --- a/doc/developers/advanced_installation.rst +++ b/doc/developers/advanced_installation.rst @@ -84,7 +84,7 @@ Scikit-learn requires the following dependencies both at build time and at runtime: - Python (>= 3.6), -- NumPy (>= 1.13), +- NumPy (>= 1.13.3), - SciPy (>= 0.19), - Joblib (>= 0.11). diff --git a/setup.py b/setup.py index 870ec555359a8..798c8bfc5d305 100755 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ NUMPY_MIN_VERSION = '1.14.0' else: SCIPY_MIN_VERSION = '0.19.1' - NUMPY_MIN_VERSION = '1.13.1' + NUMPY_MIN_VERSION = '1.13.3' JOBLIB_MIN_VERSION = '0.11' diff --git a/sklearn/utils/_mask.py b/sklearn/utils/_mask.py index 1f660205cdd47..e93171caa0dc5 100644 --- a/sklearn/utils/_mask.py +++ b/sklearn/utils/_mask.py @@ -1,21 +1,10 @@ import numpy as np -from . import is_scalar_nan -from .fixes import _object_dtype_isnan - def _get_mask(X, value_to_mask): """Compute the boolean mask X == missing_values.""" - if is_scalar_nan(value_to_mask): - if X.dtype.kind == "f": - return np.isnan(X) - elif X.dtype.kind in ("i", "u"): - # can't have NaNs in integer array. - return np.zeros(X.shape, dtype=bool) - else: - # np.isnan does not work on object dtypes. - return _object_dtype_isnan(X) + if X.dtype.kind in ("i", "u"): + # can't have NaNs in integer array. + return np.zeros(X.shape, dtype=bool) else: - # X == value_to_mask with object dtypes does not always perform - # element-wise for old versions of numpy - return np.equal(X, value_to_mask) + return np.isnan(X) diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py index 4456e9d271329..8c676be6dc50e 100644 --- a/sklearn/utils/fixes.py +++ b/sklearn/utils/fixes.py @@ -173,18 +173,6 @@ def __getstate__(self): from numpy.ma import MaskedArray # noqa -# Fix for behavior inconsistency on numpy.equal for object dtypes. -# For numpy versions < 1.13, numpy.equal tests element-wise identity of objects -# instead of equality. This fix returns the mask of NaNs in an array of -# numerical or object values for all numpy versions. -if np_version < (1, 13): - def _object_dtype_isnan(X): - return np.frompyfunc(lambda x: x != x, 1, 1)(X).astype(bool) -else: - def _object_dtype_isnan(X): - return X != X - - # TODO: replace by copy=False, when only scipy > 1.1 is supported. def _astype_copy_false(X): """Returns the copy=False parameter for From c6e867ea1fbb1b45fc8b4be1490c5e2736ad6892 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Fri, 7 Feb 2020 15:12:32 +0100 Subject: [PATCH 20/24] remove the rest of _object_dtype_isnan usages --- sklearn/utils/tests/test_fixes.py | 16 ---------------- sklearn/utils/validation.py | 3 +-- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/sklearn/utils/tests/test_fixes.py b/sklearn/utils/tests/test_fixes.py index 21ddaf7d3ec5c..cdbe28d3e76c8 100644 --- a/sklearn/utils/tests/test_fixes.py +++ b/sklearn/utils/tests/test_fixes.py @@ -14,7 +14,6 @@ from sklearn.utils.fixes import MaskedArray from sklearn.utils.fixes import _joblib_parallel_args -from sklearn.utils.fixes import _object_dtype_isnan from sklearn.utils.fixes import loguniform @@ -58,21 +57,6 @@ def test_joblib_parallel_args(monkeypatch, joblib_version): raise ValueError -@pytest.mark.parametrize("dtype, val", ([object, 1], - [object, "a"], - [float, 1])) -def test_object_dtype_isnan(dtype, val): - X = np.array([[val, np.nan], - [np.nan, val]], dtype=dtype) - - expected_mask = np.array([[False, True], - [True, False]]) - - mask = _object_dtype_isnan(X) - - assert_array_equal(mask, expected_mask) - - @pytest.mark.parametrize("low,high,base", [(-1, 0, 10), (0, 2, np.exp(1)), (-1, 1, 2)]) def test_loguniform(low, high, base): diff --git a/sklearn/utils/validation.py b/sklearn/utils/validation.py index 9929ff2f35502..b7b40b879ef07 100644 --- a/sklearn/utils/validation.py +++ b/sklearn/utils/validation.py @@ -21,7 +21,6 @@ from numpy.core.numeric import ComplexWarning import joblib -from .fixes import _object_dtype_isnan from .. import get_config as _get_config from ..exceptions import NonBLASDotWarning, PositiveSpectrumWarning from ..exceptions import NotFittedError @@ -61,7 +60,7 @@ def _assert_all_finite(X, allow_nan=False, msg_dtype=None): ) # for object dtype data, we only check for NaNs (GH-13254) elif X.dtype == np.dtype('object') and not allow_nan: - if _object_dtype_isnan(X).any(): + if np.isnan(X).any(): raise ValueError("Input contains NaN") From 49f8ee18a3f5cbe256220ff0aafe7f1477080acb Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Fri, 7 Feb 2020 16:26:52 +0100 Subject: [PATCH 21/24] Revert "remove the rest of _object_dtype_isnan usages" This reverts commit c6e867ea1fbb1b45fc8b4be1490c5e2736ad6892. --- sklearn/utils/tests/test_fixes.py | 16 ++++++++++++++++ sklearn/utils/validation.py | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/sklearn/utils/tests/test_fixes.py b/sklearn/utils/tests/test_fixes.py index cdbe28d3e76c8..21ddaf7d3ec5c 100644 --- a/sklearn/utils/tests/test_fixes.py +++ b/sklearn/utils/tests/test_fixes.py @@ -14,6 +14,7 @@ from sklearn.utils.fixes import MaskedArray from sklearn.utils.fixes import _joblib_parallel_args +from sklearn.utils.fixes import _object_dtype_isnan from sklearn.utils.fixes import loguniform @@ -57,6 +58,21 @@ def test_joblib_parallel_args(monkeypatch, joblib_version): raise ValueError +@pytest.mark.parametrize("dtype, val", ([object, 1], + [object, "a"], + [float, 1])) +def test_object_dtype_isnan(dtype, val): + X = np.array([[val, np.nan], + [np.nan, val]], dtype=dtype) + + expected_mask = np.array([[False, True], + [True, False]]) + + mask = _object_dtype_isnan(X) + + assert_array_equal(mask, expected_mask) + + @pytest.mark.parametrize("low,high,base", [(-1, 0, 10), (0, 2, np.exp(1)), (-1, 1, 2)]) def test_loguniform(low, high, base): diff --git a/sklearn/utils/validation.py b/sklearn/utils/validation.py index 2334463612fd6..ff9d8b169cb0c 100644 --- a/sklearn/utils/validation.py +++ b/sklearn/utils/validation.py @@ -23,6 +23,7 @@ from contextlib import suppress +from .fixes import _object_dtype_isnan from .. import get_config as _get_config from ..exceptions import NonBLASDotWarning, PositiveSpectrumWarning from ..exceptions import NotFittedError @@ -62,7 +63,7 @@ def _assert_all_finite(X, allow_nan=False, msg_dtype=None): ) # for object dtype data, we only check for NaNs (GH-13254) elif X.dtype == np.dtype('object') and not allow_nan: - if np.isnan(X).any(): + if _object_dtype_isnan(X).any(): raise ValueError("Input contains NaN") From bf23aa86234c9ae0922be96f329a8e3d01a00075 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Fri, 7 Feb 2020 16:30:59 +0100 Subject: [PATCH 22/24] fix issues raised by jeremy --- sklearn/utils/_mask.py | 16 +++++++++++----- sklearn/utils/fixes.py | 4 ++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/sklearn/utils/_mask.py b/sklearn/utils/_mask.py index e93171caa0dc5..46aa9d24e8a95 100644 --- a/sklearn/utils/_mask.py +++ b/sklearn/utils/_mask.py @@ -2,9 +2,15 @@ def _get_mask(X, value_to_mask): - """Compute the boolean mask X == missing_values.""" - if X.dtype.kind in ("i", "u"): - # can't have NaNs in integer array. - return np.zeros(X.shape, dtype=bool) + """Compute the boolean mask X == value_to_mask.""" + if is_scalar_nan(value_to_mask): + if X.dtype.kind == "f": + return np.isnan(X) + elif X.dtype.kind in ("i", "u"): + # can't have NaNs in integer array. + return np.zeros(X.shape, dtype=bool) + else: + # np.isnan does not work on object dtypes. + return X != X else: - return np.isnan(X) + return X == value_to_mask diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py index 8c676be6dc50e..03e220eab29ae 100644 --- a/sklearn/utils/fixes.py +++ b/sklearn/utils/fixes.py @@ -173,6 +173,10 @@ def __getstate__(self): from numpy.ma import MaskedArray # noqa +def _object_dtype_isnan(X): + return X != X + + # TODO: replace by copy=False, when only scipy > 1.1 is supported. def _astype_copy_false(X): """Returns the copy=False parameter for From aa2915af86c7048efc9cf2b504ae68b1c6d550e0 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Fri, 7 Feb 2020 16:34:18 +0100 Subject: [PATCH 23/24] minor fix --- sklearn/utils/_mask.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sklearn/utils/_mask.py b/sklearn/utils/_mask.py index 46aa9d24e8a95..48b2a11c6f87b 100644 --- a/sklearn/utils/_mask.py +++ b/sklearn/utils/_mask.py @@ -1,5 +1,8 @@ import numpy as np +from . import is_scalar_nan +from .fixes import _object_dtype_isnan + def _get_mask(X, value_to_mask): """Compute the boolean mask X == value_to_mask.""" @@ -11,6 +14,6 @@ def _get_mask(X, value_to_mask): return np.zeros(X.shape, dtype=bool) else: # np.isnan does not work on object dtypes. - return X != X + return _object_dtype_isnan(X) else: return X == value_to_mask From 783eeacdb3fd5884c2f979efa714c426aec63d48 Mon Sep 17 00:00:00 2001 From: adrinjalali Date: Mon, 10 Feb 2020 11:55:53 +0100 Subject: [PATCH 24/24] minor fixes mentioned by Olivier --- README.rst | 2 +- doc/install.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2c74b8e3000e9..96bc81fa3822d 100644 --- a/README.rst +++ b/README.rst @@ -53,7 +53,7 @@ scikit-learn requires: - joblib (>= 0.11) **Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4.** -scikit-learn 0.22 and later require Python 3.6 or newer. +scikit-learn 0.23 and later require Python 3.6 or newer. Scikit-learn plotting capabilities (i.e., functions start with ``plot_`` and classes end with "Display") require Matplotlib (>= 2.1.1). For running the diff --git a/doc/install.rst b/doc/install.rst index ec8f708a73fb4..6a2b83605c1a6 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -140,6 +140,7 @@ scikit-image >= 0.13, a few examples require pandas >= 0.18.0. Scikit-learn 0.20 was the last version to support Python 2.7 and Python 3.4. Scikit-learn 0.21 supported Python 3.5-3.7. + Scikit-learn 0.22 supported Python 3.5-3.8. Scikit-learn now requires Python 3.6 or newer.