From 5e5dfe32ddf57416f87e05cab2424ccf9e8a4350 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Sat, 8 Sep 2018 17:15:19 +0200 Subject: [PATCH 01/11] Revert "MAINT Temporarly remove PyPy3 build in CircleCI" This reverts commit 6e94a6331064c56b4922cf49a83dc9965f0be2de. --- .circleci/config.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a1bc848942d3..effb37d6b8faa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,6 +65,21 @@ jobs: path: ~/log.txt destination: log.txt + pypy3: + docker: + - image: pypy:3-6.0.0 + steps: + - restore_cache: + keys: + - pypy3-ccache-{{ .Branch }} + - pypy3-ccache + - checkout + - run: ./build_tools/circle/build_test_pypy.sh + - save_cache: + key: pypy3-ccache-{{ .Branch }}-{{ .BuildNum }} + paths: + - ~/.ccache + - ~/.cache/pip deploy: docker: @@ -89,6 +104,7 @@ workflows: jobs: - python3 - python2 + - pypy3 - deploy: requires: - python3 From 442a482ce9a3580c5e5d4512b40f255dc6a664c0 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Sat, 8 Sep 2018 18:19:43 +0200 Subject: [PATCH 02/11] Catch segfaults --- build_tools/circle/build_test_pypy.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build_tools/circle/build_test_pypy.sh b/build_tools/circle/build_test_pypy.sh index 18fa361821d14..7d81ed2a29d3d 100755 --- a/build_tools/circle/build_test_pypy.sh +++ b/build_tools/circle/build_test_pypy.sh @@ -19,7 +19,7 @@ python --version which python pip install --extra-index https://antocuni.github.io/pypy-wheels/ubuntu numpy==1.14.4 Cython pytest -pip install "scipy>=1.1.0" sphinx numpydoc docutils +pip install "scipy>=1.1.0" sphinx numpydoc docutils pytest-xdist ccache -M 512M export CCACHE_COMPRESS=1 @@ -27,4 +27,5 @@ export PATH=/usr/lib/ccache:$PATH pip install -e . -make test +pytest -n 1 sklearn/ +make test-sphinxext test-doc From 9dbdf3c684d2c13482b26a3dc4cd50fcf3a13294 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 10 Sep 2018 12:22:21 +0200 Subject: [PATCH 03/11] Ignore deprecation warnings and skip new tests that use hashing --- build_tools/circle/build_test_pypy.sh | 4 ++-- conftest.py | 4 +++- sklearn/feature_extraction/tests/test_text.py | 3 +++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/build_tools/circle/build_test_pypy.sh b/build_tools/circle/build_test_pypy.sh index 7d81ed2a29d3d..6530c7569ae68 100755 --- a/build_tools/circle/build_test_pypy.sh +++ b/build_tools/circle/build_test_pypy.sh @@ -25,7 +25,7 @@ ccache -M 512M export CCACHE_COMPRESS=1 export PATH=/usr/lib/ccache:$PATH -pip install -e . +pip install -vv -e . -pytest -n 1 sklearn/ +pytest -n 1 -Wignore::DeprecationWarning -Wignore::FutureWarning sklearn/ make test-sphinxext test-doc diff --git a/conftest.py b/conftest.py index f175661165b2d..82c4b17faeef0 100644 --- a/conftest.py +++ b/conftest.py @@ -32,7 +32,9 @@ def pytest_collection_modifyitems(config, items): skip_marker = pytest.mark.skip( reason='FeatureHasher is not compatible with PyPy') for item in items: - if item.name == 'sklearn.feature_extraction.hashing.FeatureHasher': + if item.name in ( + 'sklearn.feature_extraction.hashing.FeatureHasher', + 'sklearn.feature_extraction.text.HashingVectorizer'): item.add_marker(skip_marker) # Skip tests which require internet if the flag is provided diff --git a/sklearn/feature_extraction/tests/test_text.py b/sklearn/feature_extraction/tests/test_text.py index b9431bc5439cb..d6b1b2b64b4c0 100644 --- a/sklearn/feature_extraction/tests/test_text.py +++ b/sklearn/feature_extraction/tests/test_text.py @@ -1108,6 +1108,8 @@ def test_vectorizers_invalid_ngram_range(vec): message = ("Invalid value for ngram_range=%s " "lower boundary larger than the upper boundary." % str(invalid_range)) + if isinstance(vec, HashingVectorizer): + pytest.xfail(reason='HashingVectorizer not supported on PyPy') assert_raise_message( ValueError, message, vec.fit, ["good news everyone"]) @@ -1119,6 +1121,7 @@ def test_vectorizers_invalid_ngram_range(vec): ValueError, message, vec.transform, ["good news everyone"]) +@fails_if_pypy def test_vectorizer_stop_words_inconsistent(): if PY2: lstr = "[u'and', u'll', u've']" From ca3c1be072779a31fae8b75a76fbdb5935a975d1 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Sun, 16 Sep 2018 22:35:27 +0200 Subject: [PATCH 04/11] Use latest numpy and some tests as known failure --- build_tools/circle/build_test_pypy.sh | 4 ++-- sklearn/tests/test_docstring_parameters.py | 2 ++ sklearn/tests/test_multioutput.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build_tools/circle/build_test_pypy.sh b/build_tools/circle/build_test_pypy.sh index 6530c7569ae68..bb048a5e44241 100755 --- a/build_tools/circle/build_test_pypy.sh +++ b/build_tools/circle/build_test_pypy.sh @@ -18,7 +18,7 @@ source pypy-env/bin/activate python --version which python -pip install --extra-index https://antocuni.github.io/pypy-wheels/ubuntu numpy==1.14.4 Cython pytest +pip install --extra-index https://antocuni.github.io/pypy-wheels/ubuntu numpy Cython pytest pip install "scipy>=1.1.0" sphinx numpydoc docutils pytest-xdist ccache -M 512M @@ -27,5 +27,5 @@ export PATH=/usr/lib/ccache:$PATH pip install -vv -e . -pytest -n 1 -Wignore::DeprecationWarning -Wignore::FutureWarning sklearn/ +pytest -n 1 sklearn/ make test-sphinxext test-doc diff --git a/sklearn/tests/test_docstring_parameters.py b/sklearn/tests/test_docstring_parameters.py index 648de6b6e6ca5..ce8ec61327e00 100644 --- a/sklearn/tests/test_docstring_parameters.py +++ b/sklearn/tests/test_docstring_parameters.py @@ -17,6 +17,7 @@ from sklearn.utils.testing import check_docstring_parameters from sklearn.utils.testing import _get_func_name from sklearn.utils.testing import ignore_warnings +from sklearn.utils.testing import fails_if_pypy from sklearn.utils.deprecation import _is_deprecated PUBLIC_MODULES = set([pckg[1] for pckg in walk_packages(prefix='sklearn.', @@ -43,6 +44,7 @@ ] +@fails_if_pypy # this segfaults with PyPy-6.0 # numpydoc 0.8.0's docscrape tool raises because of collections.abc under # Python 3.7 @ignore_warnings(category=DeprecationWarning) diff --git a/sklearn/tests/test_multioutput.py b/sklearn/tests/test_multioutput.py index 1eb5a7e48f823..b181fc721f60e 100644 --- a/sklearn/tests/test_multioutput.py +++ b/sklearn/tests/test_multioutput.py @@ -14,6 +14,7 @@ from sklearn.utils.testing import assert_greater from sklearn.utils.testing import assert_not_equal from sklearn.utils.testing import assert_array_almost_equal +from sklearn.utils.testing import fails_if_pypy from sklearn import datasets from sklearn.base import clone from sklearn.datasets import make_classification @@ -162,6 +163,7 @@ def test_multi_target_sample_weights(): classes = list(map(np.unique, (y1, y2, y3))) +@fails_if_pypy # FIXME def test_multi_output_classification_partial_fit_parallelism(): sgd_linear_clf = SGDClassifier(loss='log', random_state=1, max_iter=5) mor = MultiOutputClassifier(sgd_linear_clf, n_jobs=-1) From c14542c6dba9c81fc3bb30ce5f534e4ccdb4e54d Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Sun, 16 Sep 2018 22:47:29 +0200 Subject: [PATCH 05/11] Skip test_show_versions_with_blas --- sklearn/tests/test_docstring_parameters.py | 2 +- sklearn/utils/tests/test_show_versions.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sklearn/tests/test_docstring_parameters.py b/sklearn/tests/test_docstring_parameters.py index ce8ec61327e00..8ebc1b5103327 100644 --- a/sklearn/tests/test_docstring_parameters.py +++ b/sklearn/tests/test_docstring_parameters.py @@ -44,10 +44,10 @@ ] -@fails_if_pypy # this segfaults with PyPy-6.0 # numpydoc 0.8.0's docscrape tool raises because of collections.abc under # Python 3.7 @ignore_warnings(category=DeprecationWarning) +@fails_if_pypy # this segfaults with PyPy-6.0 def test_docstring_parameters(): # Test module docstring formatting diff --git a/sklearn/utils/tests/test_show_versions.py b/sklearn/utils/tests/test_show_versions.py index f55bc8b945b6a..f51d6fcf2c2d8 100644 --- a/sklearn/utils/tests/test_show_versions.py +++ b/sklearn/utils/tests/test_show_versions.py @@ -2,6 +2,7 @@ from sklearn.utils._show_versions import _get_sys_info from sklearn.utils._show_versions import _get_deps_info from sklearn.utils._show_versions import show_versions +from sklearn.utils.testing import fails_if_pypy def test_get_sys_info(): @@ -24,6 +25,7 @@ def test_get_deps_info(): assert 'pandas' in deps_info +@fails_if_pypy def test_show_versions_with_blas(capsys): show_versions() out, err = capsys.readouterr() From 8e31357989d009f28d96d6fbcb21a5aae331b587 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 17 Sep 2018 15:53:59 +0200 Subject: [PATCH 06/11] Use LOKY_MAX_CPU_COUNT and python -m pytest --- build_tools/circle/build_test_pypy.sh | 6 ++++-- sklearn/tests/test_multioutput.py | 2 -- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build_tools/circle/build_test_pypy.sh b/build_tools/circle/build_test_pypy.sh index bb048a5e44241..ffccc0cb94c81 100755 --- a/build_tools/circle/build_test_pypy.sh +++ b/build_tools/circle/build_test_pypy.sh @@ -24,8 +24,10 @@ pip install "scipy>=1.1.0" sphinx numpydoc docutils pytest-xdist ccache -M 512M export CCACHE_COMPRESS=1 export PATH=/usr/lib/ccache:$PATH +export LOKY_MAX_CPU_COUNT="2" pip install -vv -e . -pytest -n 1 sklearn/ -make test-sphinxext test-doc +python -m pytest -n 1 sklearn/ +python -m pytest doc/sphinxext/ +python -m pytest $(find doc -name '*.rst' | sort) diff --git a/sklearn/tests/test_multioutput.py b/sklearn/tests/test_multioutput.py index b181fc721f60e..1eb5a7e48f823 100644 --- a/sklearn/tests/test_multioutput.py +++ b/sklearn/tests/test_multioutput.py @@ -14,7 +14,6 @@ from sklearn.utils.testing import assert_greater from sklearn.utils.testing import assert_not_equal from sklearn.utils.testing import assert_array_almost_equal -from sklearn.utils.testing import fails_if_pypy from sklearn import datasets from sklearn.base import clone from sklearn.datasets import make_classification @@ -163,7 +162,6 @@ def test_multi_target_sample_weights(): classes = list(map(np.unique, (y1, y2, y3))) -@fails_if_pypy # FIXME def test_multi_output_classification_partial_fit_parallelism(): sgd_linear_clf = SGDClassifier(loss='log', random_state=1, max_iter=5) mor = MultiOutputClassifier(sgd_linear_clf, n_jobs=-1) From ace507aa1f5c5702a59d84bc406b7d8db2d836e8 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 17 Sep 2018 16:39:09 +0200 Subject: [PATCH 07/11] Disable pytest-xdist --- build_tools/circle/build_test_pypy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/circle/build_test_pypy.sh b/build_tools/circle/build_test_pypy.sh index ffccc0cb94c81..922bbac8e23a6 100755 --- a/build_tools/circle/build_test_pypy.sh +++ b/build_tools/circle/build_test_pypy.sh @@ -19,7 +19,7 @@ python --version which python pip install --extra-index https://antocuni.github.io/pypy-wheels/ubuntu numpy Cython pytest -pip install "scipy>=1.1.0" sphinx numpydoc docutils pytest-xdist +pip install "scipy>=1.1.0" sphinx numpydoc docutils ccache -M 512M export CCACHE_COMPRESS=1 @@ -28,6 +28,6 @@ export LOKY_MAX_CPU_COUNT="2" pip install -vv -e . -python -m pytest -n 1 sklearn/ +python -m pytest sklearn/ python -m pytest doc/sphinxext/ python -m pytest $(find doc -name '*.rst' | sort) From 2c6816dd0c2356e41f1459c9d585e75729738e1d Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Mon, 17 Sep 2018 18:04:11 +0200 Subject: [PATCH 08/11] Skip test_docstring_parameters.py::test_docstring_parameters on PyPy --- sklearn/tests/test_docstring_parameters.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sklearn/tests/test_docstring_parameters.py b/sklearn/tests/test_docstring_parameters.py index 8ebc1b5103327..24b309d69fbbf 100644 --- a/sklearn/tests/test_docstring_parameters.py +++ b/sklearn/tests/test_docstring_parameters.py @@ -17,9 +17,10 @@ from sklearn.utils.testing import check_docstring_parameters from sklearn.utils.testing import _get_func_name from sklearn.utils.testing import ignore_warnings -from sklearn.utils.testing import fails_if_pypy from sklearn.utils.deprecation import _is_deprecated +import pytest + PUBLIC_MODULES = set([pckg[1] for pckg in walk_packages(prefix='sklearn.', path=sklearn.__path__) if not ("._" in pckg[1] or ".tests." in pckg[1])]) @@ -46,11 +47,13 @@ # numpydoc 0.8.0's docscrape tool raises because of collections.abc under # Python 3.7 -@ignore_warnings(category=DeprecationWarning) -@fails_if_pypy # this segfaults with PyPy-6.0 +@pytest.mark.filterwarnings('ignore::DeprecationWarning') def test_docstring_parameters(): # Test module docstring formatting + if IS_PYPY: + raise SkipTest('this tests segfaults on PyPy') + # Skip test if numpydoc is not found or if python version is < 3.5 try: import numpydoc # noqa From 80cfe618342b1dde2dc2488d3d45b0fea40892e7 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Tue, 18 Sep 2018 00:12:23 +0200 Subject: [PATCH 09/11] A few more fixes --- sklearn/tests/test_docstring_parameters.py | 2 +- sklearn/utils/tests/test_show_versions.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sklearn/tests/test_docstring_parameters.py b/sklearn/tests/test_docstring_parameters.py index 24b309d69fbbf..91102c00e782a 100644 --- a/sklearn/tests/test_docstring_parameters.py +++ b/sklearn/tests/test_docstring_parameters.py @@ -52,7 +52,7 @@ def test_docstring_parameters(): # Test module docstring formatting if IS_PYPY: - raise SkipTest('this tests segfaults on PyPy') + raise SkipTest('this test segfaults on PyPy') # Skip test if numpydoc is not found or if python version is < 3.5 try: diff --git a/sklearn/utils/tests/test_show_versions.py b/sklearn/utils/tests/test_show_versions.py index f51d6fcf2c2d8..f55bc8b945b6a 100644 --- a/sklearn/utils/tests/test_show_versions.py +++ b/sklearn/utils/tests/test_show_versions.py @@ -2,7 +2,6 @@ from sklearn.utils._show_versions import _get_sys_info from sklearn.utils._show_versions import _get_deps_info from sklearn.utils._show_versions import show_versions -from sklearn.utils.testing import fails_if_pypy def test_get_sys_info(): @@ -25,7 +24,6 @@ def test_get_deps_info(): assert 'pandas' in deps_info -@fails_if_pypy def test_show_versions_with_blas(capsys): show_versions() out, err = capsys.readouterr() From b82b41678c6b27654becd278b93bcb4544c08668 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Tue, 18 Sep 2018 15:10:33 +0200 Subject: [PATCH 10/11] Address Joel's comments --- sklearn/tests/test_docstring_parameters.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sklearn/tests/test_docstring_parameters.py b/sklearn/tests/test_docstring_parameters.py index 91102c00e782a..b4a831e571c4a 100644 --- a/sklearn/tests/test_docstring_parameters.py +++ b/sklearn/tests/test_docstring_parameters.py @@ -48,12 +48,10 @@ # numpydoc 0.8.0's docscrape tool raises because of collections.abc under # Python 3.7 @pytest.mark.filterwarnings('ignore::DeprecationWarning') +@pytest.mark.skipif(IS_PYPY, reason='test segfaults on PyPy') def test_docstring_parameters(): # Test module docstring formatting - if IS_PYPY: - raise SkipTest('this test segfaults on PyPy') - # Skip test if numpydoc is not found or if python version is < 3.5 try: import numpydoc # noqa From ba688b71019e8c47ce905eb82b31ef30df715538 Mon Sep 17 00:00:00 2001 From: Roman Yurchak Date: Tue, 18 Sep 2018 15:24:13 +0200 Subject: [PATCH 11/11] Setup CircleCI cron job --- .circleci/config.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index effb37d6b8faa..e1e410c440314 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -104,7 +104,21 @@ workflows: jobs: - python3 - python2 - - pypy3 + - pypy3: + filters: + branches: + only: + - 0.20.X - deploy: requires: - python3 + pypy: + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - master + jobs: + - pypy3