8000 Merge pull request #6730 from Kojoley/improve-tests · matplotlib/matplotlib@84f471e · GitHub
[go: up one dir, main page]
Skip to content

Commit 84f471e

Browse files
tacaswelltacaswell
authored
Merge pull request #6730 from Kojoley/improve-tests
TST: Add Py.test testing framework support
2 parents 36f1353 + a66d599 commit 84f471e

File tree

8 files changed

+207
-25
lines changed

8 files changed

+207
-25
lines changed

‎.travis.yml

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,13 @@ env:
4545
- NPROC=2
4646
- TEST_ARGS=--no-pep8
4747
- NOSE_ARGS="--processes=$NPROC --process-timeout=300"
48+
- PYTEST_ARGS="-ra --timeout=300 --durations=25 --cov-report= --cov=lib" # -n $NPROC
4849
- PYTHON_ARGS=
4950
- DELETE_FONT_CACHE=
51+
- USE_PYTEST=false
52+
#- PYTHONHASHSEED=0 # Workaround for pytest-xdist flaky colletion order
53+
# # https://github.com/pytest-dev/pytest/issues/920
54+
# # https://github.com/pytest-dev/pytest/issues/1075
5055

5156
matrix:
5257
include:
@@ -60,6 +65,8 @@ matrix:
6065
env: TEST_ARGS=--pep8
6166
- python: 3.5
6267
env: BUILD_DOCS=true
68+
- python: 3.5
69+
env: USE_PYTEST=true PANDAS=pandas DELETE_FONT_CACHE=1 TEST_ARGS=
6370
- python: "nightly"
6471
env: PRE=--pre
6572
- os: osx
@@ -107,10 +114,14 @@ install:
107114
# Install dependencies from pypi
108115
pip install $PRE python-dateutil $NUMPY pyparsing!=2.1.6 $PANDAS pep8 cycler coveralls coverage
109116
pip install $PRE pillow sphinx!=1.3.0 $MOCK numpydoc ipython colorspacious
117+
110118
# Install nose from a build which has partial
111119
# support for python36 and suport for coverage output suppressing
112120
pip install git+https://github.com/jenshnielsen/nose.git@matplotlibnose
113121
122+
# pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124
123+
pip install $PRE pytest 'pytest-cov>=2.3.1' pytest-timeout pytest-xdist pytest-faulthandler
124+
114125
# We manually install humor sans using the package from Ubuntu 14.10. Unfortunatly humor sans is not
115126
# availible in the Ubuntu version used by Travis but we can manually install the deb from a later
116127
# version since is it basically just a .ttf file
@@ -147,16 +158,21 @@ script:
147158
- |
148159
echo Testing import of tkagg backend
149160
MPLBACKEND="tkagg" python -c 'import matplotlib.pyplot as plt; print(plt.get_backend())'
150-
echo The following args are passed to nose $NOSE_ARGS
151161
if [[ $BUILD_DOCS == false ]]; then
152162
if [[ $DELETE_FONT_CACHE == 1 ]]; then
153163
rm -rf ~/.cache/matplotlib
154164
fi
155165
export MPL_REPO_DIR=$PWD # needed for pep8-conformance test of the examples
156-
if [[ $TRAVIS_OS_NAME == 'osx' ]]; then
157-
python tests.py $NOSE_ARGS $TEST_ARGS
166+
if [[ $USE_PYTEST == false ]]; then
167+
echo The following args are passed to nose $NOSE_ARGS
168+
if [[ $TRAVIS_OS_NAME == 'osx' ]]; then
169+
python tests.py $NOSE_ARGS $TEST_ARGS
170+
else
171+
gdb -return-child-result -batch -ex r -ex bt --args python $PYTHON_ARGS tests.py $NOSE_ARGS $TEST_ARGS
172+
fi
158173
else
159-
gdb -return-child-result -batch -ex r -ex bt --args python $PYTHON_ARGS tests.py $NOSE_ARGS $TEST_ARGS
174+
echo The following args are passed to pytest $PYTEST_ARGS
175+
py.test $PYTEST_ARGS $TEST_ARGS
160176
fi
161177
else
162178
cd doc
@@ -171,6 +187,9 @@ script:
171187
pip install $PRE requests==2.9.2 linkchecker
172188
linkchecker build/html/index.html
173189
fi
190+
# Currently disabled because of differece in behaviour
191+
# between `pytest-cov` and `nose-coverage`
192+
#- if [[ $USE_PYTEST == true ]]; then coveralls; fi
174193
- rm -rf $HOME/.cache/matplotlib/tex.cache
175194
- rm -rf $HOME/.cache/matplotlib/test_cache
176195

‎appveyor.yml

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,12 @@ environment:
1414
CMD_IN_ENV: "cmd /E:ON /V:ON /C obvci_appveyor_python_build_env.cmd"
1515
# Workaround for https://github.com/conda/conda-build/issues/636
1616
PYTHONIOENCODING: "UTF-8"
17+
TEST_ARGS: --no-pep8
18+
PYTEST_ARGS: -ra --timeout=300 --durations=25 #--cov-report= --cov=lib #-n %NUMBER_OF_PROCESSORS%
19+
USE_PYTEST: no
20+
#PYTHONHASHSEED: 0 # Workaround for pytest-xdist flaky colletion order
21+
# # https://github.com/pytest-dev/pytest/issues/920
22+
# # https://github.com/pytest-dev/pytest/issues/1075
1723

1824
matrix:
1925
# for testing purpose: numpy 1.8 on py2.7, for the rest use 1.10/latest
@@ -38,6 +44,13 @@ environment:
3844
PYTHON_VERSION: "3.5"
3945
TEST_ALL: "no"
4046
CONDA_INSTALL_LOCN: "C:\\Miniconda35-x64"
47+
- TARGET_ARCH: "x64"
48+
CONDA_PY: "35"
49+
CONDA_NPY: "110"
50+
PYTHON_VERSION: "3.5"
51+
TEST_ALL: "no"
52+
CONDA_INSTALL_LOCN: "C:\\Miniconda35-x64"
53+
USE_PYTEST: yes
4154
- TARGET_ARCH: "x86"
4255
CONDA_PY: "27"
4356
CONDA_NPY: "18"
@@ -58,7 +71,7 @@ platform:
5871
build: false
5972

6073
init:
61-
- cmd: "ECHO %PYTHON_VERSION% %CONDA_INSTALL_LOCN%"
74+
- cmd: "ECHO %PYTHON_VERSION% PYTEST=%USE_PYTEST% %CONDA_INSTALL_LOCN%"
6275

6376
install:
6477
- cmd: set PATH=%CONDA_INSTALL_LOCN%;%CONDA_INSTALL_LOCN%\scripts;%PATH%;
@@ -82,10 +95,15 @@ install:
8295
# same things as the requirements in ci/conda_recipe/meta.yaml
8396
# if conda-forge gets a new pyqt, it might be nice to install it as well to have more backends
8497
# https://github.com/conda-forge/conda-forge.github.io/issues/157#issuecomment-223536381
85-
- cmd: conda create -q -n test-environment python=%PYTHON_VERSION% pip setuptools numpy python-dateutil freetype=2.6 msinttypes "tk=8.5" pyparsing pytz tornado "libpng>=1.6.21,<1.7" "zlib=1.2" "cycler>=0.10" nose mock
98+
- conda create -q -n test-environment python=%PYTHON_VERSION%
99+
pip setuptools numpy python-dateutil freetype=2.6 msinttypes "tk=8.5"
100+
pyparsing pytz tornado "libpng>=1.6.21,<1.7" "zlib=1.2" "cycler>=0.10"
101+
nose mock sphinx
86102
- activate test-environment
87103
- cmd: echo %PYTHON_VERSION% %TARGET_ARCH%
88104
- cmd: IF %PYTHON_VERSION% == 2.7 conda install -q functools32
105+
# pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124
106+
- if x%USE_PYTEST% == xyes conda install -q pytest "pytest-cov>=2.3.1" pytest-timeout #pytest-xdist
89107

90108
# Let the install prefer the static builds of the libs
91109
- set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib
@@ -124,7 +142,9 @@ test_script:
124142
# Test import of tkagg backend
125143
- python -c "import matplotlib as m; m.use('tkagg'); import matplotlib.pyplot as plt; print(plt.get_backend())"
126144
# tests
127-
- python tests.py
145+
- if x%USE_PYTEST% == xyes echo The following args are passed to pytest %PYTEST_ARGS%
146+
- if x%USE_PYTEST% == xyes py.test %PYTEST_ARGS% %TEST_ARGS%
147+
- if x%USE_PYTEST% == xno python tests.py %TEST_ARGS%
128148
# Generate a html for visual tests
129149
- python visual_tests.py
130150

‎conftest.py

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
from __future__ import (absolute_import, division, print_function,
2+
unicode_literals)
3+
4+
import inspect
5+
import os
6+
import pytest
7+
import unittest
8+
9+
import matplotlib
10+
matplotlib.use('agg')
11+
12+
from matplotlib import default_test_modules
13+
from matplotlib.testing.decorators import ImageComparisonTest
14+
15+
16+
IGNORED_TESTS = {
17+
'matplotlib': [
18+
'test_usetex',
19+
],
20+
}
21+
22+
23+
def blacklist_check(path):
24+
"""Check if test is blacklisted and should be ignored"""
25+
head, tests_dir = os.path.split(path.dirname)
26+
if tests_dir != 'tests':
27+
return True
28+
head, top_module = os.path.split(head)
29+
return path.purebasename in IGNORED_TESTS.get(top_module, [])
30+
31+
32+
def whitelist_check(path):
33+
"""Check if test is not whitelisted and should be ignored"""
34+
left = path.dirname
35+
last_left = None
36+
module_path = path.purebasename
37+
while len(left) and left != last_left:
38+
last_left = left
39+
left, tail = os.path.split(left)
40+
module_path = '.'.join([tail, module_path])
41+
if module_path in default_test_modules:
42+
return False
43+
return True
44+
45+
46+
COLLECT_FILTERS = {
47+
'none': lambda _: False,
48+
'blacklist': blacklist_check,
49+
'whitelist': whitelist_check,
50+
}
51+
52+
53+
def is_nose_class(cls):
54+
"""Check if supplied class looks like Nose testcase"""
55+
return any(name in ['setUp', 'tearDown']
56+
for name, _ in inspect.getmembers(cls))
57+
58+
59+
def pytest_addoption(parser):
60+
group = parser.getgroup("matplotlib", "matplotlib custom options")
61+
62+
group.addoption('--collect-filter', action='store',
63+
choices=COLLECT_FILTERS, default='blacklist',
64+
help='filter tests during collection phase')
65+
66+
group.addoption('--no-pep8', action='store_true',
67+
help='skip PEP8 compliance tests')
68+
69+
70+
def pytest_configure(config):
71+
matplotlib._called_from_pytest = True
72+
matplotlib._init_tests()
73+
74+
if config.getoption('--no-pep8'):
75+
default_test_modules.remove('matplotlib.tests.test_coding_standards')
76+
IGNORED_TESTS['matplotlib'] += 'test_coding_standards'
77+
78+
79+
def pytest_unconfigure(config):
80+
matplotlib._called_from_pytest = False
81+
82+
83+
def pytest_ignore_collect(path, config):
84+
if path.ext == '.py':
85+
collect_filter = config.getoption('--collect-filter')
86+
return COLLECT_FILTERS[collect_filter](path)
87+
88+
89+
def pytest_pycollect_makeitem(collector, name, obj):
90+
if inspect.isclass(obj):
91+
if issubclass(obj, ImageComparisonTest):
92+
# Workaround `image_compare` decorator as it returns class
93+
# instead of function and this confuses pytest because it crawls
94+
# original names and sees 'test_*', but not 'Test*' in that case
95+
return pytest.Class(name, parent=collector)
96+
97+
if is_nose_class(obj) and not issubclass(obj, unittest.TestCase):
98+
# Workaround unittest-like setup/teardown names in pure classes
99+
setup = getattr(obj, 'setUp', None)
100+
if setup is not None:
101+
obj.setup_method = lambda self, _: obj.setUp(self)
102+
tearDown = getattr(obj, 'tearDown', None)
103+
if tearDown is not None:
104+
obj.teardown_method = lambda self, _: obj.tearDown(self)
105+
setUpClass = getattr(obj, 'setUpClass', None)
106+
if setUpClass is not None:
107+
obj.setup_class = obj.setUpClass
108+
tearDownClass = getattr(obj, 'tearDownClass', None)
109+
if tearDownClass is not None:
110+
obj.teardown_class = obj.tearDownClass
111+
112+
return pytest.Class(name, parent=collector)

‎lib/matplotlib/testing/__init__.py

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import warnings
66
from contextlib import contextmanager
77

8+
import matplotlib
89
from matplotlib.cbook import is_string_like, iterable
910
from matplotlib import rcParams, rcdefaults, use
1011

@@ -14,16 +15,31 @@ def _is_list_like(obj):
1415
return not is_string_like(obj) and iterable(obj)
1516

1617

18+
def is_called_from_pytest():
19+
"""Returns whether the call was done from pytest"""
20+
return getattr(matplotlib, '_called_from_pytest', False)
21+
22+
1723
def xfail(msg=""):
1824
"""Explicitly fail an currently-executing test with the given message."""
19-
from .nose import knownfail
20-
knownfail(msg)
25+
__tracebackhide__ = True
26+
if is_called_from_pytest():
27+
import pytest
28+
pytest.xfail(msg)
29+
else:
30+
from .nose import knownfail
31+
knownfail(msg)
2132

2233

2334
def skip(msg=""):
2435
"""Skip an executing test with the given message."""
25-
from nose import SkipTest
26-
raise SkipTest(msg)
36+
__tracebackhide__ = True
37+
if is_called_from_pytest():
38+
import pytest
39+
pytest.skip(msg)
40+
else:
41+
from nose import SkipTest
42+
raise SkipTest(msg)
2743

2844

2945
# stolen from pytest

‎lib/matplotlib/testing/decorators.py

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
from matplotlib import rcParams
2828
from matplotlib.testing.compare import comparable_formats, compare_images, \
2929
make_test_filename
30-
from . import copy_metadata, skip, xfail
30+
from . import copy_metadata, is_called_from_pytest, skip, xfail
3131
from .exceptions import ImageComparisonFailure
3232

3333

@@ -37,8 +37,12 @@ def skipif(condition, *args, **kwargs):
3737
3838
Optionally specify a reason for better reporting.
3939
"""
40-
from .nose.decorators import skipif
41-
return skipif(condition, *args, **kwargs)
40+
if is_called_from_pytest():
41+
import pytest
42+
return pytest.mark.skipif(condition, *args, **kwargs)
43+
else:
44+
from .nose.decorators import skipif
45+
return skipif(condition, *args, **kwargs)
4246

4347

4448
def knownfailureif(fail_condition, msg=None, known_exception_class=None):
@@ -53,8 +57,14 @@ def knownfailureif(fail_condition, msg=None, known_exception_class=None):
5357
if the exception is an instance of this class. (Default = None)
5458
5559
"""
56-
from .nose.decorators import knownfailureif
57-
return knownfailureif(fail_condition, msg, known_exception_class)
60+
if is_called_from_pytest():
61+
import pytest
62+
strict = fail_condition and fail_condition != 'indeterminate'
63+
return pytest.mark.xfail(condition=fail_condition, reason=msg,
64+
raises=known_exception_class, strict=strict)
65+
else:
66+
from .nose.decorators import knownfailureif
67+
return knownfailureif(fail_condition, msg, known_exception_class)
5868

5969

6070
def _do_cleanup(original_units_registry, original_settings):
@@ -198,7 +208,7 @@ def remove_text(figure):
198208
def test(self):
199209
baseline_dir, result_dir = _image_directories(self._func)
200210
if self._style != 'classic':
201-
xfail('temporarily disabled until 2.0 tag')
211+
skip('temporarily disabled until 2.0 tag')
202212
for fignum, baseline in zip(plt.get_fignums(), self._baseline_images):
203213
for extension in self._extensions:
204214
will_fail = not extension in comparable_formats()
@@ -228,7 +238,7 @@ def test(self):
228238
@knownfailureif(
229239
will_fail, fail_msg,
230240
known_exception_class=ImageComparisonFailure)
231-
def do_test():
241+
def do_test(fignum, actual_fname, expected_fname):
232242
figure = plt.figure(fignum)
233243

234244
if self._remove_text:
@@ -255,7 +265,7 @@ def do_test():
255265
(self._freetype_version, ft2font.__freetype_version__))
256266
raise
257267

258-
yield (do_test,)
268+
yield do_test, fignum, actual_fname, expected_fname
259269

260270

261271
def image_comparison(baseline_images=None, extensions=None, tol=0,

‎lib/matplotlib/tests/test_axes.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
import warnings
2323

2424
import matplotlib
25-
from matplotlib.testing.decorators import image_comparison, cleanup, skipif
25+
from matplotlib.testing.decorators import image_comparison, cleanup
26+
from matplotlib.testing import skip
2627
import matplotlib.pyplot as plt
2728
import matplotlib.markers as mmarkers
2829
import matplotlib.patches as mpatches
@@ -86,11 +87,11 @@ def test_formatter_ticker():
8687
ax.autoscale_view()
8788

8889

89-
@skipif(LooseVersion(np.__version__) >= LooseVersion('1.11.0'),
90-
reason="Fall out from a fixed numpy bug")
9190
@image_comparison(baseline_images=["formatter_large_small"])
9291
def test_formatter_large_small():
9392
# github issue #617, pull #619
93+
if LooseVersion(np.__version__) >= LooseVersion('1.11.0'):
94+
skip("Fall out from a fixed numpy bug")
9495
fig, ax = plt.subplots(1)
9596
x = [0.500000001, 0.500000002]
9697
y = [1e64, 1.1e64]

0 commit comments

Comments
 (0)
0