From bdb8682b3ce13e2bcd7be9e0d352f4a1b971e348 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 5 Feb 2017 17:29:33 -0500 Subject: [PATCH 01/14] TST: Close figures before image comparison. Since this decorator assumes that the figures available after are the ones to check, it should close any existing figures or it will get the wrong count. --- lib/matplotlib/testing/decorators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py index 1a6353cb8481..034a5174013a 100644 --- a/lib/matplotlib/testing/decorators.py +++ b/lib/matplotlib/testing/decorators.py @@ -252,6 +252,7 @@ def delayed_init(self, func): def setup(self): func = self.func + plt.close('all') self.setup_class() try: matplotlib.style.use(self.style) From b71447f9a4b1c801aaeed401a699838d13fb2a06 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 5 Feb 2017 17:54:53 -0500 Subject: [PATCH 02/14] Remove test_mouseclicks and copy some bits to docs. The name is getting collected as a test by pytest when it isn't one, and it isn't much of an example anyway. --- doc/users/event_handling.rst | 8 +++--- examples/event_handling/test_mouseclicks.py | 32 --------------------- 2 files changed, 4 insertions(+), 36 deletions(-) delete mode 100755 examples/event_handling/test_mouseclicks.py diff --git a/doc/users/event_handling.rst b/doc/users/event_handling.rst index 2310e93f8735..4545fefee618 100644 --- a/doc/users/event_handling.rst +++ b/doc/users/event_handling.rst @@ -29,13 +29,13 @@ connect your function to the event manager, which is part of the example that prints the location of the mouse click and which button was pressed:: - fig = plt.figure() - ax = fig.add_subplot(111) + fig, ax = plt.subplots() ax.plot(np.random.rand(10)) def onclick(event): - print('button=%d, x=%d, y=%d, xdata=%f, ydata=%f' % - (event.button, event.x, event.y, event.xdata, event.ydata)) + print('%s click: button=%d, x=%d, y=%d, xdata=%f, ydata=%f' % + ('double' if event.dblclick else 'single', event.button, + event.x, event.y, event.xdata, event.ydata)) cid = fig.canvas.mpl_connect('button_press_event', onclick) diff --git a/examples/event_handling/test_mouseclicks.py b/examples/event_handling/test_mouseclicks.py deleted file mode 100755 index bfd26ab494cf..000000000000 --- a/examples/event_handling/test_mouseclicks.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import print_function - -import matplotlib -#matplotlib.use("WxAgg") -#matplotlib.use("TkAgg") -#matplotlib.use("GTKAgg") -#matplotlib.use("Qt4Agg") -#matplotlib.use("MacOSX") -import matplotlib.pyplot as plt - -#print("***** TESTING WITH BACKEND: %s"%matplotlib.get_backend() + " *****") - - -def OnClick(event): - if event.dblclick: - print("DBLCLICK", event) - else: - print("DOWN ", event) - - -def OnRelease(event): - print("UP ", event) - - -fig = plt.gcf() -cid_up = fig.canvas.mpl_connect('button_press_event', OnClick) -cid_down = fig.canvas.mpl_connect('button_release_event', OnRelease) - -plt.gca().text(0.5, 0.5, "Click on the canvas to test mouse events.", - ha="center", va="center") - -plt.show() From c632e41acb73d47079888ca9507925687d47a436 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 5 Feb 2017 18:19:46 -0500 Subject: [PATCH 03/14] Use pytest.importorskip for consistent skip message. It's used in some places, but not all. --- lib/matplotlib/tests/test_axes.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/matplotlib/tests/test_axes.py b/lib/matplotlib/tests/test_axes.py index 40a107b252aa..5685aca786d7 100644 --- a/lib/matplotlib/tests/test_axes.py +++ b/lib/matplotlib/tests/test_axes.py @@ -4655,10 +4655,7 @@ def test_broken_barh_empty(): def test_pandas_indexing_dates(): - try: - import pandas as pd - except ImportError: - pytest.skip("Pandas not installed") + pd = pytest.importorskip('pandas') dates = np.arange('2005-02', '2005-03', dtype='datetime64[D]') values = np.sin(np.array(range(len(dates)))) @@ -4671,10 +4668,7 @@ def test_pandas_indexing_dates(): def test_pandas_errorbar_indexing(): - try: - import pandas as pd - except ImportError: - pytest.skip("Pandas not installed") + pd = pytest.importorskip('pandas') df = pd.DataFrame(np.random.uniform(size=(5, 4)), columns=['x', 'y', 'xe', 'ye'], @@ -4684,10 +4678,7 @@ def test_pandas_errorbar_indexing(): def test_pandas_indexing_hist(): - try: - import pandas as pd - except ImportError: - pytest.skip("Pandas not installed") + pd = pytest.importorskip('pandas') ser_1 = pd.Series(data=[1, 2, 2, 3, 3, 4, 4, 4, 4, 5]) ser_2 = ser_1.iloc[1:] From 113d0791cdb6b0c45b511ebc75288a20ebefb2ed Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 5 Feb 2017 18:09:46 -0500 Subject: [PATCH 04/14] Remove nose from tox environment. --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index ae8f1e90be3f..36876f1ebef4 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,6 @@ commands = sh -c 'rm -f $HOME/.matplotlib/fontList*' {envpython} {toxinidir}/tests.py --processes=-1 --process-timeout=300 deps = - nose mock numpy - pytest \ No newline at end of file + pytest From 9d36e988da3f066b0a3d83a70a927e43313b7797 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 2 Feb 2017 19:04:27 -0500 Subject: [PATCH 05/14] Document markers used by pytest. This may or may not become a requirement in the future. --- pytest.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pytest.ini b/pytest.ini index d9132bfa6002..bb9d46d4db8f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,12 @@ [pytest] norecursedirs = .git build ci dist doc extern lib/mpl_examples release tools unit venv python_files = test_*.py + +markers = + backend: Set alternate Matplotlib backend temporarily. + network: Mark a test that uses the network. + style: Set alternate Matplotlib style temporarily. + pep8ignore = * E111 E114 E115 E116 E121 E122 E123 E124 E125 E126 E127 E128 E129 E131 E226 E240 E241 E242 E243 E244 E245 E246 E247 E248 E249 E265 E266 E704 W503 From d9fbe8d14bd1f699ab6e8f41ff64ea8f62cb3b2b Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 5 Feb 2017 18:45:17 -0500 Subject: [PATCH 06/14] Fix tests.py when passing modules instead of paths. --- lib/matplotlib/__init__.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/matplotlib/__init__.py b/lib/matplotlib/__init__.py index 86631ab05300..ad1e411ea6ab 100644 --- a/lib/matplotlib/__init__.py +++ b/lib/matplotlib/__init__.py @@ -1536,8 +1536,21 @@ def test(verbosity=None, coverage=False, switch_backend_warn=True, import pytest args = kwargs.pop('argv', []) - if not any(os.path.exists(arg) for arg in args): - args += ['--pyargs'] + default_test_modules + provide_default_modules = True + use_pyargs = True + for arg in args: + if any(arg.startswith(module_path) + for module_path in default_test_modules): + provide_default_modules = False + break + if os.path.exists(arg): + provide_default_modules = False + use_pyargs = False + break + if use_pyargs: + args += ['--pyargs'] + if provide_default_modules: + args += default_test_modules if coverage: args += ['--cov'] From 244806d77f2f6df89540b217a00158d133353436 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 5 Feb 2017 20:11:05 -0500 Subject: [PATCH 07/14] Update documentation to use pytest for testing. --- INSTALL | 2 +- README.rst | 6 +- doc/devel/contributing.rst | 10 +-- doc/devel/testing.rst | 144 ++++++++++++++++++------------------- examples/README.txt | 2 +- 5 files changed, 83 insertions(+), 81 deletions(-) diff --git a/INSTALL b/INSTALL index c6b340d19e70..fc91fad8dd61 100644 --- a/INSTALL +++ b/INSTALL @@ -110,7 +110,7 @@ To run the test suite, copy the :file:`lib\\matplotlib\\tests` and :file:`lib\\mpl_toolkits\\tests` directories from the source distribution to :file:`sys.prefix\\Lib\\site-packages\\matplotlib` and :file:`sys.prefix\\Lib\\site-packages\\mpl_toolkits` respectively, and -install `nose `_, `mock +install `pytest `_, `mock `_, Pillow, MiKTeX, GhostScript, ffmpeg, avconv, mencoder, ImageMagick, and `Inkscape `_. diff --git a/README.rst b/README.rst index 8d98da834a1a..66827cac69b3 100644 --- a/README.rst +++ b/README.rst @@ -28,9 +28,9 @@ Or from the Python interpreter:: matplotlib.test() Consider reading http://matplotlib.org/devel/coding_guide.html#testing for -more information. Note that the test suite requires nose and on Python 2.7 mock -which are not installed by default. Please install with pip or your package -manager of choice. +more information. Note that the test suite requires pytest and on Python 2.7 +mock which are not installed by default. Please install with pip or your +package manager of choice. Contact ======= diff --git a/doc/devel/contributing.rst b/doc/devel/contributing.rst index 5ad8482f39ef..b6c237935b11 100644 --- a/doc/devel/contributing.rst +++ b/doc/devel/contributing.rst @@ -120,14 +120,16 @@ environment is set up properly:: python tests.py -.. _nose: https://nose.readthedocs.io/en/latest/ +.. _pytest: http://doc.pytest.org/en/latest/ .. _pep8: https://pep8.readthedocs.io/en/latest/ +.. _mock: https://docs.python.org/dev/library/unittest.mock.html +.. _Ghostscript: https://www.ghostscript.com/ +.. _Inkscape: https://inkscape.org> .. note:: - **Additional dependencies for testing**: nose_ (version 1.0 or later), `mock - `_ (if python < 3.3), `Ghostscript - `_, `Inkscape `_ + **Additional dependencies for testing**: pytest_ (version 3.0 or later), + mock_ (if python < 3.3), Ghostscript_, Inkscape_ .. seealso:: diff --git a/doc/devel/testing.rst b/doc/devel/testing.rst index 200d6a5c6507..b513cb865c47 100644 --- a/doc/devel/testing.rst +++ b/doc/devel/testing.rst @@ -4,31 +4,34 @@ Developer's tips for testing ============================ -Matplotlib has a testing infrastructure based on nose_, making it easy -to write new tests. The tests are in :mod:`matplotlib.tests`, and -customizations to the nose testing infrastructure are in -:mod:`matplotlib.testing`. (There is other old testing cruft around, -please ignore it while we consolidate our testing to these locations.) - -.. _nose: https://nose.readthedocs.io/en/latest/ +Matplotlib has a testing infrastructure based on pytest_, making it easy to +write new tests. The tests are in :mod:`matplotlib.tests`, and customizations +to the pytest testing infrastructure are in :mod:`matplotlib.tests.conftest` +and :mod:`matplotlib.testing`. (There is other old testing cruft around, please +ignore it while we consolidate our testing to these locations.) + +.. _pytest: http://doc.pytest.org/en/latest/ +.. _mock: https://docs.python.org/dev/library/unittest.mock.html> +.. _Ghostscript: https://www.ghostscript.com/ +.. _Inkscape: https://inkscape.org +.. _pytest-cov: https://pytest-cov.readthedocs.io/en/latest/ +.. _pytest-pep8: https://pypi.python.org/pypi/pytest-pep8 Requirements ------------ The following software is required to run the tests: - - nose_, version 1.0 or later - - `mock `_, when running python - versions < 3.3 - - `Ghostscript `_ (to render PDF - files) - - `Inkscape `_ (to render SVG files) + - pytest_, version 3.0.0 or later + - mock_, when running Python versions < 3.3 + - Ghostscript_ (to render PDF files) + - Inkscape_ (to render SVG files) Optionally you can install: - - `coverage `_ to collect coverage - information - - `pep8 `_ to test coding standards + - pytest-cov_ to collect coverage information + - pytest-pep8_ to test coding standards + Building matplotlib for image comparison tests ---------------------------------------------- @@ -53,7 +56,11 @@ value. Running the tests ----------------- -Running the tests is simple. Make sure you have nose installed and run:: +Running the tests is simple. Make sure you have pytest installed and run:: + + py.test + +or:: python tests.py @@ -61,50 +68,60 @@ in the root directory of the distribution. The script takes a set of commands, such as: ======================== =========== -``--pep8`` pep8 checks -``--no-pep8`` Do not perform pep8 checks +``--pep8`` Perform pep8 checks (requires pytest-pep8_) ``--no-network`` Disable tests that require network access ======================== =========== -Additional arguments are passed on to nosetests. See the nose -documentation for supported arguments. Some of the more important ones are given -here: +Additional arguments are passed on to pytest. See the pytest documentation for +supported arguments. Some of the more important ones are given here: ============================= =========== ``--verbose`` Be more verbose -``--processes=NUM`` Run tests in parallel over NUM processes -``--process-timeout=SECONDS`` Set timeout for results from test runner process -``--nocapture`` Do not capture stdout +``--n NUM`` Run tests in parallel over NUM + processes (requires pytest-xdist_) +``--timeout=SECONDS`` Set timeout for results from each test + process (requires pytest-timeout_) +``--capture=no`` or ``-s`` Do not capture stdout ============================= =========== -To run a single test from the command line, you can provide a -dot-separated path to the module followed by the function separated by -a colon, e.g., (this is assuming the test is installed):: +To run a single test from the command line, you can provide a dot-separated +path to the module, optionally followed by the function separated by two +colons, e.g., (this is assuming the test is installed):: - python tests.py matplotlib.tests.test_simplification:test_clipping + python tests.py matplotlib.tests.test_simplification::test_clipping + +or by passing a file path, optionally followed by the function separated by two +colons, e.g., (tests do not need to be installed, but Matplotlib should be):: + + python tests.py lib/matplotlib/tests/test_simplification.py::test_clipping If you want to run the full test suite, but want to save wall time try running the tests in parallel:: - python tests.py --nocapture --verbose --processes=5 --process-timeout=300 + python tests.py --capture=no --verbose -n 5 + +Depending on your version of Python and pytest-xdist, you may need to set +``PYTHONHASHSEED`` to a fixed value when running in parallel:: + PYTHONHASHSEED=0 python tests.py --capture=no --verbose -n 5 -An alternative implementation that does not look at command line -arguments works from within Python is to run the tests from the -matplotlib library function :func:`matplotlib.test`:: +An alternative implementation that does not look at command line arguments +and works from within Python is to run the tests from the Matplotlib library +function :func:`matplotlib.test`:: import matplotlib matplotlib.test() .. hint:: - To run the tests you need to install nose and mock if using python 2.7:: + To run the tests you need to install pytest and mock if using python 2.7:: - pip install nose + pip install pytest pip install mock -.. _`nosetest arguments`: http://nose.readthedocs.io/en/latest/usage.html +.. _pytest-xdist: https://pypi.python.org/pypi/pytest-xdist +.. _pytest-timeout: https://pypi.python.org/pypi/pytest-timeout Writing a simple test @@ -113,30 +130,20 @@ Writing a simple test Many elements of Matplotlib can be tested using standard tests. For example, here is a test from :mod:`matplotlib.tests.test_basic`:: - from nose.tools import assert_equal - def test_simple(): """ very simple example test """ - assert_equal(1+1,2) - -Nose determines which functions are tests by searching for functions -beginning with "test" in their name. - -If the test has side effects that need to be cleaned up, such as -creating figures using the pyplot interface, use the ``@cleanup`` -decorator:: + assert 1 + 1 == 2 - from matplotlib.testing.decorators import cleanup +Pytest determines which functions are tests by searching for files whose names +begin with ``"test_"`` and then within those files for functions beginning with +``"test"`` or classes beginning with ``"Test"``. - @cleanup - def test_create_figure(): - """ - very simple example test that creates a figure using pyplot. - """ - fig = figure() - ... +Tests that have side effects that need to be cleaned up, such as created +figures using the pyplot interface or modified rc params, will be automatically +reset by the pytest fixture +:func:`~matplotlib.tests.conftest.mpl_test_settings`. Writing an image comparison test @@ -203,24 +210,22 @@ decorator: Known failing tests ------------------- -If you're writing a test, you may mark it as a known failing test with -the :func:`~matplotlib.testing.decorators.knownfailureif` -decorator. This allows the test to be added to the test suite and run -on the buildbots without causing undue alarm. For example, although -the following test will fail, it is an expected failure:: +If you're writing a test, you may mark it as a known failing test with the +:func:`pytest.mark.xfail` decorator. This allows the test to be added to the +test suite and run on the buildbots without causing undue alarm. For example, +although the following test will fail, it is an expected failure:: - from nose.tools import assert_equal - from matplotlib.testing.decorators import knownfailureif + import pytest - @knownfailureif(True) + @pytest.mark.xfail def test_simple_fail(): '''very simple example test that should fail''' - assert_equal(1+1,3) + assert 1 + 1 == 3 -Note that the first argument to the -:func:`~matplotlib.testing.decorators.knownfailureif` decorator is a -fail condition, which can be a value such as True, False, or -'indeterminate', or may be a dynamically evaluated expression. +Note that the first argument to the :func:`~pytest.mark.xfail` decorator is a +fail condition, which can be a value such as True, False, or may be a +dynamically evaluated expression. If a condition is supplied, then a reason +must also be supplied with the ``reason='message'`` keyword argument. Creating a new module in matplotlib.tests ----------------------------------------- @@ -229,11 +234,6 @@ We try to keep the tests categorized by the primary module they are testing. For example, the tests related to the ``mathtext.py`` module are in ``test_mathtext.py``. -Let's say you've added a new module named ``whizbang.py`` and you want -to add tests for it in ``matplotlib.tests.test_whizbang``. To add -this module to the list of default tests, append its name to -``default_test_modules`` in :file:`lib/matplotlib/__init__.py`. - Using Travis CI --------------- diff --git a/examples/README.txt b/examples/README.txt index c696f1955ba3..51ab6b08c2aa 100644 --- a/examples/README.txt +++ b/examples/README.txt @@ -35,7 +35,7 @@ directories found here: * tests - tests used by matplotlib developers to check functionality (These tests are still sometimes useful, but mostly developers should - use the nosetests which perform automatic image comparison.) + use the pytest tests which perform automatic image comparison.) * units - working with unit data an custom types in matplotlib From 25192ea0b9d95e2ceef6b77b1831a1d8f02e3d12 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 5 Feb 2017 20:17:22 -0500 Subject: [PATCH 08/14] Change setup to require pytest instead of nose. --- ci/conda_recipe/meta.yaml | 2 +- setupext.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ci/conda_recipe/meta.yaml b/ci/conda_recipe/meta.yaml index 45db6b908159..2b374683d805 100644 --- a/ci/conda_recipe/meta.yaml +++ b/ci/conda_recipe/meta.yaml @@ -30,7 +30,7 @@ requirements: - freetype 2.6* - msinttypes # [win] - cycler >=0.10 - - nose + - pytest >=3.0.0 - pyparsing - pytz # - py2cairo # [linux and py2k] diff --git a/setupext.py b/setupext.py index abb7f8be76ef..75a9f8361dc1 100644 --- a/setupext.py +++ b/setupext.py @@ -788,28 +788,28 @@ def get_namespace_packages(self): class Tests(OptionalPackage): name = "tests" - nose_min_version = '0.11.1' + pytest_min_version = '3.0.0' default_config = False def check(self): super(Tests, self).check() msgs = [] - msg_template = ('{package} is required to run the matplotlib test ' - 'suite. Please install it with pip or your preferred' - ' tool to run the test suite') + msg_template = ('{package} is required to run the Matplotlib test ' + 'suite. Please install it with pip or your preferred ' + 'tool to run the test suite') - bad_nose = msg_template.format( - package='nose %s or later' % self.nose_min_version + bad_pytest = msg_template.format( + package='pytest %s or later' % self.pytest_min_version ) try: - import nose - if is_min_version(nose.__version__, self.nose_min_version): - msgs += ['using nose version %s' % nose.__version__] + import pytest + if is_min_version(pytest.__version__, self.pytest_min_version): + msgs += ['using pytest version %s' % pytest.__version__] else: - msgs += [bad_nose] + msgs += [bad_pytest] except ImportError: - msgs += [bad_nose] + msgs += [bad_pytest] if PY3min: msgs += ['using unittest.mock'] From 50eaad6036dc8bd76d0a3939067942345d03467c Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 14 Feb 2017 02:34:55 -0500 Subject: [PATCH 09/14] Remove multiprocess nose stuff. --- lib/matplotlib/tests/__init__.py | 2 -- lib/mpl_toolkits/tests/__init__.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/lib/matplotlib/tests/__init__.py b/lib/matplotlib/tests/__init__.py index 6a382247e018..b31594ee23ef 100644 --- a/lib/matplotlib/tests/__init__.py +++ b/lib/matplotlib/tests/__init__.py @@ -8,8 +8,6 @@ from matplotlib.testing import setup -_multiprocess_can_split_ = True - # Check that the test directories exist if not os.path.exists(os.path.join( diff --git a/lib/mpl_toolkits/tests/__init__.py b/lib/mpl_toolkits/tests/__init__.py index 5bc2030963a3..57fe8bcdcdbf 100644 --- a/lib/mpl_toolkits/tests/__init__.py +++ b/lib/mpl_toolkits/tests/__init__.py @@ -4,9 +4,6 @@ import os -_multiprocess_can_split_ = True - - # Check that the test directories exist if not os.path.exists(os.path.join( os.path.dirname(__file__), 'baseline_images')): From 4529002fc1df75963751e9249b091330040920c0 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 16 Feb 2017 02:11:10 -0500 Subject: [PATCH 10/14] TST: Add 3.6 to tox.ini. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 36876f1ebef4..8e02e989dcb2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py34, py35 +envlist = py27, py34, py35, py36 [testenv] changedir = /tmp From 5bc91c7c394e29e9f4b87b1d24c4a331ef30ea28 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 16 Feb 2017 02:28:35 -0500 Subject: [PATCH 11/14] Move conftest.py so it works with uninstalled tests. --- lib/matplotlib/sphinxext/tests/conftest.py | 4 +- lib/matplotlib/testing/conftest.py | 55 ++++++++++++++++++++++ lib/matplotlib/tests/conftest.py | 54 +-------------------- lib/mpl_toolkits/tests/conftest.py | 4 +- 4 files changed, 61 insertions(+), 56 deletions(-) create mode 100644 lib/matplotlib/testing/conftest.py diff --git a/lib/matplotlib/sphinxext/tests/conftest.py b/lib/matplotlib/sphinxext/tests/conftest.py index 900743e86003..dcdc3612eecb 100644 --- a/lib/matplotlib/sphinxext/tests/conftest.py +++ b/lib/matplotlib/sphinxext/tests/conftest.py @@ -1,5 +1,5 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -from matplotlib.tests.conftest import (mpl_test_settings, - pytest_configure, pytest_unconfigure) +from matplotlib.testing.conftest import (mpl_test_settings, + pytest_configure, pytest_unconfigure) diff --git a/lib/matplotlib/testing/conftest.py b/lib/matplotlib/testing/conftest.py new file mode 100644 index 000000000000..918bafc873b9 --- /dev/null +++ b/lib/matplotlib/testing/conftest.py @@ -0,0 +1,55 @@ +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import pytest + +import matplotlib + + +def pytest_configure(config): + matplotlib.use('agg') + matplotlib._called_from_pytest = True + matplotlib._init_tests() + + +def pytest_unconfigure(config): + matplotlib._called_from_pytest = False + + +@pytest.fixture(autouse=True) +def mpl_test_settings(request): + from matplotlib.testing.decorators import _do_cleanup + + original_units_registry = matplotlib.units.registry.copy() + original_settings = matplotlib.rcParams.copy() + + backend = None + backend_marker = request.keywords.get('backend') + if backend_marker is not None: + assert len(backend_marker.args) == 1, \ + "Marker 'backend' must specify 1 backend." + backend = backend_marker.args[0] + prev_backend = matplotlib.get_backend() + + style = 'classic' + style_marker = request.keywords.get('style') + if style_marker is not None: + assert len(style_marker.args) == 1, \ + "Marker 'style' must specify 1 style." + style = style_marker.args[0] + + matplotlib.testing.setup() + if backend is not None: + # This import must come after setup() so it doesn't load the default + # backend prematurely. + import matplotlib.pyplot as plt + plt.switch_backend(backend) + matplotlib.style.use(style) + try: + yield + finally: + if backend is not None: + import matplotlib.pyplot as plt + plt.switch_backend(prev_backend) + _do_cleanup(original_units_registry, + original_settings) diff --git a/lib/matplotlib/tests/conftest.py b/lib/matplotlib/tests/conftest.py index 918bafc873b9..dcdc3612eecb 100644 --- a/lib/matplotlib/tests/conftest.py +++ b/lib/matplotlib/tests/conftest.py @@ -1,55 +1,5 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -import pytest - -import matplotlib - - -def pytest_configure(config): - matplotlib.use('agg') - matplotlib._called_from_pytest = True - matplotlib._init_tests() - - -def pytest_unconfigure(config): - matplotlib._called_from_pytest = False - - -@pytest.fixture(autouse=True) -def mpl_test_settings(request): - from matplotlib.testing.decorators import _do_cleanup - - original_units_registry = matplotlib.units.registry.copy() - original_settings = matplotlib.rcParams.copy() - - backend = None - backend_marker = request.keywords.get('backend') - if backend_marker is not None: - assert len(backend_marker.args) == 1, \ - "Marker 'backend' must specify 1 backend." - backend = backend_marker.args[0] - prev_backend = matplotlib.get_backend() - - style = 'classic' - style_marker = request.keywords.get('style') - if style_marker is not None: - assert len(style_marker.args) == 1, \ - "Marker 'style' must specify 1 style." - style = style_marker.args[0] - - matplotlib.testing.setup() - if backend is not None: - # This import must come after setup() so it doesn't load the default - # backend prematurely. - import matplotlib.pyplot as plt - plt.switch_backend(backend) - matplotlib.style.use(style) - try: - yield - finally: - if backend is not None: - import matplotlib.pyplot as plt - plt.switch_backend(prev_backend) - _do_cleanup(original_units_registry, - original_settings) +from matplotlib.testing.conftest import (mpl_test_settings, + pytest_configure, pytest_unconfigure) diff --git a/lib/mpl_toolkits/tests/conftest.py b/lib/mpl_toolkits/tests/conftest.py index 900743e86003..dcdc3612eecb 100644 --- a/lib/mpl_toolkits/tests/conftest.py +++ b/lib/mpl_toolkits/tests/conftest.py @@ -1,5 +1,5 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -from matplotlib.tests.conftest import (mpl_test_settings, - pytest_configure, pytest_unconfigure) +from matplotlib.testing.conftest import (mpl_test_settings, + pytest_configure, pytest_unconfigure) From a4427a1cf03194f8ad9c6794594e80513a85e58e Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 16 Feb 2017 02:43:11 -0500 Subject: [PATCH 12/14] Fix running of tests without installation. * Check for installed test data only if testing installed package. * Remove relative import out of tests. * Replace assert_str_equal with plain assert (pytest does diffing already). --- lib/matplotlib/__init__.py | 5 ++--- lib/matplotlib/tests/test_preprocess_data.py | 2 +- lib/matplotlib/tests/test_rcparams.py | 5 ++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/matplotlib/__init__.py b/lib/matplotlib/__init__.py index ad1e411ea6ab..ad37f11a5a25 100644 --- a/lib/matplotlib/__init__.py +++ b/lib/matplotlib/__init__.py @@ -1488,9 +1488,6 @@ def _init_tests(): else: faulthandler.enable() - if not os.path.isdir(os.path.join(os.path.dirname(__file__), 'tests')): - raise ImportError("matplotlib test data is not installed") - # The version of FreeType to install locally for running the # tests. This must match the value in `setupext.py` LOCAL_FREETYPE_VERSION = '2.6.1' @@ -1526,6 +1523,8 @@ def test(verbosity=None, coverage=False, switch_backend_warn=True, recursionlimit=0, **kwargs): """run the matplotlib test suite""" _init_tests() + if not os.path.isdir(os.path.join(os.path.dirname(__file__), 'tests')): + raise ImportError("matplotlib test data is not installed") old_backend = get_backend() old_recursionlimit = sys.getrecursionlimit() diff --git a/lib/matplotlib/tests/test_preprocess_data.py b/lib/matplotlib/tests/test_preprocess_data.py index c3c968f80fba..633ce5983914 100644 --- a/lib/matplotlib/tests/test_preprocess_data.py +++ b/lib/matplotlib/tests/test_preprocess_data.py @@ -4,7 +4,7 @@ import pytest -from .. import _preprocess_data +from matplotlib import _preprocess_data # Notes on testing the plotting functions itself diff --git a/lib/matplotlib/tests/test_rcparams.py b/lib/matplotlib/tests/test_rcparams.py index b376170ac31c..0b0555852cc9 100644 --- a/lib/matplotlib/tests/test_rcparams.py +++ b/lib/matplotlib/tests/test_rcparams.py @@ -17,7 +17,6 @@ import mock import matplotlib as mpl import matplotlib.pyplot as plt -from matplotlib.tests import assert_str_equal import matplotlib.colors as mcolors from itertools import chain import numpy as np @@ -93,7 +92,7 @@ def test_RcParams_class(): u'font.size': 12.0, u'font.weight': u'normal'})""".lstrip() - assert_str_equal(expected_repr, repr(rc)) + assert expected_repr == repr(rc) if six.PY3: expected_str = """ @@ -108,7 +107,7 @@ def test_RcParams_class(): font.size: 12.0 font.weight: normal""".lstrip() - assert_str_equal(expected_str, str(rc)) + assert expected_str == str(rc) # test the find_all functionality assert ['font.cursive', 'font.size'] == sorted(rc.find_all('i[vz]')) From 6a0c3ee3dbd378add5172f84b16781361758628a Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 17 Feb 2017 19:00:32 -0500 Subject: [PATCH 13/14] Update pytest documentation based on review. * Prefer py.test over tests.py * Remove redundant hint * Add link to further pytest documentation * Tweak some sentence structure --- INSTALL | 17 +++++++------- README.rst | 7 +++--- doc/devel/testing.rst | 52 +++++++++++++++++++------------------------ 3 files changed, 35 insertions(+), 41 deletions(-) diff --git a/INSTALL b/INSTALL index fc91fad8dd61..6a5d0e5ae5d5 100644 --- a/INSTALL +++ b/INSTALL @@ -106,14 +106,15 @@ or example code. If you want to try the many demos that come in the matplotlib source distribution, download the :file:`*.tar.gz` file and look in the :file:`examples` subdirectory. -To run the test suite, copy the :file:`lib\\matplotlib\\tests` and -:file:`lib\\mpl_toolkits\\tests` directories from the source -distribution to :file:`sys.prefix\\Lib\\site-packages\\matplotlib` and -:file:`sys.prefix\\Lib\\site-packages\\mpl_toolkits` respectively, and -install `pytest `_, `mock -`_, Pillow, MiKTeX, GhostScript, -ffmpeg, avconv, mencoder, ImageMagick, and `Inkscape -`_. +To run the test suite: + + * extract the :file:`lib\\matplotlib\\tests` or + :file:`lib\\mpl_toolkits\\tests` directories from the source distribution; + * install test dependencies: `pytest `_, + `mock `_, Pillow, MiKTeX, GhostScript, + ffmpeg, avconv, mencoder, ImageMagick, and `Inkscape + `_; + * run ``py.test path\\to\\tests\\directory``. diff --git a/README.rst b/README.rst index 66827cac69b3..9574f708d51b 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ Testing After installation, you can launch the test suite:: - python tests.py + py.test Or from the Python interpreter:: @@ -28,9 +28,8 @@ Or from the Python interpreter:: matplotlib.test() Consider reading http://matplotlib.org/devel/coding_guide.html#testing for -more information. Note that the test suite requires pytest and on Python 2.7 -mock which are not installed by default. Please install with pip or your -package manager of choice. +more information. Note that the test suite requires pytest and, on Python 2.7, +mock. Please install with pip or your package manager of choice. Contact ======= diff --git a/doc/devel/testing.rst b/doc/devel/testing.rst index b513cb865c47..9d4e29d35c59 100644 --- a/doc/devel/testing.rst +++ b/doc/devel/testing.rst @@ -4,11 +4,9 @@ Developer's tips for testing ============================ -Matplotlib has a testing infrastructure based on pytest_, making it easy to -write new tests. The tests are in :mod:`matplotlib.tests`, and customizations -to the pytest testing infrastructure are in :mod:`matplotlib.tests.conftest` -and :mod:`matplotlib.testing`. (There is other old testing cruft around, please -ignore it while we consolidate our testing to these locations.) +Matplotlib's testing infrastructure depends on pytest_. The tests are in +:file:`lib/matplotlib/tests`, and customizations to the pytest testing +infrastructure are in :mod:`matplotlib.testing`. .. _pytest: http://doc.pytest.org/en/latest/ .. _mock: https://docs.python.org/dev/library/unittest.mock.html> @@ -16,6 +14,8 @@ ignore it while we consolidate our testing to these locations.) .. _Inkscape: https://inkscape.org .. _pytest-cov: https://pytest-cov.readthedocs.io/en/latest/ .. _pytest-pep8: https://pypi.python.org/pypi/pytest-pep8 +.. _pytest-xdist: https://pypi.python.org/pypi/pytest-xdist +.. _pytest-timeout: https://pypi.python.org/pypi/pytest-timeout Requirements ------------ @@ -31,6 +31,8 @@ Optionally you can install: - pytest-cov_ to collect coverage information - pytest-pep8_ to test coding standards + - pytest-timeout_ to limit runtime in case of stuck tests + - pytest-xdist_ to run tests in parallel Building matplotlib for image comparison tests @@ -69,11 +71,11 @@ commands, such as: ======================== =========== ``--pep8`` Perform pep8 checks (requires pytest-pep8_) -``--no-network`` Disable tests that require network access +``-m "not network"`` Disable tests that require network access ======================== =========== Additional arguments are passed on to pytest. See the pytest documentation for -supported arguments. Some of the more important ones are given here: +`supported arguments`_. Some of the more important ones are given here: ============================= =========== ``--verbose`` Be more verbose @@ -84,26 +86,26 @@ supported arguments. Some of the more important ones are given here: ``--capture=no`` or ``-s`` Do not capture stdout ============================= =========== -To run a single test from the command line, you can provide a dot-separated -path to the module, optionally followed by the function separated by two -colons, e.g., (this is assuming the test is installed):: +To run a single test from the command line, you can provide a file path, +optionally followed by the function separated by two colons, e.g., (tests do +not need to be installed, but Matplotlib should be):: - python tests.py matplotlib.tests.test_simplification::test_clipping + py.test lib/matplotlib/tests/test_simplification.py::test_clipping -or by passing a file path, optionally followed by the function separated by two -colons, e.g., (tests do not need to be installed, but Matplotlib should be):: +or, if tests are installed, a dot-separated path to the module, optionally +followed by the function separated by two colons, such as:: - python tests.py lib/matplotlib/tests/test_simplification.py::test_clipping + py.test --pyargs matplotlib.tests.test_simplification::test_clipping If you want to run the full test suite, but want to save wall time try running the tests in parallel:: - python tests.py --capture=no --verbose -n 5 + py.test --verbose -n 5 Depending on your version of Python and pytest-xdist, you may need to set ``PYTHONHASHSEED`` to a fixed value when running in parallel:: - PYTHONHASHSEED=0 python tests.py --capture=no --verbose -n 5 + PYTHONHASHSEED=0 py.test --verbose -n 5 An alternative implementation that does not look at command line arguments and works from within Python is to run the tests from the Matplotlib library @@ -112,16 +114,8 @@ function :func:`matplotlib.test`:: import matplotlib matplotlib.test() -.. hint:: - - To run the tests you need to install pytest and mock if using python 2.7:: - - pip install pytest - pip install mock - -.. _pytest-xdist: https://pypi.python.org/pypi/pytest-xdist -.. _pytest-timeout: https://pypi.python.org/pypi/pytest-timeout +.. _supported arguments: http://doc.pytest.org/en/latest/usage.html Writing a simple test @@ -140,10 +134,10 @@ Pytest determines which functions are tests by searching for files whose names begin with ``"test_"`` and then within those files for functions beginning with ``"test"`` or classes beginning with ``"Test"``. -Tests that have side effects that need to be cleaned up, such as created -figures using the pyplot interface or modified rc params, will be automatically -reset by the pytest fixture -:func:`~matplotlib.tests.conftest.mpl_test_settings`. +Some tests have internal side effects that need to be cleaned up after their +execution (such as created figures or modified rc params). The pytest fixture +:func:`~matplotlib.testing.conftest.mpl_test_settings` will automatically clean +these up; there is no need to do anything further. Writing an image comparison test From 29c130839ce43165e83412f205ab54dd0d7b1f2e Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 17 Feb 2017 19:18:38 -0500 Subject: [PATCH 14/14] TST: Remove extra Travis build. The only difference between the two 3.5 builds is one uses tests.py and the other uses py.test. But the 2.7, 3.4, and macOS builds all use tests.py, to this is a waste of resources. --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d1b0008ec22b..a2336148735e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,8 +56,6 @@ matrix: env: MOCK=mock NUMPY=numpy==1.7.1 - python: 3.4 env: PYTHON_ARGS=-OO - - python: 3.5 - env: PANDAS=pandas DELETE_FONT_CACHE=1 - python: 3.5 env: BUILD_DOCS=true - python: 3.5