diff --git a/.travis.yml b/.travis.yml index 2bd4d5ba9ebb..7d79f7e194f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,6 +45,7 @@ env: - NPROC=2 - INSTALL_PEP8= - RUN_PEP8= + - NOSE= - PYTEST_ARGS="-ra --maxfail=1 --timeout=300 --durations=25 --cov-report= --cov=lib -n $NPROC" - PYTHON_ARGS= - DELETE_FONT_CACHE= @@ -52,7 +53,7 @@ env: matrix: include: - python: 2.7 - env: MOCK=mock NUMPY=numpy==1.7.1 PANDAS=pandas + env: MOCK=mock NUMPY=numpy==1.7.1 PANDAS=pandas NOSE=nose - python: 2.7 env: BUILD_DOCS=true - python: 3.4 @@ -114,7 +115,7 @@ install: pip install --upgrade setuptools - | # Install dependencies from pypi - pip install $PRE python-dateutil $NUMPY pyparsing!=2.1.6 $PANDAS cycler codecov coverage $MOCK + pip install $PRE python-dateutil $NUMPY pyparsing!=2.1.6 $PANDAS cycler codecov coverage $MOCK $NOSE pip install $PRE -r doc-requirements.txt # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124 diff --git a/lib/matplotlib/sphinxext/tests/conftest.py b/lib/matplotlib/sphinxext/tests/conftest.py index dcdc3612eecb..2971a4314146 100644 --- a/lib/matplotlib/sphinxext/tests/conftest.py +++ b/lib/matplotlib/sphinxext/tests/conftest.py @@ -2,4 +2,5 @@ unicode_literals) from matplotlib.testing.conftest import (mpl_test_settings, + mpl_image_comparison_parameters, pytest_configure, pytest_unconfigure) diff --git a/lib/matplotlib/testing/compare.py b/lib/matplotlib/testing/compare.py index 0483e875eb17..c3d649e38069 100644 --- a/lib/matplotlib/testing/compare.py +++ b/lib/matplotlib/testing/compare.py @@ -366,7 +366,7 @@ def calculate_rms(expectedImage, actualImage): "Calculate the per-pixel errors, then compute the root mean square error." if expectedImage.shape != actualImage.shape: raise ImageComparisonFailure( - "image sizes do not match expected size: {0} " + "Image sizes do not match expected size: {0} " "actual size {1}".format(expectedImage.shape, actualImage.shape)) num_values = expectedImage.size abs_diff_image = abs(expectedImage - actualImage) @@ -470,7 +470,10 @@ def save_diff_image(expected, actual, output): actual, actualImage, expected, expectedImage) expectedImage = np.array(expectedImage).astype(float) actualImage = np.array(actualImage).astype(float) - assert expectedImage.shape == actualImage.shape + if expectedImage.shape != actualImage.shape: + raise ImageComparisonFailure( + "Image sizes do not match expected size: {0} " + "actual size {1}".format(expectedImage.shape, actualImage.shape)) absDiffImage = np.abs(expectedImage - actualImage) # expand differences in luminance domain diff --git a/lib/matplotlib/testing/conftest.py b/lib/matplotlib/testing/conftest.py index 918bafc873b9..ac79e63ccd85 100644 --- a/lib/matplotlib/testing/conftest.py +++ b/lib/matplotlib/testing/conftest.py @@ -31,7 +31,7 @@ def mpl_test_settings(request): backend = backend_marker.args[0] prev_backend = matplotlib.get_backend() - style = 'classic' + style = '_classic_test' # Default of cleanup and image_comparison too. style_marker = request.keywords.get('style') if style_marker is not None: assert len(style_marker.args) == 1, \ @@ -53,3 +53,27 @@ def mpl_test_settings(request): plt.switch_backend(prev_backend) _do_cleanup(original_units_registry, original_settings) + + +@pytest.fixture +def mpl_image_comparison_parameters(request, extension): + # This fixture is applied automatically by the image_comparison decorator. + # + # The sole purpose of this fixture is to provide an indirect method of + # obtaining parameters *without* modifying the decorated function + # signature. In this way, the function signature can stay the same and + # pytest won't get confused. + # We annotate the decorated function with any parameters captured by this + # fixture so that they can be used by the wrapper in image_comparison. + baseline_images = request.keywords['baseline_images'].args[0] + if baseline_images is None: + # Allow baseline image list to be produced on the fly based on current + # parametrization. + baseline_images = request.getfixturevalue('baseline_images') + + func = request.function + func.__wrapped__.parameters = (baseline_images, extension) + try: + yield + finally: + delattr(func.__wrapped__, 'parameters') diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py index 0c07a62c13de..3a0dd4e0f9a8 100644 --- a/lib/matplotlib/testing/decorators.py +++ b/lib/matplotlib/testing/decorators.py @@ -150,6 +150,7 @@ def wrapped_callable(*args, **kwargs): return make_cleanup else: result = make_cleanup(style) + # Default of mpl_test_settings fixture and image_comparison too. style = '_classic_test' return result @@ -232,42 +233,24 @@ def _mark_xfail_if_format_is_uncomparable(extension): return extension -class ImageComparisonDecorator(CleanupTest): - def __init__(self, baseline_images, extensions, tol, - freetype_version, remove_text, savefig_kwargs, style): +class _ImageComparisonBase(object): + """ + Image comparison base class + + This class provides *just* the comparison-related functionality and avoids + any code that would be specific to any testing framework. + """ + def __init__(self, tol, remove_text, savefig_kwargs): self.func = self.baseline_dir = self.result_dir = None - self.baseline_images = baseline_images - self.extensions = extensions self.tol = tol - self.freetype_version = freetype_version self.remove_text = remove_text self.savefig_kwargs = savefig_kwargs - self.style = style def delayed_init(self, func): assert self.func is None, "it looks like same decorator used twice" self.func = func self.baseline_dir, self.result_dir = _image_directories(func) - def setup(self): - func = self.func - plt.close('all') - self.setup_class() - try: - matplotlib.style.use(self.style) - matplotlib.testing.set_font_settings_for_testing() - func() - assert len(plt.get_fignums()) == len(self.baseline_images), ( - "Test generated {} images but there are {} baseline images" - .format(len(plt.get_fignums()), len(self.baseline_images))) - except: - # Restore original settings before raising errors during the update. - self.teardown_class() - raise - - def teardown(self): - self.teardown_class() - def copy_baseline(self, baseline, extension): baseline_path = os.path.join(self.baseline_dir, baseline) orig_expected_fname = baseline_path + '.' + extension @@ -303,6 +286,50 @@ def compare(self, idx, baseline, extension): expected_fname = self.copy_baseline(baseline, extension) _raise_on_image_difference(expected_fname, actual_fname, self.tol) + +class ImageComparisonTest(CleanupTest, _ImageComparisonBase): + """ + Nose-based image comparison class + + This class generates tests for a nose-based testing framework. Ideally, + this class would not be public, and the only publically visible API would + be the :func:`image_comparison` decorator. Unfortunately, there are + existing downstream users of this class (e.g., pytest-mpl) so it cannot yet + be removed. + """ + def __init__(self, baseline_images, extensions, tol, + freetype_version, remove_text, savefig_kwargs, style): + _ImageComparisonBase.__init__(self, tol, remove_text, savefig_kwargs) + self.baseline_images = baseline_images + self.extensions = extensions + self.freetype_version = freetype_version + self.style = style + + def setup(self): + func = self.func + plt.close('all') + self.setup_class() + try: + matplotlib.style.use(self.style) + matplotlib.testing.set_font_settings_for_testing() + func() + assert len(plt.get_fignums()) == len(self.baseline_images), ( + "Test generated {} images but there are {} baseline images" + .format(len(plt.get_fignums()), len(self.baseline_images))) + except: + # Restore original settings before raising errors. + self.teardown_class() + raise + + def teardown(self): + self.teardown_class() + + @staticmethod + @cbook.deprecated('2.1', + alternative='remove_ticks_and_titles') + def remove_text(figure): + remove_ticks_and_titles(figure) + def nose_runner(self): func = self.compare func = _checked_on_freetype_version(self.freetype_version)(func) @@ -312,57 +339,74 @@ def nose_runner(self): for extension in self.extensions: yield funcs[extension], idx, baseline, extension - def pytest_runner(self): - from pytest import mark + def __call__(self, func): + self.delayed_init(func) + import nose.tools - extensions = map(_mark_xfail_if_format_is_uncomparable, - self.extensions) + @nose.tools.with_setup(self.setup, self.teardown) + def runner_wrapper(): + for case in self.nose_runner(): + yield case - if len(set(self.baseline_images)) == len(self.baseline_images): - @mark.parametrize("extension", extensions) - @mark.parametrize("idx,baseline", enumerate(self.baseline_images)) - @_checked_on_freetype_version(self.freetype_version) - def wrapper(idx, baseline, extension): - __tracebackhide__ = True - self.compare(idx, baseline, extension) - else: - # Some baseline images are repeated, so run this in serial. - @mark.parametrize("extension", extensions) - @_checked_on_freetype_version(self.freetype_version) - def wrapper(extension): - __tracebackhide__ = True - for idx, baseline in enumerate(self.baseline_images): - self.compare(idx, baseline, extension) + return _copy_metadata(func, runner_wrapper) - # sadly we cannot use fixture here because of visibility problems - # and for for obvious reason avoid `_nose.tools.with_setup` - wrapper.setup, wrapper.teardown = self.setup, self.teardown +def _pytest_image_comparison(baseline_images, extensions, tol, + freetype_version, remove_text, savefig_kwargs, + style): + """ + Decorate function with image comparison for pytest. - return wrapper + This function creates a decorator that wraps a figure-generating function + with image comparison code. Pytest can become confused if we change the + signature of the function, so we indirectly pass anything we need via the + `mpl_image_comparison_parameters` fixture and extra markers. + """ + import pytest + + extensions = map(_mark_xfail_if_format_is_uncomparable, extensions) + + def decorator(func): + # Parameter indirection; see docstring above and comment below. + @pytest.mark.usefixtures('mpl_image_comparison_parameters') + @pytest.mark.parametrize('extension', extensions) + @pytest.mark.baseline_images(baseline_images) + # END Parameter indirection. + @pytest.mark.style(style) + @_checked_on_freetype_version(freetype_version) + @functools.wraps(func) + def wrapper(*args, **kwargs): + __tracebackhide__ = True + img = _ImageComparisonBase(tol=tol, remove_text=remove_text, + savefig_kwargs=savefig_kwargs) + img.delayed_init(func) + matplotlib.testing.set_font_settings_for_testing() + func(*args, **kwargs) - def __call__(self, func): - self.delayed_init(func) - if is_called_from_pytest(): - return _copy_metadata(func, self.pytest_runner()) - else: - import nose.tools + # Parameter indirection: + # This is hacked on via the mpl_image_comparison_parameters fixture + # so that we don't need to modify the function's real signature for + # any parametrization. Modifying the signature is very very tricky + # and likely to confuse pytest. + baseline_images, extension = func.parameters - @nose.tools.with_setup(self.setup, self.teardown) - def runner_wrapper(): - try: - for case in self.nose_runner(): - yield case - except GeneratorExit: - # nose bug... - self.teardown() + assert len(plt.get_fignums()) == len(baseline_images), ( + "Test generated {} images but there are {} baseline images" + .format(len(plt.get_fignums()), len(baseline_images))) + for idx, baseline in enumerate(baseline_images): + img.compare(idx, baseline, extension) - return _copy_metadata(func, runner_wrapper) + wrapper.__wrapped__ = func # For Python 2.7. + return _copy_metadata(func, wrapper) + return decorator -def image_comparison(baseline_images=None, extensions=None, tol=0, + +def image_comparison(baseline_images, extensions=None, tol=0, freetype_version=None, remove_text=False, - savefig_kwarg=None, style='_classic_test'): + savefig_kwarg=None, + # Default of mpl_test_settings fixture and cleanup too. + style='_classic_test'): """ Compare images generated by the test with those specified in *baseline_images*, which must correspond else an @@ -370,10 +414,14 @@ def image_comparison(baseline_images=None, extensions=None, tol=0, Arguments --------- - baseline_images : list + baseline_images : list or None A list of strings specifying the names of the images generated by calls to :meth:`matplotlib.figure.savefig`. + If *None*, the test function must use the ``baseline_images`` fixture, + either as a parameter or with pytest.mark.usefixtures. This value is + only allowed when using pytest. + extensions : [ None | list ] If None, defaults to all supported extensions. @@ -400,9 +448,6 @@ def image_comparison(baseline_images=None, extensions=None, tol=0, '_classic_test' style. """ - if baseline_images is None: - raise ValueError('baseline_images must be specified') - if extensions is None: # default extensions to test extensions = ['png', 'pdf', 'svg'] @@ -411,10 +456,19 @@ def image_comparison(baseline_images=None, extensions=None, tol=0, #default no kwargs to savefig savefig_kwarg = dict() - return ImageComparisonDecorator( - baseline_images=baseline_images, extensions=extensions, tol=tol, - freetype_version=freetype_version, remove_text=remove_text, - savefig_kwargs=savefig_kwarg, style=style) + if is_called_from_pytest(): + return _pytest_image_comparison( + baseline_images=baseline_images, extensions=extensions, tol=tol, + freetype_version=freetype_version, remove_text=remove_text, + savefig_kwargs=savefig_kwarg, style=style) + else: + if baseline_images is None: + raise ValueError('baseline_images must be specified') + + return ImageComparisonTest( + baseline_images=baseline_images, extensions=extensions, tol=tol, + freetype_version=freetype_version, remove_text=remove_text, + savefig_kwargs=savefig_kwarg, style=style) def _image_directories(func): diff --git a/lib/matplotlib/tests/baseline_images/test_compare_images/simple.pdf b/lib/matplotlib/tests/baseline_images/test_compare_images/simple.pdf new file mode 100644 index 000000000000..43c04c8531a9 Binary files /dev/null and b/lib/matplotlib/tests/baseline_images/test_compare_images/simple.pdf differ diff --git a/lib/matplotlib/tests/baseline_images/test_compare_images/simple.png b/lib/matplotlib/tests/baseline_images/test_compare_images/simple.png new file mode 100644 index 000000000000..070a0a9917df Binary files /dev/null and b/lib/matplotlib/tests/baseline_images/test_compare_images/simple.png differ diff --git a/lib/matplotlib/tests/baseline_images/test_compare_images/simple.svg b/lib/matplotlib/tests/baseline_images/test_compare_images/simple.svg new file mode 100644 index 000000000000..a092bb50b9b2 --- /dev/null +++ b/lib/matplotlib/tests/baseline_images/test_compare_images/simple.svg @@ -0,0 +1,206 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/matplotlib/tests/conftest.py b/lib/matplotlib/tests/conftest.py index dcdc3612eecb..2971a4314146 100644 --- a/lib/matplotlib/tests/conftest.py +++ b/lib/matplotlib/tests/conftest.py @@ -2,4 +2,5 @@ unicode_literals) from matplotlib.testing.conftest import (mpl_test_settings, + mpl_image_comparison_parameters, pytest_configure, pytest_unconfigure) diff --git a/lib/matplotlib/tests/test_compare_images.py b/lib/matplotlib/tests/test_compare_images.py index 10896dd90212..883d9b7a9165 100644 --- a/lib/matplotlib/tests/test_compare_images.py +++ b/lib/matplotlib/tests/test_compare_images.py @@ -1,13 +1,19 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import six + +import io import os import shutil +import warnings from numpy.testing import assert_equal, assert_almost_equal +import pytest from matplotlib.testing.compare import compare_images -from matplotlib.testing.decorators import _image_directories +from matplotlib.testing.decorators import _image_directories, image_comparison +from matplotlib.testing.exceptions import ImageComparisonFailure baseline_dir, result_dir = _image_directories(lambda: 'dummy func') @@ -97,3 +103,121 @@ def test_image_compare_shade_difference(): # Now test the reverse comparison. image_comparison_expect_rms(im2, im1, tol=0, expect_rms=1.0) + + +# +# The following tests are used by test_nose_image_comparison to ensure that the +# image_comparison decorator continues to work with nose. They should not be +# prefixed by test_ so they don't run with pytest. +# + +def nosetest_empty(): + pass + + +def nosetest_simple_figure(): + import matplotlib.pyplot as plt + fig, ax = plt.subplots(figsize=(6.4, 4), dpi=100) + ax.plot([1, 2, 3], [3, 4, 5]) + return fig + + +def nosetest_manual_text_removal(): + from matplotlib.testing.decorators import ImageComparisonTest + + fig = nosetest_simple_figure() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + # Make sure this removes text like it should. + ImageComparisonTest.remove_text(fig) + + assert len(w) == 1 + assert 'remove_text function was deprecated in version 2.1.' in str(w[0]) + + +@pytest.mark.parametrize( + 'func, kwargs, errors, failures, dots', + [ + (nosetest_empty, {'baseline_images': []}, [], [], ''), + (nosetest_empty, {'baseline_images': ['foo']}, + [(AssertionError, + 'Test generated 0 images but there are 1 baseline images')], + [], + 'E'), + (nosetest_simple_figure, + {'baseline_images': ['basn3p02'], 'extensions': ['png'], + 'remove_text': True}, + [], + [(ImageComparisonFailure, 'Image sizes do not match expected size:')], + 'F'), + (nosetest_simple_figure, + {'baseline_images': ['simple']}, + [], + [(ImageComparisonFailure, 'images not close')] * 3, + 'FFF'), + (nosetest_simple_figure, + {'baseline_images': ['simple'], 'remove_text': True}, + [], + [], + '...'), + (nosetest_manual_text_removal, + {'baseline_images': ['simple']}, + [], + [], + '...'), + ], + ids=[ + 'empty', + 'extra baselines', + 'incorrect shape', + 'failing figure', + 'passing figure', + 'manual text removal', + ]) +def test_nose_image_comparison(func, kwargs, errors, failures, dots, + monkeypatch): + nose = pytest.importorskip('nose') + monkeypatch.setattr('matplotlib._called_from_pytest', False) + + class TestResultVerifier(nose.result.TextTestResult): + def __init__(self, *args, **kwargs): + super(TestResultVerifier, self).__init__(*args, **kwargs) + self.error_count = 0 + self.failure_count = 0 + + def addError(self, test, err): + super(TestResultVerifier, self).addError(test, err) + + if self.error_count < len(errors): + assert err[0] is errors[self.error_count][0] + assert errors[self.error_count][1] in str(err[1]) + else: + raise err[1] + self.error_count += 1 + + def addFailure(self, test, err): + super(TestResultVerifier, self).addFailure(test, err) + + assert self.failure_count < len(failures), err[1] + assert err[0] is failures[self.failure_count][0] + assert failures[self.failure_count][1] in str(err[1]) + self.failure_count += 1 + + func = image_comparison(**kwargs)(func) + loader = nose.loader.TestLoader() + suite = loader.loadTestsFromGenerator( + func, + 'matplotlib.tests.test_compare_images') + if six.PY2: + output = io.BytesIO() + else: + output = io.StringIO() + result = TestResultVerifier(stream=output, descriptions=True, verbosity=1) + with warnings.catch_warnings(): + # Nose uses deprecated stuff; we don't care about it. + warnings.simplefilter('ignore', DeprecationWarning) + suite.run(result=result) + + assert output.getvalue() == dots + assert result.error_count == len(errors) + assert result.failure_count == len(failures) diff --git a/lib/matplotlib/tests/test_mathtext.py b/lib/matplotlib/tests/test_mathtext.py index de089f4e1c4f..544d3ef89201 100644 --- a/lib/matplotlib/tests/test_mathtext.py +++ b/lib/matplotlib/tests/test_mathtext.py @@ -14,6 +14,7 @@ import matplotlib.pyplot as plt from matplotlib import mathtext + math_tests = [ r'$a+b+\dot s+\dot{s}+\ldots$', r'$x \doteq y$', @@ -160,36 +161,39 @@ for set in chars: font_tests.append(wrapper % set) -def make_set(basename, fontset, tests, extensions=None): - def make_test(filename, test): - @image_comparison(baseline_images=[filename], extensions=extensions) - def single_test(): - matplotlib.rcParams['mathtext.fontset'] = fontset - fig = plt.figure(figsize=(5.25, 0.75)) - fig.text(0.5, 0.5, test, horizontalalignment='center', verticalalignment='center') - func = single_test - func.__name__ = str("test_" + filename) - return func - # We inject test functions into the global namespace, rather than - # using a generator, so that individual tests can be run more - # easily from the commandline and so each test will have its own - # result. - for i, test in enumerate(tests): - filename = '%s_%s_%02d' % (basename, fontset, i) - globals()['test_%s' % filename] = make_test(filename, test) +@pytest.fixture +def baseline_images(request, fontset, index): + return ['%s_%s_%02d' % (request.param, fontset, index)] + + +@pytest.mark.parametrize('index, test', enumerate(math_tests), + ids=[str(index) for index in range(len(math_tests))]) +@pytest.mark.parametrize('fontset', + ['cm', 'stix', 'stixsans', 'dejavusans', + 'dejavuserif']) +@pytest.mark.parametrize('baseline_images', ['mathtext'], indirect=True) +@image_comparison(baseline_images=None) +def test_mathtext_rendering(baseline_images, fontset, index, test): + matplotlib.rcParams['mathtext.fontset'] = fontset + fig = plt.figure(figsize=(5.25, 0.75)) + fig.text(0.5, 0.5, test, + horizontalalignment='center', verticalalignment='center') + -make_set('mathtext', 'cm', math_tests) -make_set('mathtext', 'stix', math_tests) -make_set('mathtext', 'stixsans', math_tests) -make_set('mathtext', 'dejavusans', math_tests) -make_set('mathtext', 'dejavuserif', math_tests) +@pytest.mark.parametrize('index, test', enumerate(font_tests), + ids=[str(index) for index in range(len(font_tests))]) +@pytest.mark.parametrize('fontset', + ['cm', 'stix', 'stixsans', 'dejavusans', + 'dejavuserif']) +@pytest.mark.parametrize('baseline_images', ['mathfont'], indirect=True) +@image_comparison(baseline_images=None, extensions=['png']) +def test_mathfont_rendering(baseline_images, fontset, index, test): + matplotlib.rcParams['mathtext.fontset'] = fontset + fig = plt.figure(figsize=(5.25, 0.75)) + fig.text(0.5, 0.5, test, + horizontalalignment='center', verticalalignment='center') -make_set('mathfont', 'cm', font_tests, ['png']) -make_set('mathfont', 'stix', font_tests, ['png']) -make_set('mathfont', 'stixsans', font_tests, ['png']) -make_set('mathfont', 'dejavusans', font_tests, ['png']) -make_set('mathfont', 'dejavuserif', font_tests, ['png']) def test_fontinfo(): import matplotlib.font_manager as font_manager diff --git a/lib/mpl_toolkits/tests/conftest.py b/lib/mpl_toolkits/tests/conftest.py index dcdc3612eecb..2971a4314146 100644 --- a/lib/mpl_toolkits/tests/conftest.py +++ b/lib/mpl_toolkits/tests/conftest.py @@ -2,4 +2,5 @@ unicode_literals) from matplotlib.testing.conftest import (mpl_test_settings, + mpl_image_comparison_parameters, pytest_configure, pytest_unconfigure)