From 14b5d1758d8c8270b89b38137c436279d5250c99 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 26 Mar 2017 02:41:00 -0400 Subject: [PATCH 01/10] TST: Use same default style in the pytest fixture. --- lib/matplotlib/testing/conftest.py | 2 +- lib/matplotlib/testing/decorators.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/matplotlib/testing/conftest.py b/lib/matplotlib/testing/conftest.py index 918bafc873b9..ab8ef1f57a29 100644 --- a/lib/matplotlib/testing/conftest.py +++ b/lib/matplotlib/testing/conftest.py @@ -31,7 +31,7 @@ def mpl_test_settings(request): backend = backend_marker.args[0] prev_backend = matplotlib.get_backend() - style = 'classic' + style = '_classic_test' # Default of cleanup and image_comparison too. style_marker = request.keywords.get('style') if style_marker is not None: assert len(style_marker.args) == 1, \ diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py index 0c07a62c13de..4160c3628a19 100644 --- a/lib/matplotlib/testing/decorators.py +++ b/lib/matplotlib/testing/decorators.py @@ -150,6 +150,7 @@ def wrapped_callable(*args, **kwargs): return make_cleanup else: result = make_cleanup(style) + # Default of mpl_test_settings fixture and image_comparison too. style = '_classic_test' return result @@ -362,7 +363,9 @@ def runner_wrapper(): def image_comparison(baseline_images=None, extensions=None, tol=0, freetype_version=None, remove_text=False, - savefig_kwarg=None, style='_classic_test'): + savefig_kwarg=None, + # Default of mpl_test_settings fixture and cleanup too. + style='_classic_test'): """ Compare images generated by the test with those specified in *baseline_images*, which must correspond else an From abc944549a4ae0b8f985d222655af2d22eb4941f Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 26 Mar 2017 02:52:45 -0400 Subject: [PATCH 02/10] TST: Make image_comparison more pytest-y. Instead of a heavy do-it-all class, split ImageComparisonDecorator into a smaller class that does just the comparison stuff and one that does only nose. For pytest, use a wrapper function that's decorated only by pytest decorators, and don't try to modify the function signature. By using a separate fixture, we can indirectly return the parameterized arguments instead. This stops pytest from getting confused about what takes what argument. The biggest benefit is that test code is now run as the *test*, whereas previously, it was run as the *setup* causing it to have all sorts of semantic irregularities. --- lib/matplotlib/sphinxext/tests/conftest.py | 1 + lib/matplotlib/testing/conftest.py | 16 +++ lib/matplotlib/testing/decorators.py | 154 ++++++++++++--------- lib/matplotlib/tests/conftest.py | 1 + lib/mpl_toolkits/tests/conftest.py | 1 + 5 files changed, 105 insertions(+), 68 deletions(-) diff --git a/lib/matplotlib/sphinxext/tests/conftest.py b/lib/matplotlib/sphinxext/tests/conftest.py index dcdc3612eecb..2971a4314146 100644 --- a/lib/matplotlib/sphinxext/tests/conftest.py +++ b/lib/matplotlib/sphinxext/tests/conftest.py @@ -2,4 +2,5 @@ unicode_literals) from matplotlib.testing.conftest import (mpl_test_settings, + mpl_image_comparison_parameters, pytest_configure, pytest_unconfigure) diff --git a/lib/matplotlib/testing/conftest.py b/lib/matplotlib/testing/conftest.py index ab8ef1f57a29..aab3318a8c65 100644 --- a/lib/matplotlib/testing/conftest.py +++ b/lib/matplotlib/testing/conftest.py @@ -53,3 +53,19 @@ def mpl_test_settings(request): plt.switch_backend(prev_backend) _do_cleanup(original_units_registry, original_settings) + + +@pytest.fixture +def mpl_image_comparison_parameters(request, extension): + # This fixture is applied automatically by the image_comparison decorator. + # + # The sole purpose of this fixture is to provide an indirect method of + # obtaining parameters *without* modifying the decorated function + # signature. In this way, the function signature can stay the same and + # pytest won't get confused. + # We annotate the decorated function with any parameters captured by this + # fixture so that they can be used by the wrapper in image_comparison. + func = request.function + func.__wrapped__.parameters = (extension, ) + yield + delattr(func.__wrapped__, 'parameters') diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py index 4160c3628a19..29908d6af6db 100644 --- a/lib/matplotlib/testing/decorators.py +++ b/lib/matplotlib/testing/decorators.py @@ -233,42 +233,18 @@ def _mark_xfail_if_format_is_uncomparable(extension): return extension -class ImageComparisonDecorator(CleanupTest): - def __init__(self, baseline_images, extensions, tol, - freetype_version, remove_text, savefig_kwargs, style): +class _ImageComparisonBase(object): + def __init__(self, tol, remove_text, savefig_kwargs): self.func = self.baseline_dir = self.result_dir = None - self.baseline_images = baseline_images - self.extensions = extensions self.tol = tol - self.freetype_version = freetype_version self.remove_text = remove_text self.savefig_kwargs = savefig_kwargs - self.style = style def delayed_init(self, func): assert self.func is None, "it looks like same decorator used twice" self.func = func self.baseline_dir, self.result_dir = _image_directories(func) - def setup(self): - func = self.func - plt.close('all') - self.setup_class() - try: - matplotlib.style.use(self.style) - matplotlib.testing.set_font_settings_for_testing() - func() - assert len(plt.get_fignums()) == len(self.baseline_images), ( - "Test generated {} images but there are {} baseline images" - .format(len(plt.get_fignums()), len(self.baseline_images))) - except: - # Restore original settings before raising errors during the update. - self.teardown_class() - raise - - def teardown(self): - self.teardown_class() - def copy_baseline(self, baseline, extension): baseline_path = os.path.join(self.baseline_dir, baseline) orig_expected_fname = baseline_path + '.' + extension @@ -304,6 +280,35 @@ def compare(self, idx, baseline, extension): expected_fname = self.copy_baseline(baseline, extension) _raise_on_image_difference(expected_fname, actual_fname, self.tol) + +class ImageComparisonDecorator(CleanupTest, _ImageComparisonBase): + def __init__(self, baseline_images, extensions, tol, + freetype_version, remove_text, savefig_kwargs, style): + _ImageComparisonBase.__init__(self, tol, remove_text, savefig_kwargs) + self.baseline_images = baseline_images + self.extensions = extensions + self.freetype_version = freetype_version + self.style = style + + def setup(self): + func = self.func + plt.close('all') + self.setup_class() + try: + matplotlib.style.use(self.style) + matplotlib.testing.set_font_settings_for_testing() + func() + assert len(plt.get_fignums()) == len(self.baseline_images), ( + "Test generated {} images but there are {} baseline images" + .format(len(plt.get_fignums()), len(self.baseline_images))) + except: + # Restore original settings before raising errors. + self.teardown_class() + raise + + def teardown(self): + self.teardown_class() + def nose_runner(self): func = self.compare func = _checked_on_freetype_version(self.freetype_version)(func) @@ -313,52 +318,59 @@ def nose_runner(self): for extension in self.extensions: yield funcs[extension], idx, baseline, extension - def pytest_runner(self): - from pytest import mark + def __call__(self, func): + self.delayed_init(func) + import nose.tools - extensions = map(_mark_xfail_if_format_is_uncomparable, - self.extensions) + @nose.tools.with_setup(self.setup, self.teardown) + def runner_wrapper(): + try: + for case in self.nose_runner(): + yield case + except GeneratorExit: + # nose bug... + self.teardown() - if len(set(self.baseline_images)) == len(self.baseline_images): - @mark.parametrize("extension", extensions) - @mark.parametrize("idx,baseline", enumerate(self.baseline_images)) - @_checked_on_freetype_version(self.freetype_version) - def wrapper(idx, baseline, extension): - __tracebackhide__ = True - self.compare(idx, baseline, extension) - else: - # Some baseline images are repeated, so run this in serial. - @mark.parametrize("extension", extensions) - @_checked_on_freetype_version(self.freetype_version) - def wrapper(extension): - __tracebackhide__ = True - for idx, baseline in enumerate(self.baseline_images): - self.compare(idx, baseline, extension) + return _copy_metadata(func, runner_wrapper) - # sadly we cannot use fixture here because of visibility problems - # and for for obvious reason avoid `_nose.tools.with_setup` - wrapper.setup, wrapper.teardown = self.setup, self.teardown +def _pytest_image_comparison(baseline_images, extensions, tol, + freetype_version, remove_text, savefig_kwargs, + style): + import pytest - return wrapper + extensions = map(_mark_xfail_if_format_is_uncomparable, extensions) - def __call__(self, func): - self.delayed_init(func) - if is_called_from_pytest(): - return _copy_metadata(func, self.pytest_runner()) - else: - import nose.tools + def decorator(func): + @pytest.mark.usefixtures('mpl_image_comparison_parameters') + @pytest.mark.parametrize('extension', extensions) + @pytest.mark.style(style) + @_checked_on_freetype_version(freetype_version) + @functools.wraps(func) + def wrapper(*args, **kwargs): + __tracebackhide__ = True + img = _ImageComparisonBase(tol=tol, remove_text=remove_text, + savefig_kwargs=savefig_kwargs) + img.delayed_init(func) + matplotlib.testing.set_font_settings_for_testing() + func(*args, **kwargs) - @nose.tools.with_setup(self.setup, self.teardown) - def runner_wrapper(): - try: - for case in self.nose_runner(): - yield case - except GeneratorExit: - # nose bug... - self.teardown() + # This is hacked on via the mpl_image_comparison_parameters fixture + # so that we don't need to modify the function's real signature for + # any parametrization. Modifying the signature is very very tricky + # and likely to confuse pytest. + extension, = func.parameters + + assert len(plt.get_fignums()) == len(baseline_images), ( + "Test generated {} images but there are {} baseline images" + .format(len(plt.get_fignums()), len(baseline_images))) + for idx, baseline in enumerate(baseline_images): + img.compare(idx, baseline, extension) + + wrapper.__wrapped__ = func # For Python 2.7. + return _copy_metadata(func, wrapper) - return _copy_metadata(func, runner_wrapper) + return decorator def image_comparison(baseline_images=None, extensions=None, tol=0, @@ -414,10 +426,16 @@ def image_comparison(baseline_images=None, extensions=None, tol=0, #default no kwargs to savefig savefig_kwarg = dict() - return ImageComparisonDecorator( - baseline_images=baseline_images, extensions=extensions, tol=tol, - freetype_version=freetype_version, remove_text=remove_text, - savefig_kwargs=savefig_kwarg, style=style) + if is_called_from_pytest(): + return _pytest_image_comparison( + baseline_images=baseline_images, extensions=extensions, tol=tol, + freetype_version=freetype_version, remove_text=remove_text, + savefig_kwargs=savefig_kwarg, style=style) + else: + return ImageComparisonDecorator( + baseline_images=baseline_images, extensions=extensions, tol=tol, + freetype_version=freetype_version, remove_text=remove_text, + savefig_kwargs=savefig_kwarg, style=style) def _image_directories(func): diff --git a/lib/matplotlib/tests/conftest.py b/lib/matplotlib/tests/conftest.py index dcdc3612eecb..2971a4314146 100644 --- a/lib/matplotlib/tests/conftest.py +++ b/lib/matplotlib/tests/conftest.py @@ -2,4 +2,5 @@ unicode_literals) from matplotlib.testing.conftest import (mpl_test_settings, + mpl_image_comparison_parameters, pytest_configure, pytest_unconfigure) diff --git a/lib/mpl_toolkits/tests/conftest.py b/lib/mpl_toolkits/tests/conftest.py index dcdc3612eecb..2971a4314146 100644 --- a/lib/mpl_toolkits/tests/conftest.py +++ b/lib/mpl_toolkits/tests/conftest.py @@ -2,4 +2,5 @@ unicode_literals) from matplotlib.testing.conftest import (mpl_test_settings, + mpl_image_comparison_parameters, pytest_configure, pytest_unconfigure) From f736296e634a724002461558d91353d67417b105 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 26 Mar 2017 15:38:19 -0400 Subject: [PATCH 03/10] TST: Allow baseline_images to be a fixture. The main point is it can be indirectly determined from other parametrizations. --- lib/matplotlib/testing/conftest.py | 8 +++++++- lib/matplotlib/testing/decorators.py | 17 +++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/lib/matplotlib/testing/conftest.py b/lib/matplotlib/testing/conftest.py index aab3318a8c65..fde9e0f99d72 100644 --- a/lib/matplotlib/testing/conftest.py +++ b/lib/matplotlib/testing/conftest.py @@ -65,7 +65,13 @@ def mpl_image_comparison_parameters(request, extension): # pytest won't get confused. # We annotate the decorated function with any parameters captured by this # fixture so that they can be used by the wrapper in image_comparison. + baseline_images = request.keywords['baseline_images'].args[0] + if baseline_images is None: + # Allow baseline image list to be produced on the fly based on current + # parametrization. + baseline_images = request.getfixturevalue('baseline_images') + func = request.function - func.__wrapped__.parameters = (extension, ) + func.__wrapped__.parameters = (baseline_images, extension) yield delattr(func.__wrapped__, 'parameters') diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py index 29908d6af6db..7f398a6493d9 100644 --- a/lib/matplotlib/testing/decorators.py +++ b/lib/matplotlib/testing/decorators.py @@ -344,6 +344,7 @@ def _pytest_image_comparison(baseline_images, extensions, tol, def decorator(func): @pytest.mark.usefixtures('mpl_image_comparison_parameters') @pytest.mark.parametrize('extension', extensions) + @pytest.mark.baseline_images(baseline_images) @pytest.mark.style(style) @_checked_on_freetype_version(freetype_version) @functools.wraps(func) @@ -359,7 +360,7 @@ def wrapper(*args, **kwargs): # so that we don't need to modify the function's real signature for # any parametrization. Modifying the signature is very very tricky # and likely to confuse pytest. - extension, = func.parameters + baseline_images, extension = func.parameters assert len(plt.get_fignums()) == len(baseline_images), ( "Test generated {} images but there are {} baseline images" @@ -373,7 +374,7 @@ def wrapper(*args, **kwargs): return decorator -def image_comparison(baseline_images=None, extensions=None, tol=0, +def image_comparison(baseline_images, extensions=None, tol=0, freetype_version=None, remove_text=False, savefig_kwarg=None, # Default of mpl_test_settings fixture and cleanup too. @@ -385,10 +386,14 @@ def image_comparison(baseline_images=None, extensions=None, tol=0, Arguments --------- - baseline_images : list + baseline_images : list or None A list of strings specifying the names of the images generated by calls to :meth:`matplotlib.figure.savefig`. + If *None*, the test function must use the ``baseline_images`` fixture, + either as a parameter or with pytest.mark.usefixtures. This value is + only allowed when using pytest. + extensions : [ None | list ] If None, defaults to all supported extensions. @@ -415,9 +420,6 @@ def image_comparison(baseline_images=None, extensions=None, tol=0, '_classic_test' style. """ - if baseline_images is None: - raise ValueError('baseline_images must be specified') - if extensions is None: # default extensions to test extensions = ['png', 'pdf', 'svg'] @@ -432,6 +434,9 @@ def image_comparison(baseline_images=None, extensions=None, tol=0, freetype_version=freetype_version, remove_text=remove_text, savefig_kwargs=savefig_kwarg, style=style) else: + if baseline_images is None: + raise ValueError('baseline_images must be specified') + return ImageComparisonDecorator( baseline_images=baseline_images, extensions=extensions, tol=tol, freetype_version=freetype_version, remove_text=remove_text, From 3810c2a19cbe16ccc0f20a21adf5b350cea58504 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sun, 26 Mar 2017 15:41:32 -0400 Subject: [PATCH 04/10] TST: Properly parametrize the last mathtext tests. --- lib/matplotlib/tests/test_mathtext.py | 58 ++++++++++++++------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/lib/matplotlib/tests/test_mathtext.py b/lib/matplotlib/tests/test_mathtext.py index de089f4e1c4f..544d3ef89201 100644 --- a/lib/matplotlib/tests/test_mathtext.py +++ b/lib/matplotlib/tests/test_mathtext.py @@ -14,6 +14,7 @@ import matplotlib.pyplot as plt from matplotlib import mathtext + math_tests = [ r'$a+b+\dot s+\dot{s}+\ldots$', r'$x \doteq y$', @@ -160,36 +161,39 @@ for set in chars: font_tests.append(wrapper % set) -def make_set(basename, fontset, tests, extensions=None): - def make_test(filename, test): - @image_comparison(baseline_images=[filename], extensions=extensions) - def single_test(): - matplotlib.rcParams['mathtext.fontset'] = fontset - fig = plt.figure(figsize=(5.25, 0.75)) - fig.text(0.5, 0.5, test, horizontalalignment='center', verticalalignment='center') - func = single_test - func.__name__ = str("test_" + filename) - return func - # We inject test functions into the global namespace, rather than - # using a generator, so that individual tests can be run more - # easily from the commandline and so each test will have its own - # result. - for i, test in enumerate(tests): - filename = '%s_%s_%02d' % (basename, fontset, i) - globals()['test_%s' % filename] = make_test(filename, test) +@pytest.fixture +def baseline_images(request, fontset, index): + return ['%s_%s_%02d' % (request.param, fontset, index)] + + +@pytest.mark.parametrize('index, test', enumerate(math_tests), + ids=[str(index) for index in range(len(math_tests))]) +@pytest.mark.parametrize('fontset', + ['cm', 'stix', 'stixsans', 'dejavusans', + 'dejavuserif']) +@pytest.mark.parametrize('baseline_images', ['mathtext'], indirect=True) +@image_comparison(baseline_images=None) +def test_mathtext_rendering(baseline_images, fontset, index, test): + matplotlib.rcParams['mathtext.fontset'] = fontset + fig = plt.figure(figsize=(5.25, 0.75)) + fig.text(0.5, 0.5, test, + horizontalalignment='center', verticalalignment='center') + -make_set('mathtext', 'cm', math_tests) -make_set('mathtext', 'stix', math_tests) -make_set('mathtext', 'stixsans', math_tests) -make_set('mathtext', 'dejavusans', math_tests) -make_set('mathtext', 'dejavuserif', math_tests) +@pytest.mark.parametrize('index, test', enumerate(font_tests), + ids=[str(index) for index in range(len(font_tests))]) +@pytest.mark.parametrize('fontset', + ['cm', 'stix', 'stixsans', 'dejavusans', + 'dejavuserif']) +@pytest.mark.parametrize('baseline_images', ['mathfont'], indirect=True) +@image_comparison(baseline_images=None, extensions=['png']) +def test_mathfont_rendering(baseline_images, fontset, index, test): + matplotlib.rcParams['mathtext.fontset'] = fontset + fig = plt.figure(figsize=(5.25, 0.75)) + fig.text(0.5, 0.5, test, + horizontalalignment='center', verticalalignment='center') -make_set('mathfont', 'cm', font_tests, ['png']) -make_set('mathfont', 'stix', font_tests, ['png']) -make_set('mathfont', 'stixsans', font_tests, ['png']) -make_set('mathfont', 'dejavusans', font_tests, ['png']) -make_set('mathfont', 'dejavuserif', font_tests, ['png']) def test_fontinfo(): import matplotlib.font_manager as font_manager From 878dfaae1487ff20d01070001ae89a70703c624a Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Wed, 29 Mar 2017 21:07:24 -0400 Subject: [PATCH 05/10] Restore ImageComparisonTest and its static methods. --- lib/matplotlib/testing/decorators.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py index 7f398a6493d9..712e6a867897 100644 --- a/lib/matplotlib/testing/decorators.py +++ b/lib/matplotlib/testing/decorators.py @@ -281,7 +281,7 @@ def compare(self, idx, baseline, extension): _raise_on_image_difference(expected_fname, actual_fname, self.tol) -class ImageComparisonDecorator(CleanupTest, _ImageComparisonBase): +class ImageComparisonTest(CleanupTest, _ImageComparisonBase): def __init__(self, baseline_images, extensions, tol, freetype_version, remove_text, savefig_kwargs, style): _ImageComparisonBase.__init__(self, tol, remove_text, savefig_kwargs) @@ -309,6 +309,12 @@ def setup(self): def teardown(self): self.teardown_class() + @staticmethod + @cbook.deprecated('2.1', + alternative='remove_ticks_and_titles') + def remove_text(figure): + remove_ticks_and_titles(figure) + def nose_runner(self): func = self.compare func = _checked_on_freetype_version(self.freetype_version)(func) @@ -437,7 +443,7 @@ def image_comparison(baseline_images, extensions=None, tol=0, if baseline_images is None: raise ValueError('baseline_images must be specified') - return ImageComparisonDecorator( + return ImageComparisonTest( baseline_images=baseline_images, extensions=extensions, tol=tol, freetype_version=freetype_version, remove_text=remove_text, savefig_kwargs=savefig_kwarg, style=style) From fd8e844ab20e213ba08d3bf833e8cf3da1bbc88f Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 13 Apr 2017 14:17:54 -0400 Subject: [PATCH 06/10] Raise consistent shape error out of save_diff_image. A similar check is done in calculate_rms. --- lib/matplotlib/testing/compare.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/matplotlib/testing/compare.py b/lib/matplotlib/testing/compare.py index 0483e875eb17..c3d649e38069 100644 --- a/lib/matplotlib/testing/compare.py +++ b/lib/matplotlib/testing/compare.py @@ -366,7 +366,7 @@ def calculate_rms(expectedImage, actualImage): "Calculate the per-pixel errors, then compute the root mean square error." if expectedImage.shape != actualImage.shape: raise ImageComparisonFailure( - "image sizes do not match expected size: {0} " + "Image sizes do not match expected size: {0} " "actual size {1}".format(expectedImage.shape, actualImage.shape)) num_values = expectedImage.size abs_diff_image = abs(expectedImage - actualImage) @@ -470,7 +470,10 @@ def save_diff_image(expected, actual, output): actual, actualImage, expected, expectedImage) expectedImage = np.array(expectedImage).astype(float) actualImage = np.array(actualImage).astype(float) - assert expectedImage.shape == actualImage.shape + if expectedImage.shape != actualImage.shape: + raise ImageComparisonFailure( + "Image sizes do not match expected size: {0} " + "actual size {1}".format(expectedImage.shape, actualImage.shape)) absDiffImage = np.abs(expectedImage - actualImage) # expand differences in luminance domain From 75bebef83188c0407a921d511dc784dbb616d36d Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 13 Apr 2017 17:38:12 -0400 Subject: [PATCH 07/10] Stop handling GeneratorExit in nose image comparison. While this does cause a periodic printout if using the image_comparison decorator from nose, trying to handle this exception means that test images don't get checked at all, which is even worse. --- lib/matplotlib/testing/decorators.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py index 712e6a867897..329c90805ff1 100644 --- a/lib/matplotlib/testing/decorators.py +++ b/lib/matplotlib/testing/decorators.py @@ -330,12 +330,8 @@ def __call__(self, func): @nose.tools.with_setup(self.setup, self.teardown) def runner_wrapper(): - try: - for case in self.nose_runner(): - yield case - except GeneratorExit: - # nose bug... - self.teardown() + for case in self.nose_runner(): + yield case return _copy_metadata(func, runner_wrapper) From ba7106acdbd9a6f92f886044736e89f6d8758be8 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 15 Apr 2017 02:46:32 -0400 Subject: [PATCH 08/10] Add tests for nose version of image_comparison. This should ensure that it continues to work for downstream users even though we've stopped using it. --- .../test_compare_images/simple.pdf | Bin 0 -> 1425 bytes .../test_compare_images/simple.png | Bin 0 -> 9396 bytes .../test_compare_images/simple.svg | 206 ++++++++++++++++++ lib/matplotlib/tests/test_compare_images.py | 121 +++++++++- 4 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 lib/matplotlib/tests/baseline_images/test_compare_images/simple.pdf create mode 100644 lib/matplotlib/tests/baseline_images/test_compare_images/simple.png create mode 100644 lib/matplotlib/tests/baseline_images/test_compare_images/simple.svg diff --git a/lib/matplotlib/tests/baseline_images/test_compare_images/simple.pdf b/lib/matplotlib/tests/baseline_images/test_compare_images/simple.pdf new file mode 100644 index 0000000000000000000000000000000000000000..43c04c8531a9962ba23fb29a2be8297fd3ff5636 GIT binary patch literal 1425 zcmZ`(e@qi+7*58V@zm%NqL~K$VuoZ=yt`gGucR`eP(naydev=FmmGH=rC9Ik^{W;W zHIr##G{&J$MMXu7L~9^qF;OQ@kw5%}h(V1p7`M2{OePzfN+faL*Xz-)-R_^e@5%dq z&-*^_`&_oW#FfV`v{Bj7C*bTwiUkbNs_Q6+1JG{C2SdP$5f7lf^+5<|r-Y<{<^zR= z6jT)wC?qA6Mo^iEBm{xYNOEZ^0-W*fmJouv3i8R3qTRYCix7d8n5V>r?}rft=<>Cg zvpD{C#^2>&HK4^>N%5;bz#HjbR#n$Q8G%Ky%M<0W3-{qHlwoRo^6HWr3Q9Uu<$AMoBOVjcOQGVIinR#7myp?bW}yJA zfoOLRs5CAj(KSgyQjl4(%OAiZ0ooP7DwaT5Q?TfjQ1zi&z>?W` zNMZ=-P+Chxj)WT*tKH0$Z5x`m=Ir^~SDGbfGJSbNeJ5V*yQuDJT{7$Vh0UMsY_wee z8GSc(c*33UZ?=2y zJ1>2_+v6>3?W{g|vH%VgPFiZ6(RuLFs*azw7nk<9sGV0bx}HtrS7xdA-WLBpzo2Jb z@J#TjbHp*0HLbhr)o}3K?b=_y)!U+%-+a*C(Ot3WYMx`_+&5>(#D?ec-Q1auIhD(L zd)8QH?^P=M-&5@eTFwoAK6scbIREOm*OteRe$0SJhYP=pQiuC1eTVAj1Q!2#?X+#= ztD38+SP^n;WuHV#nhI~umsox0mhwmPnEe7|F= zlksqz-Q0Py(ea5THjk{lImRN^*sLOq#hsJz1}#f~C1a$EtY#Sd|6xg+LadX+B%QXfc{`~j%P{+;D&jRs<(VcUB&jc6VF1&IEQ5DRA5Skh5ULY5JDDm%Nh H%0>MHA^yZ| literal 0 HcmV?d00001 diff --git a/lib/matplotlib/tests/baseline_images/test_compare_images/simple.png b/lib/matplotlib/tests/baseline_images/test_compare_images/simple.png new file mode 100644 index 0000000000000000000000000000000000000000..070a0a9917df1a69a15ebc7e1bf47cb51c47ca48 GIT binary patch literal 9396 zcmds7c{r4P)E-M@e~Xm8-o`Ta5W=98dZn#-p#~!4GcVEs*VU@aOdMqi4YX`EWbzd=Ur{4E&Gdwn?@Jc+nca z?I3=yw=3Qs{{2zDH2Fa2GP-{Czuxe?=p4p{q&V57Yc+CwM1PeXCw>*V1#W)OIs> z?9Un=?=Qm}q?r?u(Ah%eF$DN&Z^vT_5y{aT5EewF`~?ncA`*oW3

ETo4IJ$YKpw z1`@KVTZ52?h%28Wlptb)=Ks}OuD&Vcedd~Ddrrh;=JIN%(^u;1>S(cY83w<__Uo9If2oYIgq2C9bMmvW?^~Tn)ub$VaS{^+Y z?ij?#Dy6t@tC;+l;_wKM8e3z(F0`xRzy7pxS>tHZcfm*Hm{x^=@81lIA5_lPlX{_} zh|@fC+_cwQN^{1k(S9BIecCn;SY6J zet%cuzNnc#}(E2bEQ3fnsHZ9Qi^Wa9noHrS)?^)d+jC6 zjiTlmqv8@Xh;QfaI`2@fxvBgDzY~KEMNyV(0kLugg=$~j#N02*TGmV=a691xm30Bt zP%2H$TaK%Ewu_4!^6=9C+eYbbE#Ls;lgO`@`#gX-!r2wbyPFg{Geg^p)sO;wO(?vT zZN^m5AbcB`W1S<=hudg?zvK(7T3+T;DNSwB<_aC18s*$vd3v=u`1L)vZuRtR4*0T> zRwOTtH$GX%EJ^SLjJ7e~kxg$*ENCka+UQWuvu|J3heGKlD)>l`g2$cTsq6c2JBF+p z>fj@!357kUz$d(6cQ8Com{uV%)D_d^7%+@hQjbI1d1=GXS28#6Njy*oCC?cSR=8OW z<`8S)TlT;1kWMw^&C0@1{GAcAlO+0Nzl&9pw0Z=f-YVbk2GWK+y}it_b~-6ClD1d= zk`PEadp{9b)4igDv=!H@=$MJGmxRI!*@wiwX_p$B$g;uEZ|3lgj2y8_^n`=xVMn&H zJ|r+bR;yRFBFX4_icr|K8W?u9oCQt@x;ch^Q@Ncxddo?n5Du^9$JvnEx5KYX(#o`r zddDh7`k)VM0R92PA9U?7-@~ho!(xtCCHT5LHx4a;kOogqj*|)<{hJX}ZOw z=kRAkKWwveKM`}Hz3xbT&sv8H_lf7a@L?E*;2lEHJ9pL?4%aYRFO%egmC-^l@~o^y zdL|V71w=#{>n{`3*Pi)E}j`Aad_sDoU5jL_$+4d-vj)Yo+xOhaN^pH7o93 zGwTxrhj;2GUNX;tFQqc~0Oi4P>wSHevSm-DH6S9Q)L+E!OXT|PJ#P-sn`I`N{RP9y zo&gwb>eN(CxAS^-3HlFj@z$sqmla!GpT6_?6Zl%>#S1lKU+CoLg)Vf75_IqptE@g( zP(M)-A3yeVY*JE-;T3UqMyNZ*-nc=F_mD5;{ujKVL$&Pts;?&lroS+)Hg6uxDJH_C zv^c{n0PQ+6n3l>e8i#}7JhkX zUtd8ex}%ii#EDoFv<&>ZmV=cQZ7A&;pPQqkq;uk`xb%p*$sNRmOm@QS;}8!$`FPW; zQZ;87ZEYY$6EIW;6Mx83;I5<4yS&Z&5;X$d851&Fh-5*ZVcEg=%eKeWNnwO_%1goTc4t>oh4K9P*aNmR6r=Ql-2itJ1`59}+Y%TcV zv+Qq);%#}&ZRl4s`jW!L1yJ5GvlC(5sr-zAsNVW@iTna`yLMepULC9~e+9E#27|Ch-VRbKE3b!0xnkaA)DYcrq>Os7C)?ecmh-Cg8N6^QD+8a z2v;>$^we}_m8^n7Pnv%Z%j4P0OW-jR@L7U(wzjUDNKB?oeZ%#dvxDO40tdzA6N9h4 zQd^teWNF5Ax5#`?(um1nh*7R<-Fy7wO7_NKj(t%y`xMa`4B)IB+q$cIMfeE{((GX`5dJD zN=+@!_7Z~02-;88sP`dU?a6%@5KH;@zgGK&jEA1r%U_mfeoXN$a(!|A<&6EQQybsB zd2>Yp#4&E*omsn{v9R0A*p$#^%L=opHjn(iNWRNEJl&KZGSZ-UUL8G!+(2-g1^-yELVyh^r`=perkC}*1 zPlBJ|lGRFQJWLWlSZQFy@cVRGBXGyd#^K#SZ{UB_jM?uC#3>eo4}PJj0FkU-`|$;C zT%9#m_a@icK{{9qTE`crf7UiI?fLy--;_VAMG;FX862C@QspZm5Y! z)-c<%XRv|X@nPi|B~JSO2x z-|yfgCv75}BZkKV74f0(NxFrFFhG%d4$8`SC_Xk6b5V-^R?Xc@cLq{<0bnm4Tmrj| zD*OoSsX1d?Thm=a35hXL&8JSqnxGT_h#q#Te<>MA_)5dYi!l%L3s>C9#o)!1L938w z%)uwObPbh#-=0b4R#42WmV5Nb`NefwJT&DC>!75^E&oWPC3S-j+1Kr=2C{rFG-U)o z-c8F;;NHE06Ne5Cu91#Dzgj5FMgi^>=m)c*xAO^S`ATu?biyVl3$ux=eK5RMo(s6^ zdWNUy(P7*Z)Yuouzu|7MxlwsmLj?*_{fT>Qt2$gT$ewW7ow8X{Fl=;3gHws##t?vN!c1E(%rSwX8p&s-S>8lhlx zs1tnR;sm&oA{I1)kW>{H7j9BZY}J$Wom3lNzOBT^-v%ZMKOi>8`}_yQN+3MQwZ2tn^O2#Xg83px7pe5?2+-R=2691}Ei#9qIx>#8g`>2h#{ z4LfuM9Fp(KZo>NJ?A^6Dx6?~U>69*3Jnb^WqRw$)XO^trb34NnGvb(tuY? zMaC%LwN28%Fkxz{fDugJ;yybv3b=)eXTN?uJvNp{NrNCH)$?(ezVB0TLoR$l{PTR9 zU=EdMayaxI2!bv(bgKiYv2#~#Z4|Bt;n^vLH2%^n^!rp%gf(yZ9Ib8uLa z;jl>jH+4$iPY~7!cJ9Zc;Hdy=+y-D2YiklrHR1yrK|rHsULz>ZGh%B<2tCe8d6|q1 z&=7%vfLsX!@Nps+tNaGlKpj1)S>!!I5McYw47~n-r$37s1#ut9t6>9QTm)|Lh}pT* zblbAmuTAaj^glTqaGQuP9sw*ZZW3Red0(_@@BSZ$tGYdAEZrytOB78jd_h61jzIYHIFx&(XFeSkf6FB^Lixw`Vs8Jk*mvxouQ5v+WD@fp{NFWwn>OLUP*f29Y zUJ$HzdT#n>(xVK424q;Wlb$*994_|XV#Etd^+8IoSRiPuW#;D2(_=VlR+X`|Ust8! zm@Sbt2wzzJ@nhY9OjeerHWwEcvz4?3F7`+dk5cX;?yl3LyWY+`;UWQrV0+-e_Uk*S zty9Bu!Y@lNUZS+!Z+XiDs{|seIPY-z(+9`j8IK+(ogRV69$NLj47`C%@EOmIMn+9Z zYKPWHO5P|g))~_$rq?zJR(#i;8FZtgN}Oi?ovAbdU570ZwmLbL_9k0w-p;GDakHZ% z2&>2Bgg-)Fem{Gx@YnF{STjFR5`9QQ8RW-MHf2Ts-t3-N>q~u>3CoS$B9T9Uc|Ac% zRMdJ8$Ht8XzHST5Nl*nf<7`3c^0YMO3-3CnCW}=QHp6@O(CT0<$UYEU+~n;k*w~oy zYqNm9f@aZo_ouqNWE`2NK5_nj8Zkj1^omBv%#5BfdhZZ!3LnsR6CNvw2@-(1eF?Bc z5UNil1q%ZoLD6Kx3`8Y?s+gb+t4m|gDaTTTni_kWgc{-QQq;xL(a|vt+~i3^=y}&& z(~dQPrv8O5*u!&0$izQgA$mIDxiiO9Ag9J|1$k^%4oW}ZA7Sb6dxz*_Uzq9nbSq-j zbr6-i#NCuPxy-kR3W81rP`|XSa~io!R;2r5A-mr3YC>l@eY%_QeVU@}wQnDIN{|gW zyu`e|Om>Pb_s`7NC=P$}q);)QJP34E!mFjsN$3R3(V{NM%io!Ung&^g>UF3Vu8*so z&`(7RoH$`^{ZFtm)F=baqxJOHx}lkB;yn>h&T{2aLahgs1vBL@I)UgD56*Mj26g*h zOl2^p_i4e0R~??<_xnIsM4iJMNS`!5#V?DFR5{SY#CT6ZR11K9@qd4g7K)Gxn2kN2 z{8cB&pXacqo?G*0#~>0Lz%6z^XUfy875G!N*Ehm(`QXF1m0Bbr8F_O%#dVm`%DgGJ zncXrVaKfaHUnh&u5m5>WiG}AX%R37ulDb29k~n*^Kr&u}@?gKRwY8-M0ftHpS4Rb( zTl1P(BuW%q2y#WEG3DcQvw-o=7DV32Ko%0K8g@R3^?Grt^onUOp#%f91vSF%OZq_r znRKt?9uaSHDe+ze;YlI^i4~~Mwha1re5f|>Fz&|X@xji;P?6``_59&3w&MeO>OiDeC;+ zOmZ$_W5~PC4(Z8w$ln1c8 z_tz{L3y4HOMSJrl`Vcw?EJb^T@87MRL_|e#i~0~E3S|_3Sa1!0szazzS~r}!azP!= zn+PMvlaMIN#7)SabIDAAgsy?3X z%Os|!TCTP1d7;;R^o5hrC;P*Pcbz(QihQ;>A25GKo!#)G zQi$t^$+GPyh}w9c!?rmkE#~c8T+?J(zw^BP1QCgVHkNmO{YyBOKJrY zIH*1?ymB>C-Zekm=9xR|A*$S@%brax$M}V1qu)73-8)3_NETHD#Rd6YUdj|(i8nCn zd`sobJwQq;b61l;v>K^-ue>jZz{P0@MFwT=Am@(B;U zPfIta$2`h0dENOhsl0SfPMx2_(VLC!DvG$yL|h@a0a`S?*PV= zOL0f4tpdlq!0r%%TB(6$3(^p*8GyYYL7)AOdscRf$8SiGH*MfEd`dN5u8bS@4$;lK zX~le@aJnJ!1NhIttAuw~#Ne06T> zi#KI+KQKeHZtu6WVA%MyA&#}}v?BBz za1IFlVD5 z{8BWHvK#`86ZZ?^0$QxA)QqfG+v?<>NAVVne4wgMx8U=mHW37GhYXVG;AcC3SX z#HYf;^V+y%nWQLvp&haP=C$*v?@8 zqUwqGR`wV8`#8Z3<_0ox`lR2|p6mdkZvw2tv0(yx)AXZTBpeqEKQiAe6ec={GvI{- zrt=YSgoxxmXT;9Z!VO9)QNJu4zydlco}>&q|=`!AzNV5zLg3y_b3f zvc)@0O4y+L(XyX*0#K)g96K+!*-g}n^c?EuKQ9v%h5LE~hLa2~oLO*@0{Ue@8D<*_ z5b;O>N7KnaZD&OY=EcVX5#w;ZuJCKua38ZWIrd9Tf`yGyuc`3$>yhB$j=@3?y<%Ry znjZ{6t#IBAWz!E-^QTl!PST+gGs5Wx!`i#a2n5#<{9OPuknBlm)^YPoC!lM6z5Itv zVPgT*jg4H+Nr(QjZ3L$UB&ku=)#A*rSh@R=w4r$V$@Y03pjssm_ggxoXP5+Qdwwwb z{vsKmwi@}%s=hxY!;kF)*mkDS5Uyu`-8#{kvG8PrM5Cl?R9o5w>Mv2|W~>(XP5>fL zEZ5cA1s?Qx&xtQs$bf8MgD0VsCcHh>DjlpG^Bf{5gi3}Y054X^Ikqg!MA|1WCnC)d zq$%tn1LO>NY6J^Qm(Pu86nKcv4EeYGv=<^`s)kU zQVt5319|EP%XPu%jYK7vaj$In3m0rw(hzwCf8hf1p0rfWDppFjsTtRATSrjvbaca2 zoP#7k576c2RdIE zWsi_xzj<6eLN#JIM58!l*Df5L6ZRB$fP_CU0lw9cO$z0&nVTJ!37xc@D6ftx_g^oP zh5go`NYXSik`5NU_m9c&(XXSs;|%?W5MfiB(zh^6O;*b4Cjn9e7rw+$@vVZ@BBlyQ;AVwBD`omIQpU nq$K?R!B + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/matplotlib/tests/test_compare_images.py b/lib/matplotlib/tests/test_compare_images.py index 10896dd90212..4f8cbbb42555 100644 --- a/lib/matplotlib/tests/test_compare_images.py +++ b/lib/matplotlib/tests/test_compare_images.py @@ -1,13 +1,17 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import io import os import shutil +import warnings from numpy.testing import assert_equal, assert_almost_equal +import pytest from matplotlib.testing.compare import compare_images -from matplotlib.testing.decorators import _image_directories +from matplotlib.testing.decorators import _image_directories, image_comparison +from matplotlib.testing.exceptions import ImageComparisonFailure baseline_dir, result_dir = _image_directories(lambda: 'dummy func') @@ -97,3 +101,118 @@ def test_image_compare_shade_difference(): # Now test the reverse comparison. image_comparison_expect_rms(im2, im1, tol=0, expect_rms=1.0) + + +# +# The following tests are used by test_nose_image_comparison to ensure that the +# image_comparison decorator continues to work with nose. They should not be +# prefixed by test_ so they don't run with pytest. +# + +def nosetest_empty(): + pass + + +def nosetest_simple_figure(): + import matplotlib.pyplot as plt + fig, ax = plt.subplots(figsize=(6.4, 4), dpi=100) + ax.plot([1, 2, 3], [3, 4, 5]) + return fig + + +def nosetest_manual_text_removal(): + from matplotlib.testing.decorators import ImageComparisonTest + + fig = nosetest_simple_figure() + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + # Make sure this removes text like it should. + ImageComparisonTest.remove_text(fig) + + assert len(w) == 1 + assert 'remove_text function was deprecated in version 2.1.' in str(w[0]) + + +@pytest.mark.parametrize( + 'func, kwargs, errors, failures, dots', + [ + (nosetest_empty, {'baseline_images': []}, [], [], ''), + (nosetest_empty, {'baseline_images': ['foo']}, + [(AssertionError, + 'Test generated 0 images but there are 1 baseline images')], + [], + 'E'), + (nosetest_simple_figure, + {'baseline_images': ['basn3p02'], 'extensions': ['png'], + 'remove_text': True}, + [], + [(ImageComparisonFailure, 'Image sizes do not match expected size:')], + 'F'), + (nosetest_simple_figure, + {'baseline_images': ['simple']}, + [], + [(ImageComparisonFailure, 'images not close')] * 3, + 'FFF'), + (nosetest_simple_figure, + {'baseline_images': ['simple'], 'remove_text': True}, + [], + [], + '...'), + (nosetest_manual_text_removal, + {'baseline_images': ['simple']}, + [], + [], + '...'), + ], + ids=[ + 'empty', + 'extra baselines', + 'incorrect shape', + 'failing figure', + 'passing figure', + 'manual text removal', + ]) +def test_nose_image_comparison(func, kwargs, errors, failures, dots, + monkeypatch): + nose = pytest.importorskip('nose') + monkeypatch.setattr('matplotlib._called_from_pytest', False) + + class TestResultVerifier(nose.result.TextTestResult): + def __init__(self, *args, **kwargs): + super(TestResultVerifier, self).__init__(*args, **kwargs) + self.error_count = 0 + self.failure_count = 0 + + def addError(self, test, err): + super(TestResultVerifier, self).addError(test, err) + + if self.error_count < len(errors): + assert err[0] is errors[self.error_count][0] + assert errors[self.error_count][1] in str(err[1]) + else: + raise err[1] + self.error_count += 1 + + def addFailure(self, test, err): + super(TestResultVerifier, self).addFailure(test, err) + + assert self.failure_count < len(failures), err[1] + assert err[0] is failures[self.failure_count][0] + assert failures[self.failure_count][1] in str(err[1]) + self.failure_count += 1 + + func = image_comparison(**kwargs)(func) + loader = nose.loader.TestLoader() + suite = loader.loadTestsFromGenerator( + func, + 'matplotlib.tests.test_compare_images') + output = io.StringIO() + result = TestResultVerifier(stream=output, descriptions=True, verbosity=1) + with warnings.catch_warnings(): + # Nose uses deprecated stuff; we don't care about it. + warnings.simplefilter('ignore', DeprecationWarning) + suite.run(result=result) + + assert output.getvalue() == dots + assert result.error_count == len(errors) + assert result.failure_count == len(failures) From c1864083bd4ead53b16ab7b4901c247e9bfeb79d Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 15 Apr 2017 18:15:14 -0400 Subject: [PATCH 09/10] Handle comments from PR. * Install nose in one build. * Add docstrings and comments on new functions/classes to clarify what they do. * Use consistent `yield` pattern for fixture. --- .travis.yml | 5 +++-- lib/matplotlib/testing/conftest.py | 6 ++++-- lib/matplotlib/testing/decorators.py | 26 ++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2bd4d5ba9ebb..7d79f7e194f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,6 +45,7 @@ env: - NPROC=2 - INSTALL_PEP8= - RUN_PEP8= + - NOSE= - PYTEST_ARGS="-ra --maxfail=1 --timeout=300 --durations=25 --cov-report= --cov=lib -n $NPROC" - PYTHON_ARGS= - DELETE_FONT_CACHE= @@ -52,7 +53,7 @@ env: matrix: include: - python: 2.7 - env: MOCK=mock NUMPY=numpy==1.7.1 PANDAS=pandas + env: MOCK=mock NUMPY=numpy==1.7.1 PANDAS=pandas NOSE=nose - python: 2.7 env: BUILD_DOCS=true - python: 3.4 @@ -114,7 +115,7 @@ install: pip install --upgrade setuptools - | # Install dependencies from pypi - pip install $PRE python-dateutil $NUMPY pyparsing!=2.1.6 $PANDAS cycler codecov coverage $MOCK + pip install $PRE python-dateutil $NUMPY pyparsing!=2.1.6 $PANDAS cycler codecov coverage $MOCK $NOSE pip install $PRE -r doc-requirements.txt # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124 diff --git a/lib/matplotlib/testing/conftest.py b/lib/matplotlib/testing/conftest.py index fde9e0f99d72..ac79e63ccd85 100644 --- a/lib/matplotlib/testing/conftest.py +++ b/lib/matplotlib/testing/conftest.py @@ -73,5 +73,7 @@ def mpl_image_comparison_parameters(request, extension): func = request.function func.__wrapped__.parameters = (baseline_images, extension) - yield - delattr(func.__wrapped__, 'parameters') + try: + yield + finally: + delattr(func.__wrapped__, 'parameters') diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py index 329c90805ff1..3a0dd4e0f9a8 100644 --- a/lib/matplotlib/testing/decorators.py +++ b/lib/matplotlib/testing/decorators.py @@ -234,6 +234,12 @@ def _mark_xfail_if_format_is_uncomparable(extension): class _ImageComparisonBase(object): + """ + Image comparison base class + + This class provides *just* the comparison-related functionality and avoids + any code that would be specific to any testing framework. + """ def __init__(self, tol, remove_text, savefig_kwargs): self.func = self.baseline_dir = self.result_dir = None self.tol = tol @@ -282,6 +288,15 @@ def compare(self, idx, baseline, extension): class ImageComparisonTest(CleanupTest, _ImageComparisonBase): + """ + Nose-based image comparison class + + This class generates tests for a nose-based testing framework. Ideally, + this class would not be public, and the only publically visible API would + be the :func:`image_comparison` decorator. Unfortunately, there are + existing downstream users of this class (e.g., pytest-mpl) so it cannot yet + be removed. + """ def __init__(self, baseline_images, extensions, tol, freetype_version, remove_text, savefig_kwargs, style): _ImageComparisonBase.__init__(self, tol, remove_text, savefig_kwargs) @@ -339,14 +354,24 @@ def runner_wrapper(): def _pytest_image_comparison(baseline_images, extensions, tol, freetype_version, remove_text, savefig_kwargs, style): + """ + Decorate function with image comparison for pytest. + + This function creates a decorator that wraps a figure-generating function + with image comparison code. Pytest can become confused if we change the + signature of the function, so we indirectly pass anything we need via the + `mpl_image_comparison_parameters` fixture and extra markers. + """ import pytest extensions = map(_mark_xfail_if_format_is_uncomparable, extensions) def decorator(func): + # Parameter indirection; see docstring above and comment below. @pytest.mark.usefixtures('mpl_image_comparison_parameters') @pytest.mark.parametrize('extension', extensions) @pytest.mark.baseline_images(baseline_images) + # END Parameter indirection. @pytest.mark.style(style) @_checked_on_freetype_version(freetype_version) @functools.wraps(func) @@ -358,6 +383,7 @@ def wrapper(*args, **kwargs): matplotlib.testing.set_font_settings_for_testing() func(*args, **kwargs) + # Parameter indirection: # This is hacked on via the mpl_image_comparison_parameters fixture # so that we don't need to modify the function's real signature for # any parametrization. Modifying the signature is very very tricky From d1c1e1be440122673b1555d047cea49094a70dc3 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 15 Apr 2017 18:29:31 -0400 Subject: [PATCH 10/10] Fix nose image comparison test on Python 2. --- lib/matplotlib/tests/test_compare_images.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/matplotlib/tests/test_compare_images.py b/lib/matplotlib/tests/test_compare_images.py index 4f8cbbb42555..883d9b7a9165 100644 --- a/lib/matplotlib/tests/test_compare_images.py +++ b/lib/matplotlib/tests/test_compare_images.py @@ -1,6 +1,8 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import six + import io import os import shutil @@ -206,7 +208,10 @@ def addFailure(self, test, err): suite = loader.loadTestsFromGenerator( func, 'matplotlib.tests.test_compare_images') - output = io.StringIO() + if six.PY2: + output = io.BytesIO() + else: + output = io.StringIO() result = TestResultVerifier(stream=output, descriptions=True, verbosity=1) with warnings.catch_warnings(): # Nose uses deprecated stuff; we don't care about it.