Figure equality-based tests.

anntzer · anntzer · commit 9cd3fbeda676 · 2018-06-19T18:59:57.000+02:00
Implement a `check_figures_equal` decorator, which allows tests where
both the reference and the test image are generated.  The idea is to
allow tests of the form "this feature should be equivalent to that
(usually more complex) way of achieving the same thing" without further
bloating the test image directory.

The saved images are properly created in the `result_images` folder, but
cannot be "accepted" or "rejected" using the triage_tests UI (as there
is indeed no reference image to be saved in the repo).

Includes an example use case.
diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py
@@ -407,6 +407,46 @@ def image_comparison(baseline_images, extensions=None, tol=0,
             savefig_kwargs=savefig_kwarg, style=style)
 
 
+def check_figures_equal(*, extensions=("png", "pdf", "svg"), tol=0):
+    """
+    Decorator for test cases that generate and compare two figures.
+
+    The decorated function must take two arguments, *fig_test* and *fig_ref*,
+    and draw the test and reference images on them.  After the function
+    returns, the figures are saved and compared.
+
+    Arguments
+    ---------
+    extensions : list, default: ["png", "pdf", "svg"]
+        The extensions to test.
+    tol : float
+        The RMS threshold above which the test is considered failed.
+    """
+
+    def decorator(func):
+        import pytest
+
+        _, result_dir = map(Path, _image_directories(func))
+
+        @pytest.mark.parametrize("ext", extensions)
+        def wrapper(ext):
+            fig_test = plt.figure("test")
+            fig_ref = plt.figure("reference")
+            func(fig_test, fig_ref)
+            test_image_path = str(
+                result_dir / (func.__name__ + "." + ext))
+            ref_image_path = str(
+                result_dir / (func.__name__ + "-expected." + ext))
+            fig_test.savefig(test_image_path)
+            fig_ref.savefig(ref_image_path)
+            _raise_on_image_difference(
+                ref_image_path, test_image_path, tol=tol)
+
+        return wrapper
+
+    return decorator
+
+
 def _image_directories(func):
     """
     Compute the baseline and result image directories for testing *func*.
diff --git a/lib/matplotlib/tests/test_axes.py b/lib/matplotlib/tests/test_axes.py
@@ -14,7 +14,7 @@
 import warnings
 
 import matplotlib
-from matplotlib.testing.decorators import image_comparison
+from matplotlib.testing.decorators import image_comparison, check_figures_equal
 import matplotlib.pyplot as plt
 import matplotlib.markers as mmarkers
 import matplotlib.patches as mpatches
@@ -5702,18 +5702,11 @@ def test_plot_columns_cycle_deprecation():
         plt.plot(np.zeros((2, 2)), np.zeros((2, 3)))
 
 
-def test_markerfacecolor_none_alpha():
-    fig1, ax1 = plt.subplots()
-    ax1.plot(0, "o", mfc="none", alpha=.5)
-    buf1 = io.BytesIO()
-    fig1.savefig(buf1)
-
-    fig2, ax2 = plt.subplots()
-    ax2.plot(0, "o", mfc="w", alpha=.5)
-    buf2 = io.BytesIO()
-    fig2.savefig(buf2)
-
-    assert buf1.getvalue() == buf2.getvalue()
+# pdf and svg tests fail using travis' old versions of gs and inkscape.
+@check_figures_equal(extensions=["png"])
+def test_markerfacecolor_none_alpha(fig_test, fig_ref):
+    fig_test.subplots().plot(0, "o", mfc="none", alpha=.5)
+    fig_ref.subplots().plot(0, "o", mfc="w", alpha=.5)
 
 
 def test_tick_padding_tightbbox():
diff --git a/tools/triage_tests.py b/tools/triage_tests.py
@@ -192,14 +192,22 @@ def set_large_image(self, index):
         self.thumbnails[self.current_thumbnail].setFrameShape(1)
 
     def accept_test(self):
-        self.entries[self.current_entry].accept()
+        entry = self.entries[self.current_entry]
+        if entry.status == 'autogen':
+            print('Cannot accept autogenerated test cases.')
+            return
+        entry.accept()
         self.filelist.currentItem().setText(
             self.entries[self.current_entry].display)
         # Auto-move to the next entry
         self.set_entry(min((self.current_entry + 1), len(self.entries) - 1))
 
     def reject_test(self):
-        self.entries[self.current_entry].reject()
+        entry = self.entries[self.current_entry]
+        if entry.status == 'autogen':
+            print('Cannot reject autogenerated test cases.')
+            return
+        entry.reject()
         self.filelist.currentItem().setText(
             self.entries[self.current_entry].display)
         # Auto-move to the next entry
@@ -261,11 +269,14 @@ def __init__(self, path, root, source):
             ]
         self.thumbnails = [os.path.join(self.dir, x) for x in self.thumbnails]
 
-        self.status = 'unknown'
-
-        if self.same(os.path.join(self.dir, self.generated),
+        if not Path(self.destdir, self.generated).exists():
+            # This case arises from a check_figures_equal test.
+            self.status = 'autogen'
+        elif self.same(os.path.join(self.dir, self.generated),
                      os.path.join(self.destdir, self.generated)):
             self.status = 'accept'
+        else:
+            self.status = 'unknown'
 
     def same(self, a, b):
         """
@@ -297,16 +308,18 @@ def display(self):
         Get the display string for this entry.  This is the text that
         appears in the list widget.
         """
-        status_map = {'unknown': '\N{BALLOT BOX}',
-                      'accept':  '\N{BALLOT BOX WITH CHECK}',
-                      'reject':  '\N{BALLOT BOX WITH X}'}
+        status_map = {
+            'unknown': '\N{BALLOT BOX}',
+            'accept':  '\N{BALLOT BOX WITH CHECK}',
+            'reject':  '\N{BALLOT BOX WITH X}',
+            'autogen': '\N{WHITE SQUARE CONTAINING BLACK SMALL SQUARE}',
+        }
         box = status_map[self.status]
         return '{} {} [{}]'.format(box, self.name, self.extension)
 
     def accept(self):
         """
-        Accept this test by copying the generated result to the
-        source tree.
+        Accept this test by copying the generated result to the source tree.
         """
         a = os.path.join(self.dir, self.generated)
         b = os.path.join(self.destdir, self.generated)
@@ -315,8 +328,7 @@ def accept(self):
 
     def reject(self):
         """
-        Reject this test by copying the expected result to the
-        source tree.
+        Reject this test by copying the expected result to the source tree.
         """
         a = os.path.join(self.dir, self.expected)
         b = os.path.join(self.destdir, self.generated)
