diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py
index a026eb8c02ea..f2e9340034ee 100644
--- a/lib/matplotlib/testing/decorators.py
+++ b/lib/matplotlib/testing/decorators.py
@@ -281,12 +281,7 @@ def copy_baseline(self, baseline, extension):
             reason = ("Do not have baseline image {0} because this "
                       "file does not exist: {1}".format(expected_fname,
                                                         orig_expected_fname))
-            if is_called_from_pytest():
-                import pytest
-                pytest.xfail(reason)
-            else:
-                from ._nose import knownfail
-                knownfail(reason)
+            raise ImageComparisonFailure(reason)
         return expected_fname
 
     def compare(self, idx, baseline, extension):