testing: add support for KnownFailures

astraw · astraw · commit 43791657afa3 · 2009-08-30T04:44:28.000Z
svn path=/trunk/matplotlib/; revision=7595
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,7 @@
+2009-08-29 Added matplotlib.testing package, which contains a Nose
+           plugin and a decorator that lets tests be marked as
+           KnownFailures - ADS
+
 2009-08-20 Added scaled dict to AutoDateFormatter for customized
            scales - JDH
 
diff --git a/lib/matplotlib/testing/__init__.py b/lib/matplotlib/testing/__init__.py
diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py
@@ -0,0 +1,24 @@
+from matplotlib.testing.noseclasses import KnownFailureTest
+import sys
+
+def knownfailureif(fail_condition, msg=None):
+    # based on numpy.testing.dec.knownfailureif
+    if msg is None:
+        msg = 'Test known to fail'
+    def known_fail_decorator(f):
+        # Local import to avoid a hard nose dependency and only incur the
+        # import time overhead at actual test-time.
+        import nose
+        def failer(*args, **kwargs):
+            try:
+                # Always run the test (to generate images).
+                result = f(*args, **kwargs)
+            except:
+                if fail_condition:
+                    raise KnownFailureTest(msg)
+                else:
+                    raise
+            # Fixme: Should raise KnownFailureDidNotFail if fail_condition==True?
+            return result
+        return nose.tools.make_decorator(f)(failer)
+    return known_fail_decorator
diff --git a/lib/matplotlib/testing/noseclasses.py b/lib/matplotlib/testing/noseclasses.py
@@ -0,0 +1,41 @@
+import os
+from nose.plugins.errorclass import ErrorClass, ErrorClassPlugin
+
+class KnownFailureTest(Exception):
+    '''Raise this exception to mark a test as a known failing test.'''
+    pass
+
+class KnownFailure(ErrorClassPlugin):
+    '''Plugin that installs a KNOWNFAIL error class for the
+    KnownFailureClass exception.  When KnownFailureTest is raised,
+    the exception will be logged in the knownfail attribute of the
+    result, 'K' or 'KNOWNFAIL' (verbose) will be output, and the
+    exception will not be counted as an error or failure.
+
+    This is based on numpy.testing.noseclasses.KnownFailure.
+    '''
+    enabled = True
+    knownfail = ErrorClass(KnownFailureTest,
+                           label='KNOWNFAIL',
+                           isfailure=False)
+
+    def options(self, parser, env=os.environ):
+        env_opt = 'NOSE_WITHOUT_KNOWNFAIL'
+        parser.add_option('--no-knownfail', action='store_true',
+                          dest='noKnownFail', default=env.get(env_opt, False),
+                          help='Disable special handling of KnownFailureTest '
+                               'exceptions')
+
+    def configure(self, options, conf):
+        if not self.can_configure:
+            return
+        self.conf = conf
+        disable = getattr(options, 'noKnownFail', False)
+        if disable:
+            self.enabled = False
+
+    def addError( self, test, err ):
+        # Fixme (Really weird): if I don't leave empty method here,
+        # nose gets confused and KnownFails become testing errors when
+        # using the MplNosePlugin and MplTestCase.
+        pass
diff --git a/setup.py b/setup.py
@@ -50,6 +50,7 @@
     'matplotlib',
     'matplotlib.backends',
     'matplotlib.projections',
+    'matplotlib.testing',
 #   'matplotlib.toolkits',
     'mpl_toolkits',
     'mpl_toolkits.mplot3d',
diff --git a/test/mplTest/MplNosePlugin.py b/test/mplTest/MplNosePlugin.py
@@ -12,6 +12,7 @@
 from path_utils import *
 import directories as dirs
 from MplTestCase import MplTestCase
+from matplotlib.testing.noseclasses import KnownFailureTest
 
 #=======================================================================
 
@@ -48,7 +49,8 @@ class MplNosePlugin( Plugin ):
 
    TEST_ERRORED = -1
    TEST_FAILED = 0
-   TEST_PASSED = 1
+   TEST_KNOWN_FAILED = 1
+   TEST_PASSED = 2
 
    #--------------------------------------------------------------------
    # Some 'property' functions
@@ -148,7 +150,11 @@ def addError( self, test, err ):
          err : 3-tuple
                sys.exc_info() tuple
       """
-      self.testResults.append( (test, self.TEST_ERRORED, err) )
+      (type, value, traceback) = err
+      if isinstance(value,KnownFailureTest):
+         self.testResults.append( (test, self.TEST_KNOWN_FAILED, err) )
+      else:
+         self.testResults.append( (test, self.TEST_ERRORED, err) )
 
    #--------------------------------------------------------------------
    def addFailure( self, test, err ):
diff --git a/test/run-mpl-test.py b/test/run-mpl-test.py
@@ -44,6 +44,7 @@
 
 import nose
 from mplTest import MplNosePlugin, path_utils
+from matplotlib.testing.noseclasses import KnownFailure
 
 if '--clean' in args:
    # perform the cleaning process and exit
@@ -90,7 +91,7 @@
 
 ### Run nose
 success = nose.run( argv = args,
-                    plugins = [ MplNosePlugin() ] )
+                    plugins = [ MplNosePlugin(), KnownFailure() ] )
 
 ### do other stuff here
 
diff --git a/test/test_matplotlib/TestAxes.py b/test/test_matplotlib/TestAxes.py
@@ -3,6 +3,7 @@
 #=======================================================================
 
 from mplTest import MplTestCase, units
+from matplotlib.testing.decorators import knownfailureif
 
 #=======================================================================
 # Add import modules below.
@@ -57,6 +58,7 @@ def test_empty_datetime( self ):
       self.checkImage( fname )
 
    #--------------------------------------------------------------------
+   @knownfailureif(True, "Fails due to SF bug 2846058")
    def test_formatter_ticker( self ):
       """Test Some formatter and ticker issues."""
 
