Merge pull request #8346 from QuLogic/pytest-plugins

phobson · web-flow · commit 405e595234aa · 2017-04-29T06:47:23.000-07:00
TST: Use some more pytest plugins: warnings &amp; rerunfailures
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -14,7 +14,7 @@ environment:
     CMD_IN_ENV: cmd /E:ON /V:ON /C obvci_appveyor_python_build_env.cmd
     # Workaround for https://github.com/conda/conda-build/issues/636
     PYTHONIOENCODING: UTF-8
-    PYTEST_ARGS: -ra --timeout=300 --durations=25 -n %NUMBER_OF_PROCESSORS% --cov-report= --cov=lib -m "not network"
+    PYTEST_ARGS: -rawR --timeout=300 --durations=25 -n %NUMBER_OF_PROCESSORS% --cov-report= --cov=lib -m "not network"
     PYTHONHASHSEED: 0  # Workaround for pytest-xdist flaky collection order
                        # https://github.com/pytest-dev/pytest/issues/920
                        # https://github.com/pytest-dev/pytest/issues/1075
@@ -94,7 +94,7 @@ install:
   - echo %PYTHON_VERSION% %TARGET_ARCH%
   - if %PYTHON_VERSION% == 2.7 conda install -q backports.functools_lru_cache
   # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124
-  - conda install -q pytest "pytest-cov>=2.3.1" pytest-timeout pytest-xdist
+  - pip install -q pytest "pytest-cov>=2.3.1" pytest-rerunfailures pytest-timeout pytest-warnings pytest-xdist
 
   # Let the install prefer the static builds of the libs
   - set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib
diff --git a/.travis.yml b/.travis.yml
@@ -46,7 +46,7 @@ env:
     - INSTALL_PEP8=
     - RUN_PEP8=
     - NOSE=
-    - PYTEST_ARGS="-ra --maxfail=1 --timeout=300 --durations=25 --cov-report= --cov=lib -n $NPROC"
+    - PYTEST_ARGS="-rawR --maxfail=1 --timeout=300 --durations=25 --cov-report= --cov=lib -n $NPROC"
     - PYTHON_ARGS=
     - DELETE_FONT_CACHE=
 
@@ -119,7 +119,7 @@ install:
     pip install $PRE -r doc-requirements.txt
 
     # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124
-    pip install $PRE pytest 'pytest-cov>=2.3.1' pytest-timeout pytest-xdist pytest-faulthandler $INSTALL_PEP8
+    pip install $PRE pytest 'pytest-cov>=2.3.1' pytest-faulthandler pytest-rerunfailures pytest-timeout pytest-warnings pytest-xdist $INSTALL_PEP8
 
     # We manually install humor sans using the package from Ubuntu 14.10. Unfortunatly humor sans is not
     # availible in the Ubuntu version used by Travis but we can manually install the deb from a later
@@ -142,7 +142,7 @@ install:
     # Install matplotlib
     pip install -ve .
 
-script: source ci/travis/test_script.sh
+script: ci/travis/test_script.sh
 
 before_cache:
   - rm -rf $HOME/.cache/matplotlib/tex.cache
diff --git a/ci/travis/test_script.sh b/ci/travis/test_script.sh
@@ -1,5 +1,7 @@
 #! /bin/bash
 
+set -e
+
 # This script is meant to be called by the "script" step defined in
 # .travis.yml. See http://docs.travis-ci.com/ for more details.
 # The behavior of the script is controlled by environment variabled defined
@@ -34,7 +36,6 @@ else
   # We don't build the LaTeX docs here, so linkchecker will complain
   touch build/html/Matplotlib.pdf
   # Linkchecker only works with python 2.7 for the time being
-  deactivate
   source ~/virtualenv/python2.7/bin/activate
   pip install pip --upgrade
   # linkchecker is currently broken with requests 2.10.0 so force an earlier version
diff --git a/lib/matplotlib/tests/test_lines.py b/lib/matplotlib/tests/test_lines.py
@@ -16,6 +16,9 @@
 from matplotlib.testing.decorators import image_comparison
 
 
+# Runtimes on a loaded system are inherently flaky. Not so much that a rerun
+# won't help, hopefully.
+@pytest.mark.flaky(reruns=3)
 def test_invisible_Line_rendering():
     """
     Github issue #1256 identified a bug in Line.draw method
diff --git a/lib/matplotlib/tests/test_mathtext.py b/lib/matplotlib/tests/test_mathtext.py
@@ -167,6 +167,9 @@ def baseline_images(request, fontset, index):
     return ['%s_%s_%02d' % (request.param, fontset, index)]
 
 
+# See #7911 for why these tests are flaky and #7107 for why they are not so
+# easy to fix.
+@pytest.mark.flaky(reruns=3)
 @pytest.mark.parametrize('index, test', enumerate(math_tests),
                          ids=[str(index) for index in range(len(math_tests))])
 @pytest.mark.parametrize('fontset',
@@ -181,6 +184,9 @@ def test_mathtext_rendering(baseline_images, fontset, index, test):
              horizontalalignment='center', verticalalignment='center')
 
 
+# See #7911 for why these tests are flaky and #7107 for why they are not so
+# easy to fix.
+@pytest.mark.flaky(reruns=3)
 @pytest.mark.parametrize('index, test', enumerate(font_tests),
                          ids=[str(index) for index in range(len(font_tests))])
 @pytest.mark.parametrize('fontset',
diff --git a/lib/mpl_toolkits/tests/test_mplot3d.py b/lib/mpl_toolkits/tests/test_mplot3d.py
@@ -108,7 +108,10 @@ def test_lines3d():
     ax.plot(x, y, z)
 
 
-@image_comparison(baseline_images=['mixedsubplot'], remove_text=True)
+# Reason for flakiness of SVG test is still unknown.
+@image_comparison(baseline_images=['mixedsubplot'], remove_text=True,
+                  extensions=['png', 'pdf',
+                              pytest.mark.xfail('svg', strict=False)])
 def test_mixedsubplots():
     def f(t):
         s1 = np.cos(2*np.pi*t)
