sphinx-gallery · lesteve · Jul 25, 2016 · Feb 14, 2016 · Feb 14, 2016 · Feb 14, 2016
diff --git a/.travis.yml b/.travis.yml
@@ -56,4 +56,5 @@ script:
     - if [ "$DISTRIB" == "ubuntu" ]; then python setup.py nosetests; fi
     - if [ "$DISTRIB" == "conda" ]; then nosetests; fi
     - cd doc
+    - make html-noplot
     - make html
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -6,11 +6,19 @@ git master
 
 New features
 ''''''''''''
-
+* Summary of failing examples with traceback at the end of the sphinx
+  build. By default the build exits with a 1 exit code if an example
+  has failed. A list of examples that are expected to fail can be
+  defined in `conf.py` and exit the build with 0
+  exit code. Alternatively it is possible to exit the build as soon as
+  one example has failed.
 * Print aggregated and sorted list of computation times of all examples
   in the console during the build.
 * For examples that create multiple figures, set the thumbnail image.
-
+* The ``plot_gallery`` and ``abort_on_example_error`` options can now
+  be specified in ``sphinx_gallery_conf``. The build option (``-D``
+  flag passed to ``sphinx-build``) takes precedence over the
+  ``sphinx_gallery_conf`` option.
 
 v0.1.2
 ------
@@ -41,7 +49,7 @@ Example scripts are now available for download as IPython Notebooks
 `#75 <https://github.com/sphinx-gallery/sphinx-gallery/pull/75>`_
 
 New features
-------------
+''''''''''''
 
 * Configurable filename pattern to select which example scripts are
   executed while building the Gallery

diff --git a/doc/advanced_configuration.rst b/doc/advanced_configuration.rst
@@ -55,11 +55,11 @@ you would do:
     }
 
 Here, one should escape the dot ``'\.'`` as otherwise python `regular expressions`_ matches any character. Nevertheless, as
-one is targetting a specific file, it is most certainly going to match the dot in the filename.
+one is targeting a specific file, it is most certainly going to match the dot in the filename.
 
 Similarly, to build only examples in a specific directory, you can do:
 
-.. code-blocK:: python
+.. code-block:: python
 
     sphinx_gallery_conf = {
         'filename_pattern' : '/directory/plot_'
@@ -251,7 +251,13 @@ your ``Makefile`` with::
         @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
 Remember that for ``Makefile`` white space is significant and the indentation are tabs
-and not spaces
+and not spaces.
+
+Alternatively, you can add the ``plot_gallery`` option to the
+``sphinx_gallery_conf`` dictionary inside your ``conf.py``
+configuration file to have it as a default. The highest precedence is
+always given to the `-D` flag of the ``sphinx-build`` command.
+
 
 Dealing with failing Gallery example scripts
 ============================================
@@ -267,7 +273,17 @@ failing code block. Refer to example
 :ref:`sphx_glr_auto_examples_plot_raise.py` to view the default
 behavior.
 
-An extra functionality of Sphinx-Gallery is the early fail option. In
+The build is also failed exiting with code 1 and giving you a summary
+of the failed examples with their respective traceback. This way you
+are aware of failing examples right after the build and can find them
+easily.
+
+There are some additional options at your hand to deal with broken examples.
+
+Abort build on first fail
+-------------------------
+
+Sphinx-Gallery provides the early fail option. In
 this mode the gallery build process breaks as soon as an exception
 occurs in the execution of the examples scripts. To activate this
 behavior you need to pass a flag at the build process. It can be done
@@ -279,6 +295,34 @@ by including in your ``Makefile``::
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
 Remember that for ``Makefile`` white space is significant and the indentation are tabs
-and not spaces
+and not spaces.
+
+Alternatively, you can add the ``abort_on_example_error`` option to
+the ``sphinx_gallery_conf`` dictionary inside your ``conf.py``
+configuration file to have it as a default. The highest precedence is
+always given to the `-D` flag of the ``sphinx-build`` command.
+
+
+Don't fail the build on exit
+----------------------------
+
+It might be the case that you want to keep the gallery even with
+failed examples. Thus you can configure Sphinx-Gallery to allow
+certain examples to fail and still exit with a 0 exit code. For this
+you need to list all the examples you want to allow to fail during
+build. Change your `conf.py` accordingly:
+
+
+.. code-block:: python
+
+    sphinx_gallery_conf = {
+        ...
+	'expected_failing_examples': ['../examples/plot_raise.py']
+    }
+
+Here you list the examples you allow to fail during the build process,
+keep in mind to specify the full relative path from your `conf.py` to
+the example script.
+
 
 .. _regular expressions: https://docs.python.org/2/library/re.html
diff --git a/doc/conf.py b/doc/conf.py
@@ -32,6 +32,7 @@
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
+    'sphinx.ext.napoleon',
     'sphinx.ext.autosummary',
     'sphinx.ext.doctest',
     'sphinx.ext.intersphinx',
@@ -114,6 +115,7 @@
 # a list of builtin themes.
 html_theme = 'default'
 
+
 def setup(app):
     app.add_stylesheet('theme_override.css')
 
@@ -199,22 +201,22 @@ def setup(app):
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
 
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
 
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'Sphinx-Gallery.tex', u'Sphinx-Gallery Documentation',
-   u'Óscar Nájera', 'manual'),
+    ('index', 'Sphinx-Gallery.tex', u'Sphinx-Gallery Documentation',
+     u'Óscar Nájera', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -257,9 +259,9 @@ def setup(app):
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Sphinx-Gallery', u'Sphinx-Gallery Documentation',
-   u'Óscar Nájera', 'Sphinx-Gallery', 'One line description of project.',
-   'Miscellaneous'),
+    ('index', 'Sphinx-Gallery', u'Sphinx-Gallery Documentation',
+     u'Óscar Nájera', 'Sphinx-Gallery', 'One line description of project.',
+     'Miscellaneous'),
 ]
 
 # Documents to append as an appendix to all manuals.
@@ -304,4 +306,5 @@ def setup(app):
     'examples_dirs': examples_dirs,
     'gallery_dirs': gallery_dirs,
     'find_mayavi_figures': find_mayavi_figures,
-    }
+    'expected_failing_examples': ['../examples/plot_raise.py']
+}
diff --git a/examples/plot_quantum.py b/examples/plot_quantum.py
@@ -60,7 +60,7 @@
 for b in [10, 20, 30]:
     n = 2 * (np.exp(b * (mu - 1)) + np.exp(b * (2 * mu - 3))) / \
         (1 + np.exp(b * (mu - 1)) * (2 + np.exp(b * (mu - 2))))
-    plt.plot(mu, n, label=r"$\beta={}$".format(b))
+    plt.plot(mu, n, label=r"$\beta={0}$".format(b))
 plt.xlabel(r'$\mu$ ($\epsilon=1$, $U=1$)')
 plt.ylabel(r'$\langle N \rangle=\langle n_\uparrow \rangle+\langle n_\downarrow\rangle$')
 plt.legend(loc=0)

diff --git a/examples/plot_raise.py b/examples/plot_raise.py
@@ -10,14 +10,6 @@
 You also get the python traceback of the failed code block
 """
 
-iae
-
-###############################################################################
-# Sphinx gallery as it executes scripts by block will continue
-# evaluating the script after exceptions, but there is no warranty
-# figure ordering will continue to match block's code. Anyway when the
-# script is broken, you should try to fix it first.
-
 # Code source: Óscar Nájera
 # License: BSD 3 clause
 
@@ -26,8 +18,18 @@
 
 plt.pcolormesh(np.random.randn(100, 100))
 
+###############################################################################
+# This next block will raise a NameError
+
+iae
+
+###############################################################################
+# Sphinx gallery will stop executing the remaining code blocks after
+# the exception has occurred in the example script. Nevertheless the
+# html will still render all the example annotated text and
+# code blocks, but no output will be shown.
 
 ###############################################################################
-# Here is another error raising Block
+# Here is another error raising block but will not be executed
 
 plt.plot('Strings are not a valid argument for the plot function')
diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
@@ -12,12 +12,27 @@
 
 
 from __future__ import division, print_function, absolute_import
+import copy
 import re
 import os
 from . import glr_path_static
 from .gen_rst import generate_dir_rst, SPHX_GLR_SIG
 from .docs_resolv import embed_code_links
 
+DEFAULT_GALLERY_CONF = {
+    'filename_pattern': re.escape(os.sep) + 'plot',
+    'examples_dirs': os.path.join('..', 'examples'),
+    'gallery_dirs': 'auto_examples',
+    'mod_example_dir': os.path.join('modules', 'generated'),
+    'doc_module': (),
+    'reference_url': {},
+    # build options
+    'plot_gallery': True,
+    'abort_on_example_error': False,
+    'failing_examples': {},
+    'expected_failing_examples': set(),
+}
+
 
 def clean_gallery_out(build_dir):
     """Deletes images under the sphx_glr namespace in the build directory"""
@@ -55,6 +70,7 @@ def generate_gallery_rst(app):
     except TypeError:
         plot_gallery = bool(app.builder.config.plot_gallery)
 
+    gallery_conf = copy.deepcopy(DEFAULT_GALLERY_CONF)
     gallery_conf.update(app.config.sphinx_gallery_conf)
     gallery_conf.update(plot_gallery=plot_gallery)
     gallery_conf.update(
@@ -143,28 +159,76 @@ def touch_empty_backreferences(app, what, name, obj, options, lines):
         open(examples_path, 'w').close()
 
 
-gallery_conf = {
-    'filename_pattern': re.escape(os.sep) + 'plot',
-    'examples_dirs': '../examples',
-    'gallery_dirs': 'auto_examples',
-    'mod_example_dir': os.path.join('modules', 'generated'),
-    'doc_module': (),
-    'reference_url': {},
-}
+def sumarize_failing_examples(app, exception):
+    """Collects the list of falling examples during build and prints them with the traceback
+
+    Raises ValueError if there where failing examples
+    """
+    if exception is not None:
+        return
+
+    # Under no-plot Examples are not run so nothing to summarize
+    if not app.config.sphinx_gallery_conf['plot_gallery']:
+        return
+
+    gallery_conf = app.config.sphinx_gallery_conf
+    failing_examples = set(gallery_conf['failing_examples'])
+    expected_failing_examples = set(gallery_conf['expected_failing_examples'])
+
+    examples_expected_to_fail = failing_examples.intersection(
+        expected_failing_examples)
+    expected_fail_msg = []
+    if examples_expected_to_fail:
+        expected_fail_msg.append("Examples failing as expected:")
+        for fail_example in examples_expected_to_fail:
+            expected_fail_msg.append(fail_example + ' failed leaving traceback:\n' +
+                                     gallery_conf['failing_examples'][fail_example] + '\n')
+        print("\n".join(expected_fail_msg))
+
+    examples_not_expected_to_fail = failing_examples.difference(
+        expected_failing_examples)
+    fail_msgs = []
+    if examples_not_expected_to_fail:
+        fail_msgs.append("Unexpected failing examples:")
+        for fail_example in examples_not_expected_to_fail:
+            fail_msgs.append(fail_example + ' failed leaving traceback:\n' +
+                             gallery_conf['failing_examples'][fail_example] + '\n')
+
+    examples_not_expected_to_pass = expected_failing_examples.difference(
+        failing_examples)
+    if examples_not_expected_to_pass:
+        fail_msgs.append("Examples expected to fail, but not failling:\n" +
+                         "Please remove this examples from\n" +
+                         "sphinx_gallery_conf['expected_failing_examples']\n" +
+                         "in your conf.py file"
+                         "\n".join(examples_not_expected_to_pass))
+
+    if fail_msgs:
+        raise ValueError("Here is a summary of the problems encountered when "
+                         "running the examples\n\n" + "\n".join(fail_msgs) +
+                         "\n" + "-" * 79)
+
+
+def get_default_config_value(key):
+    def default_getter(conf):
+        return conf['sphinx_gallery_conf'].get(key, DEFAULT_GALLERY_CONF[key])
+    return default_getter
 
 
 def setup(app):
     """Setup sphinx-gallery sphinx extension"""
-    app.add_config_value('plot_gallery', True, 'html')
-    app.add_config_value('abort_on_example_error', False, 'html')
-    app.add_config_value('sphinx_gallery_conf', gallery_conf, 'html')
+    app.add_config_value('sphinx_gallery_conf', DEFAULT_GALLERY_CONF, 'html')
+    for key in ['plot_gallery', 'abort_on_example_error']:
+        app.add_config_value(key, get_default_config_value(key), 'html')
+
     app.add_stylesheet('gallery.css')
 
     if 'sphinx.ext.autodoc' in app._extensions:
         app.connect('autodoc-process-docstring', touch_empty_backreferences)
 
     app.connect('builder-inited', generate_gallery_rst)
 
+    app.connect('build-finished', sumarize_failing_examples)
     app.connect('build-finished', embed_code_links)