sphinx-gallery · lucyleeow · Jul 10, 2024 · Oct 11, 2022 · Dec 2, 2022 · Jun 4, 2024
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -44,7 +44,7 @@ jobs:
                 "git+https://github.com/pyvista/pyvista" \
                 "docutils>=0.18" imageio pydata-sphinx-theme \
                 "jupyterlite-sphinx>=0.8.0,<0.9.0" "jupyterlite-pyodide-kernel<0.1.0" \
-                libarchive-c "sphinxcontrib-video>=0.2.1rc0"
+                libarchive-c "sphinxcontrib-video>=0.2.1rc0" intersphinx_registry
             pip uninstall -yq vtk  # pyvista installs vtk above
             pip install --upgrade --only-binary ":all" --extra-index-url https://wheels.vtk.org vtk-osmesa
       - save_cache:

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -32,10 +32,6 @@ jobs:
             sphinx_version: dev
             distrib: pip
             locale: C
-          - os: ubuntu-latest  # oldest supported Python and Sphinx
-            python: '3.8'
-            sphinx_version: '4'
-            distrib: mamba
           - os: ubuntu-latest
             python: '3.11'
             sphinx_version: '5'
@@ -79,7 +75,7 @@ jobs:
             python=${{ env.PYTHON_VERSION }} pip numpy setuptools matplotlib pillow
             pytest pytest-cov coverage seaborn statsmodels plotly joblib wheel libiconv
             pygraphviz memory_profiler ipython pypandoc lxml conda-libmamba-solver mamba
-            ffmpeg
+            ffmpeg intersphinx-registry
         if: matrix.distrib == 'mamba'
       # Make sure that things work even if the locale is set to C (which
       # effectively means ASCII). Some of the input rst files have unicode

diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,6 +1,12 @@
 Changelog
 =========
 
+v0.17.0
+-------
+
+Support for Python 3.8 and Sphinx 4 dropped in this release.
+Requirement is now Python >= 3.9 and Sphinx >= 5.
+
 v0.16.0
 -------
 Sphinx 7.3.0 and above changed caching and serialization checks. Now instead of passing

diff --git a/doc/conf.py b/doc/conf.py
@@ -17,6 +17,8 @@
 import warnings
 from datetime import date
 
+from intersphinx_registry import get_intersphinx_mapping
+
 import sphinx_gallery
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -332,15 +334,18 @@ def setup(app):
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {
-    "python": (f"https://docs.python.org/{sys.version_info.major}", None),
-    "numpy": ("https://numpy.org/doc/stable/", None),
-    "matplotlib": ("https://matplotlib.org/stable", None),
-    "pyvista": ("https://docs.pyvista.org/version/stable", None),
-    "sklearn": ("https://scikit-learn.org/stable", None),
-    "sphinx": ("https://www.sphinx-doc.org/en/master", None),
-    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
-}
+intersphinx_mapping = get_intersphinx_mapping(
+    packages={
+        "joblib",
+        "matplotlib",
+        "numpy",
+        "pandas",
+        "python",
+        "pyvista",
+        "sklearn",
+        "sphinx",
+    },
+)
 
 examples_dirs = ["../examples", "../tutorials"]
 gallery_dirs = ["auto_examples", "tutorials"]
@@ -352,30 +357,19 @@ def setup(app):
     # installed
     import pyvista
 except Exception:  # can raise all sorts of errors
-    pass
+    pyvista = None
 else:
     image_scrapers += ("pyvista",)
     examples_dirs.append("../pyvista_examples")
     gallery_dirs.append("auto_pyvista_examples")
-    pyvista.OFF_SCREEN = True
-    # Preferred plotting style for documentation
-    pyvista.set_plot_theme("document")
-    pyvista.global_theme.window_size = [1024, 768]
-    pyvista.global_theme.font.size = 22
-    pyvista.global_theme.font.label_size = 22
-    pyvista.global_theme.font.title_size = 22
-    pyvista.global_theme.return_cpos = False
-    # necessary when building the sphinx gallery
-    pyvista.BUILDING_GALLERY = True
-    pyvista.set_jupyter_backend(None)
 
 # Set plotly renderer to capture _repr_html_ for sphinx-gallery
 try:
+    import plotly
     import plotly.io
 except ImportError:
-    pass
+    plotly = None
 else:
-    plotly.io.renderers.default = "sphinx_gallery"
     examples_dirs.append("../plotly_examples")
     gallery_dirs.append("auto_plotly_examples")
 
@@ -393,6 +387,7 @@ def setup(app):
     "examples_dirs": examples_dirs,
     "gallery_dirs": gallery_dirs,
     "image_scrapers": image_scrapers,
+    "reset_modules": ("matplotlib", "seaborn", "sg_doc_build.reset_others"),
     "compress_images": ("images", "thumbnails"),
     # specify the order of examples to be according to filename
     "within_subsection_order": "FileNameSortKey",

diff --git a/doc/configuration.rst b/doc/configuration.rst
@@ -44,6 +44,7 @@ file, inside a ``sphinx_gallery_conf`` dictionary.
 - ``abort_on_example_error`` (:ref:`abort_on_first`)
 - ``expected_failing_examples`` (:ref:`dont_fail_exit`)
 - ``only_warn_on_example_error`` (:ref:`warning_on_error`)
+- ``parallel`` (:ref:`parallel`)
 
 **Cross-referencing**
 
@@ -2093,6 +2094,35 @@ flag is passed to ``sphinx-build``. This can be enabled by setting::
     }
 
 
+.. _parallel:
+
+Build examples in parallel
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Sphinx-Gallery can be configured to run examples simultaneously using
+:mod:`joblib`. This can be enabled by setting::
+
+    sphinx_gallery_conf = {
+        ...
+        'parallel': 2,
+    }
+
+If an ``int``, then that number of jobs will be passed to :class:`joblib.Parallel`.
+If ``True``, then the same number of jobs will be used as the ``-j`` flag for
+Sphinx.
+
+.. warning::
+    Some packages might not play nicely with parallel processing, so this feature
+    is considered **experimental**!
+
+    For example, you might need to set variables or call functions in a
+    :ref:`custom resetter <custom_reset>` to ensure that all spawned processes are
+    properly set up and torn down. Parallelism is achieved through the Loky backend of
+    joblib, see :ref:`joblib:parallel` for documentation of many relevant conisderations
+    (e.g., pickling, oversubscription of CPU resources, etc.).
+
+    Using parallel building will also disable memory measurements.
+
 .. _recommend_examples:
 
 Enabling the example recommender system

diff --git a/doc/getting_started.rst b/doc/getting_started.rst
@@ -178,7 +178,7 @@ generated:
 * ``.py`` - to enable the user to download a ``.py`` version of the example.
 * ``.py.md5`` - a md5 hash of the ``.py`` file, used to determine if changes
   have been made to the file and thus if new output files need to be generated.
-* ``_codeobj.pickle`` - used to identify function names and to which module
+* ``.codeobj.json`` - used to identify function names and to which module
   they belong (more details in
   :ref:`sphx_glr_auto_examples_plot_6_function_identifier.py`)
 

diff --git a/doc/sphinxext/sg_doc_build.py b/doc/sphinxext/sg_doc_build.py
@@ -56,3 +56,29 @@ def notebook_modification_function(notebook_content, notebook_filename):
     notebook_content["cells"] = (
         dummy_notebook_content["cells"] + notebook_content["cells"]
     )
+
+
+def reset_others(gallery_conf, fname):
+    """Reset plotting functions."""
+    try:
+        import pyvista
+    except Exception:
+        pass
+    else:
+        pyvista.OFF_SCREEN = True
+        # Preferred plotting style for documentation
+        pyvista.set_plot_theme("document")
+        pyvista.global_theme.window_size = [1024, 768]
+        pyvista.global_theme.font.size = 22
+        pyvista.global_theme.font.label_size = 22
+        pyvista.global_theme.font.title_size = 22
+        pyvista.global_theme.return_cpos = False
+        # necessary when building the sphinx gallery
+        pyvista.BUILDING_GALLERY = True
+        pyvista.set_jupyter_backend(None)
+    try:
+        import plotly.io
+    except Exception:
+        pass
+    else:
+        plotly.io.renderers.default = "sphinx_gallery"
diff --git a/pyproject.toml b/pyproject.toml
@@ -31,14 +31,15 @@ dynamic = [
 ]
 dependencies = [
   "pillow",
-  "sphinx>=4",
+  "sphinx>=5",
 ]
 optional-dependencies.animations = [
   "sphinxcontrib-video",
 ]
 optional-dependencies.dev = [
   "absl-py",
   "graphviz",
+  "intersphinx-registry",
   "ipython",
   "joblib",
   "jupyterlite-sphinx",
@@ -57,6 +58,9 @@ optional-dependencies.dev = [
 optional-dependencies.jupyterlite = [
   "jupyterlite-sphinx",
 ]
+optional-dependencies.parallel = [
+  "joblib",
+]
 optional-dependencies.recommender = [
   "numpy",
 ]

diff --git a/sphinx_gallery/backreferences.py b/sphinx_gallery/backreferences.py
@@ -6,8 +6,6 @@
 """
 
 import ast
-import codecs
-import collections
 import inspect
 import os
 import re
@@ -19,7 +17,7 @@
 from sphinx.errors import ExtensionError
 
 from .scrapers import _find_image_ext
-from .utils import _replace_md5
+from .utils import _W_KW, _replace_md5
 
 THUMBNAIL_PARENT_DIV = """
 .. raw:: html
@@ -249,9 +247,9 @@ def identify_names(script_blocks, ref_regex, global_variables=None, node=""):
 
     Returns
     -------
-    example_code_obj : OrderedDict[str, Any]
-        OrderedDict with information about all code object references found in an
-        example. OrderedDict contains the following keys:
+    example_code_obj : Dict[str, Any]
+        Dict with information about all code object references found in an
+        example. Dict contains the following keys:
 
             - example_code_obj['name'] : function or class name (str)
             - example_code_obj['module'] : module name (str)
@@ -271,7 +269,7 @@ def identify_names(script_blocks, ref_regex, global_variables=None, node=""):
     # Get matches from docstring inspection (explicit matches)
     text = "\n".join(block.content for block in script_blocks if block.type == "text")
     names.extend((x, x, False, False, True) for x in re.findall(ref_regex, text))
-    example_code_obj = collections.OrderedDict()  # order is important
+    example_code_obj = dict()  # native dict preserves order nowadays
     # Make a list of all guesses, in `_embed_code_links` we will break
     # when we find a match
     for name, full_name, class_like, is_class, is_explicit in names:
@@ -292,13 +290,13 @@ def identify_names(script_blocks, ref_regex, global_variables=None, node=""):
 
         # get shortened module name
         module_short = _get_short_module_name(module, attribute)
-        cobj = {
-            "name": attribute,
-            "module": module,
-            "module_short": module_short or module,
-            "is_class": is_class,
-            "is_explicit": is_explicit,
-        }
+        cobj = dict(
+            name=attribute,
+            module=module,
+            module_short=module_short or module,
+            is_class=is_class,
+            is_explicit=is_explicit,
+        )
         example_code_obj[name].append(cobj)
     return example_code_obj
 
@@ -391,9 +389,8 @@ def _write_backreferences(
             f"{backref}.examples.new",
         )
         seen = backref in seen_backrefs
-        with codecs.open(
-            include_path, "a" if seen else "w", encoding="utf-8"
-        ) as ex_file:
+        mode = "a" if seen else "w"
+        with open(include_path, mode, **_W_KW) as ex_file:
             if not seen:
                 # Be aware that if the number of lines of this heading changes,
                 #   the minigallery directive should be modified accordingly
@@ -432,7 +429,7 @@ def _finalize_backreferences(seen_backrefs, gallery_conf):
         if os.path.isfile(path):
             # Close div containing all thumbnails
             # (it was open in _write_backreferences)
-            with codecs.open(path, "a", encoding="utf-8") as ex_file:
+            with open(path, "a", **_W_KW) as ex_file:
                 ex_file.write(THUMBNAIL_PARENT_DIV_CLOSE)
             _replace_md5(path, mode="t")
         else: