From d46991faa630cb9ff3185dd7266e6158dbfa0166 Mon Sep 17 00:00:00 2001
From: Julien Schueller <schueller@phimeca.com>
Date: Tue, 11 Oct 2022 16:29:29 +0200
Subject: [PATCH 01/40] Try at parallel gallery generation

---
 sphinx_gallery/gen_rst.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 6c2dd0589..fe4ef1c04 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -32,6 +32,7 @@
 import sys
 import traceback
 import codeop
+from concurrent.futures import ProcessPoolExecutor, as_completed
 
 from sphinx.errors import ExtensionError
 import sphinx.util
@@ -427,20 +428,23 @@ def generate_dir_rst(
         sorted_listdir,
         'generating gallery for %s... ' % build_target_dir,
         length=len(sorted_listdir))
-    for fname in iterator:
-        intro, title, cost = generate_file_rst(
-            fname, target_dir, src_dir, gallery_conf, seen_backrefs)
-        src_file = os.path.normpath(os.path.join(src_dir, fname))
-        costs.append((cost, src_file))
-        gallery_item_filename = os.path.join(
-            build_target_dir,
-            fname[:-3]
-        ).replace(os.sep, '/')
-        this_entry = _thumbnail_div(
-            target_dir, gallery_conf['src_dir'], fname, intro, title
-        )
-        entries_text.append(this_entry)
-        subsection_toctree_filenames.append("/" + gallery_item_filename)
+
+    with ProcessPoolExecutor(max_workers=8) as executor:
+        future_gen_file = {executor.submit(generate_file_rst, *(fname, target_dir, src_dir, gallery_conf, seen_backrefs)): fname for fname in sorted_listdir}
+        for future in as_completed(future_gen_file):
+            fname = future_gen_file[future][0]
+            intro, title, cost = future.result()
+            src_file = os.path.normpath(os.path.join(src_dir, fname))
+            costs.append((cost, src_file))
+            gallery_item_filename = os.path.join(
+                build_target_dir,
+                fname[:-3]
+            ).replace(os.sep, '/')
+            this_entry = _thumbnail_div(
+                target_dir, gallery_conf['src_dir'], fname, intro, title
+            )
+            entries_text.append(this_entry)
+            subsection_toctree_filenames.append("/" + gallery_item_filename)
 
     for entry_text in entries_text:
         subsection_index_content += entry_text

From 62dd2e4e23729ba2c0ca38b53a7cfb4d647bb701 Mon Sep 17 00:00:00 2001
From: Julien Schueller <schueller@phimeca.com>
Date: Fri, 2 Dec 2022 18:21:01 +0100
Subject: [PATCH 02/40] pathos

---
 sphinx_gallery/gen_gallery.py |  1 +
 sphinx_gallery/gen_rst.py     | 43 +++++++++++++++++++++--------------
 2 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
index 7b6f9a439..a6c0bfcfa 100644
--- a/sphinx_gallery/gen_gallery.py
+++ b/sphinx_gallery/gen_gallery.py
@@ -104,6 +104,7 @@ def __call__(self, gallery_conf, script_vars):
     'prefer_full_module': [],
     'api_usage_ignore': '.*__.*__',
     'show_api_usage': False,
+    'parallel': False,
 }
 
 logger = sphinx.util.logging.getLogger('sphinx-gallery')
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index fe4ef1c04..84bee4542 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -32,7 +32,6 @@
 import sys
 import traceback
 import codeop
-from concurrent.futures import ProcessPoolExecutor, as_completed
 
 from sphinx.errors import ExtensionError
 import sphinx.util
@@ -429,22 +428,32 @@ def generate_dir_rst(
         'generating gallery for %s... ' % build_target_dir,
         length=len(sorted_listdir))
 
-    with ProcessPoolExecutor(max_workers=8) as executor:
-        future_gen_file = {executor.submit(generate_file_rst, *(fname, target_dir, src_dir, gallery_conf, seen_backrefs)): fname for fname in sorted_listdir}
-        for future in as_completed(future_gen_file):
-            fname = future_gen_file[future][0]
-            intro, title, cost = future.result()
-            src_file = os.path.normpath(os.path.join(src_dir, fname))
-            costs.append((cost, src_file))
-            gallery_item_filename = os.path.join(
-                build_target_dir,
-                fname[:-3]
-            ).replace(os.sep, '/')
-            this_entry = _thumbnail_div(
-                target_dir, gallery_conf['src_dir'], fname, intro, title
-            )
-            entries_text.append(this_entry)
-            subsection_toctree_filenames.append("/" + gallery_item_filename)
+    try:
+        from pathos.pools import ProcessPool
+        has_pathos = True
+    except ImportError:
+        has_pathos = False
+    if gallery_conf['parallel'] and has_pathos:
+        pool = ProcessPool()
+        inputs = [(fname, target_dir, src_dir, gallery_conf, seen_backrefs) for fname in iterator]
+        results = pool.map(lambda x:generate_file_rst(*x), inputs)
+    else:
+        results = [generate_file_rst(*(fname, target_dir, src_dir, gallery_conf, seen_backrefs)) for fname in iterator]
+
+    for i in range(len(results)):
+        fname = sorted_listdir[i]
+        intro, title, cost = results[i]
+        src_file = os.path.normpath(os.path.join(src_dir, fname))
+        costs.append((cost, src_file))
+        gallery_item_filename = os.path.join(
+            build_target_dir,
+            fname[:-3]
+        ).replace(os.sep, '/')
+        this_entry = _thumbnail_div(
+            target_dir, gallery_conf['src_dir'], fname, intro, title
+        )
+        entries_text.append(this_entry)
+        subsection_toctree_filenames.append("/" + gallery_item_filename)
 
     for entry_text in entries_text:
         subsection_index_content += entry_text

From 01023a3ae26fc6ef561a062398efefe1fb046de0 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Tue, 4 Jun 2024 15:25:18 -0700
Subject: [PATCH 03/40] FIX: Many fixes

---
 .circleci/config.yml                       |  2 +-
 doc/conf.py                                | 66 +++++++++++++---------
 doc/configuration.rst                      | 23 ++++++++
 sphinx_gallery/gen_gallery.py              | 21 ++++++-
 sphinx_gallery/gen_rst.py                  | 65 +++++++++++++--------
 sphinx_gallery/tests/test_full.py          |  1 +
 sphinx_gallery/tests/test_gen_rst.py       | 13 ++++-
 sphinx_gallery/tests/tinybuild/doc/conf.py |  1 +
 8 files changed, 138 insertions(+), 54 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index aa11a4cc8..3a2c99a3d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -70,7 +70,7 @@ jobs:
       - attach_workspace:
           at: ~/
       - bash_env
-      - run: sphinx-build doc doc/_build/html -nW --keep-going -b html 2>&1 | tee sphinx_log.txt
+      - run: sphinx-build doc doc/_build/html -nW --keep-going -b html -j2 2>&1 | tee sphinx_log.txt
       - run:
           name: Check sphinx log for warnings (which are treated as errors)
           when: always
diff --git a/doc/conf.py b/doc/conf.py
index 072b18074..b91671163 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -17,6 +17,8 @@
 from datetime import date
 import warnings
 
+from intersphinx_registry import get_intersphinx_mapping
+
 import sphinx_gallery
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -332,15 +334,18 @@ def setup(app):
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {
-    "python": (f"https://docs.python.org/{sys.version_info.major}", None),
-    "numpy": ("https://numpy.org/doc/stable/", None),
-    "matplotlib": ("https://matplotlib.org/stable", None),
-    "pyvista": ("https://docs.pyvista.org/version/stable", None),
-    "sklearn": ("https://scikit-learn.org/stable", None),
-    "sphinx": ("https://www.sphinx-doc.org/en/master", None),
-    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
-}
+intersphinx_mapping = get_intersphinx_mapping(
+    packages={
+        "joblib",
+        "matplotlib",
+        "numpy",
+        "pandas",
+        "python",
+        "pyvista",
+        "sklearn",
+        "sphinx",
+    },
+)
 
 examples_dirs = ["../examples", "../tutorials"]
 gallery_dirs = ["auto_examples", "tutorials"]
@@ -352,32 +357,39 @@ def setup(app):
     # installed
     import pyvista
 except Exception:  # can raise all sorts of errors
-    pass
+    pyvista = None
 else:
     image_scrapers += ("pyvista",)
     examples_dirs.append("../pyvista_examples")
     gallery_dirs.append("auto_pyvista_examples")
-    pyvista.OFF_SCREEN = True
-    # Preferred plotting style for documentation
-    pyvista.set_plot_theme("document")
-    pyvista.global_theme.window_size = [1024, 768]
-    pyvista.global_theme.font.size = 22
-    pyvista.global_theme.font.label_size = 22
-    pyvista.global_theme.font.title_size = 22
-    pyvista.global_theme.return_cpos = False
-    # necessary when building the sphinx gallery
-    pyvista.BUILDING_GALLERY = True
-    pyvista.set_jupyter_backend(None)
 
 # Set plotly renderer to capture _repr_html_ for sphinx-gallery
 try:
+    import plotly
     import plotly.io
 except ImportError:
-    pass
-else:
-    plotly.io.renderers.default = "sphinx_gallery"
-    examples_dirs.append("../plotly_examples")
-    gallery_dirs.append("auto_plotly_examples")
+    plotly = None
+
+
+def reset_others(gallery_conf, fname):
+    """Reset plotting functions."""
+    if pyvista is not None:
+        pyvista.OFF_SCREEN = True
+        # Preferred plotting style for documentation
+        pyvista.set_plot_theme("document")
+        pyvista.global_theme.window_size = [1024, 768]
+        pyvista.global_theme.font.size = 22
+        pyvista.global_theme.font.label_size = 22
+        pyvista.global_theme.font.title_size = 22
+        pyvista.global_theme.return_cpos = False
+        # necessary when building the sphinx gallery
+        pyvista.BUILDING_GALLERY = True
+        pyvista.set_jupyter_backend(None)
+    if plotly is not None:
+        plotly.io.renderers.default = "sphinx_gallery"
+        examples_dirs.append("../plotly_examples")
+        gallery_dirs.append("auto_plotly_examples")
+
 
 min_reported_time = 0
 if "SOURCE_DATE_EPOCH" in os.environ:
@@ -393,6 +405,7 @@ def setup(app):
     "examples_dirs": examples_dirs,
     "gallery_dirs": gallery_dirs,
     "image_scrapers": image_scrapers,
+    "reset_modules": ("matplotlib", "seaborn", reset_others),
     "compress_images": ("images", "thumbnails"),
     # specify the order of examples to be according to filename
     "within_subsection_order": "FileNameSortKey",
@@ -423,6 +436,7 @@ def setup(app):
     "image_srcset": ["2x"],
     "nested_sections": True,
     "show_api_usage": True,
+    "parallel": True,
 }
 
 # Remove matplotlib agg warnings from generated doc when using plt.show
diff --git a/doc/configuration.rst b/doc/configuration.rst
index bd1efbceb..3c49f0409 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -44,6 +44,7 @@ file, inside a ``sphinx_gallery_conf`` dictionary.
 - ``abort_on_example_error`` (:ref:`abort_on_first`)
 - ``expected_failing_examples`` (:ref:`dont_fail_exit`)
 - ``only_warn_on_example_error`` (:ref:`warning_on_error`)
+- ``parallel`` (:ref:`parallel`)
 
 **Cross-referencing**
 
@@ -2092,6 +2093,28 @@ flag is passed to ``sphinx-build``. This can be enabled by setting::
     }
 
 
+.. _parallel:
+
+Build examples in parallel
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Sphinx-Gallery can be configured to run examples simultaneously using
+:mod:`joblib`. This can be enabled by setting::
+
+    sphinx_gallery_conf = {
+        ...
+        'parallel': True,
+    }
+
+If ``True``, then the same number of jobs will be used as the ``-j`` flag for
+Sphinx. If an ``int``, then that number of jobs will be passed to
+:class:`joblib.Parallel`.
+
+.. warning::
+    Some packages might not play nicely with parallel processing. You might need to
+    set variables in a :ref:`custom resetter <custom_reset>` for example to ensure
+    that all spawned processes are properly set up and torn down.
+
 .. _recommend_examples:
 
 Enabling the example recommender system
diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
index 8698a2abb..e1e19a074 100644
--- a/sphinx_gallery/gen_gallery.py
+++ b/sphinx_gallery/gen_gallery.py
@@ -460,6 +460,19 @@ def _fill_gallery_conf_defaults(sphinx_gallery_conf, app=None, check_keys=True):
 
     _update_gallery_conf_exclude_implicit_doc(gallery_conf)
 
+    if not isinstance(gallery_conf["parallel"], (bool, int)):
+        raise TypeError(
+            'gallery_conf["parallel"] must be bool or int, got '
+            f'{type(gallery_conf["parallel"])}'
+        )
+    if gallery_conf["parallel"] is True:
+        gallery_conf["parallel"] = app.parallel
+    if gallery_conf["parallel"]:
+        try:
+            import joblib  # noqa
+        except Exception:
+            raise ValueError("joblib must be importable when parallel mode is enabled")
+
     return gallery_conf
 
 
@@ -571,8 +584,11 @@ def generate_gallery_rst(app):
     Eventually, we create a toctree in the current index file
     which points to section index files.
     """
-    logger.info("generating gallery...", color="white")
     gallery_conf = app.config.sphinx_gallery_conf
+    extra = ""
+    if gallery_conf["parallel"]:
+        extra = f" (with parallel={gallery_conf['parallel']})"
+    logger.info(f"generating gallery{extra}...", color="white")
 
     seen_backrefs = set()
 
@@ -1356,7 +1372,8 @@ def _expected_failing_examples(gallery_conf):
 
 def _parse_failures(gallery_conf):
     """Split the failures."""
-    failing_examples = set(gallery_conf["failing_examples"].keys())
+    print(gallery_conf["failing_examples"])
+    failing_examples = set(gallery_conf["failing_examples"])
     expected_failing_examples = _expected_failing_examples(gallery_conf)
     failing_as_expected = failing_examples.intersection(expected_failing_examples)
     failing_unexpectedly = failing_examples.difference(expected_failing_examples)
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 2200426fd..5fd108a23 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -398,7 +398,7 @@ def save_thumbnail(image_path_template, src_file, script_vars, file_conf, galler
     base_image_name = os.path.splitext(os.path.basename(src_file))[0]
     thumb_file = os.path.join(thumb_dir, f"sphx_glr_{base_image_name}_thumb.{ext}")
 
-    if src_file in gallery_conf["failing_examples"]:
+    if script_vars.get("formatted_exception", None):
         img = os.path.join(glr_path_static(), "broken_example.png")
     elif os.path.exists(thumbnail_image_path):
         img = thumbnail_image_path
@@ -543,18 +543,25 @@ def generate_dir_rst(
         length=len(sorted_listdir),
     )
 
-    # if gallery_conf['parallel']:
-    #    if gallery_conf["parallel"] is True:
     parallel = list
     p_fun = generate_file_rst
+    if gallery_conf["parallel"]:
+        from joblib import Parallel, delayed
+
+        p_fun = delayed(generate_file_rst)
+        parallel = Parallel(n_jobs=gallery_conf["parallel"])
 
-    # TODO: seen_backrefs gets modified in place and needs to be merged
     results = parallel(
-        p_fun(fname, target_dir, src_dir, gallery_conf, seen_backrefs)
-        for fname in iterator
+        p_fun(fname, target_dir, src_dir, gallery_conf) for fname in iterator
     )
-    for fname, (intro, title, (t, mem)) in zip(sorted_listdir, results):
+    for fi, (intro, title, (t, mem), out_vars) in enumerate(results):
+        fname = sorted_listdir[fi]
         src_file = os.path.normpath(os.path.join(src_dir, fname))
+        gallery_conf["titles"][src_file] = title
+        if "formatted_exception" in out_vars:
+            gallery_conf["failing_examples"][src_file] = out_vars["formatted_exception"]
+        if "passing" in out_vars:
+            gallery_conf["passing_examples"].append(src_file)
         costs.append(dict(t=t, mem=mem, src_file=src_file, target_dir=target_dir))
         gallery_item_filename = (
             (Path(build_target_dir) / fname).with_suffix("").as_posix()
@@ -565,6 +572,18 @@ def generate_dir_rst(
         entries_text.append(this_entry)
         subsection_toctree_filenames.append("/" + gallery_item_filename)
 
+        # Write backreferences
+        if "backrefs" in out_vars:
+            _write_backreferences(
+                out_vars["backrefs"],
+                seen_backrefs,
+                gallery_conf,
+                target_dir,
+                fname,
+                intro,
+                title,
+            )
+
     for entry_text in entries_text:
         subsection_index_content += entry_text
 
@@ -688,7 +707,7 @@ def handle_exception(exc_info, src_file, script_vars, gallery_conf):
     if gallery_conf["abort_on_example_error"]:
         raise
     # Stores failing file
-    gallery_conf["failing_examples"][src_file] = formatted_exception
+    script_vars["formatted_exception"] = formatted_exception
     script_vars["execute_script"] = False
 
     # Ensure it's marked as our style
@@ -1141,12 +1160,12 @@ def execute_script(script_blocks, script_vars, gallery_conf, file_conf):
         # shall not cache md5sum) and has built correctly
         with open(script_vars["target_file"] + ".md5", "w") as file_checksum:
             file_checksum.write(get_md5sum(script_vars["target_file"], "t"))
-        gallery_conf["passing_examples"].append(script_vars["src_file"])
+        script_vars["passing"] = True
 
     return output_blocks, time_elapsed
 
 
-def generate_file_rst(fname, target_dir, src_dir, gallery_conf, seen_backrefs=None):
+def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
     """Generate the rst file for a given example.
 
     Parameters
@@ -1159,18 +1178,21 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf, seen_backrefs=No
         Absolute path to directory where source examples are stored.
     gallery_conf : dict
         Contains the configuration of Sphinx-Gallery.
-    seen_backrefs : set
-        The seen backreferences.
 
     Returns
     -------
     intro: str
         The introduction of the example.
+    title : str
+        The example title.
     cost : tuple
         A tuple containing the ``(time_elapsed, memory_used)`` required to run the
         script.
+    backrefs : set
+        The backrefs seen in this example.
+    out_vars : dict
+        Variables used to run the script.
     """
-    seen_backrefs = set() if seen_backrefs is None else seen_backrefs
     src_file = os.path.normpath(os.path.join(src_dir, fname))
     target_file = Path(target_dir) / fname
     _replace_md5(src_file, target_file, "copy", mode="t")
@@ -1187,7 +1209,6 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf, seen_backrefs=No
     )
 
     intro, title = extract_intro_and_title(fname, script_blocks[0].content)
-    gallery_conf["titles"][src_file] = title
 
     executable = executable_script(src_file, gallery_conf)
 
@@ -1199,11 +1220,10 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf, seen_backrefs=No
             else:
                 gallery_conf["stale_examples"].append(str(target_file))
         if do_return:
-            return intro, title, (0, 0)
+            return intro, title, (0, 0), {}
 
     image_dir = os.path.join(target_dir, "images")
-    if not os.path.exists(image_dir):
-        os.makedirs(image_dir)
+    os.makedirs(image_dir, exist_ok=True)
 
     base_image_name = os.path.splitext(fname)[0]
     image_fname = "sphx_glr_" + base_image_name + "_{0:03}.png"
@@ -1315,19 +1335,18 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf, seen_backrefs=No
         )
     }
 
-    # Write backreferences
-    _write_backreferences(
-        backrefs, seen_backrefs, gallery_conf, target_dir, fname, intro, title
-    )
-
     # This can help with garbage collection in some instances
     if global_variables is not None and "___" in global_variables:
         del global_variables["___"]
+    out_vars = dict(backrefs=backrefs)
+    for key in ("passing", "formatted_exception"):
+        if key in script_vars:
+            out_vars[key] = script_vars[key]
     del script_vars, global_variables  # don't keep these during reset
     if executable and gallery_conf["reset_modules_order"] in ["after", "both"]:
         clean_modules(gallery_conf, fname, "after")
 
-    return intro, title, (time_elapsed, memory_used)
+    return intro, title, (time_elapsed, memory_used), out_vars
 
 
 EXAMPLE_HEADER = """
diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index 57fd8f67a..e129731d0 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -55,6 +55,7 @@
 manim = pytest.importorskip("matplotlib.animation")
 if not manim.writers.is_available("ffmpeg"):
     pytest.skip("ffmpeg is not available", allow_module_level=True)
+pytest.importorskip("joblib")
 
 
 @pytest.fixture(scope="module")
diff --git a/sphinx_gallery/tests/test_gen_rst.py b/sphinx_gallery/tests/test_gen_rst.py
index ed909a6c3..a36d9af2f 100644
--- a/sphinx_gallery/tests/test_gen_rst.py
+++ b/sphinx_gallery/tests/test_gen_rst.py
@@ -477,9 +477,18 @@ def _generate_rst(gallery_conf, fname, content):
         os.path.join(gallery_conf["examples_dir"], fname), mode="w", encoding="utf-8"
     ) as f:
         f.write("\n".join(content))
+    with codecs.open(
+        os.path.join(gallery_conf["examples_dir"], "README.txt"), "w", "utf8"
+    ):
+        pass
+
     # generate rst file
-    sg.generate_file_rst(
-        fname, gallery_conf["gallery_dir"], gallery_conf["examples_dir"], gallery_conf
+    generate_dir_rst(
+        gallery_conf["examples_dir"],
+        gallery_conf["gallery_dir"],
+        gallery_conf,
+        set(),
+        include_toctree=False,
     )
     # read rst file and check if it contains code output
     rst_fname = os.path.splitext(fname)[0] + ".rst"
diff --git a/sphinx_gallery/tests/tinybuild/doc/conf.py b/sphinx_gallery/tests/tinybuild/doc/conf.py
index a6cdc3a82..2bd853b83 100644
--- a/sphinx_gallery/tests/tinybuild/doc/conf.py
+++ b/sphinx_gallery/tests/tinybuild/doc/conf.py
@@ -83,6 +83,7 @@
     "show_api_usage": True,
     "copyfile_regex": r".*\.rst",
     "recommender": {"enable": True, "n_examples": 3},
+    "parallel": 2,
 }
 nitpicky = True
 highlight_language = "python3"

From e2a5b8e9512ae6e60f9277e09a83ac6081a124cd Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Tue, 4 Jun 2024 17:03:19 -0700
Subject: [PATCH 04/40] FIX: More

---
 doc/conf.py                                | 28 ++---------
 doc/sphinxext/sg_doc_build.py              | 26 ++++++++++
 sphinx_gallery/gen_gallery.py              | 30 +++++------
 sphinx_gallery/gen_rst.py                  | 58 ++++++++++------------
 sphinx_gallery/scrapers.py                 |  2 +-
 sphinx_gallery/tests/tinybuild/doc/conf.py |  2 +-
 6 files changed, 75 insertions(+), 71 deletions(-)

diff --git a/doc/conf.py b/doc/conf.py
index b91671163..9fc549275 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -369,27 +369,9 @@ def setup(app):
     import plotly.io
 except ImportError:
     plotly = None
-
-
-def reset_others(gallery_conf, fname):
-    """Reset plotting functions."""
-    if pyvista is not None:
-        pyvista.OFF_SCREEN = True
-        # Preferred plotting style for documentation
-        pyvista.set_plot_theme("document")
-        pyvista.global_theme.window_size = [1024, 768]
-        pyvista.global_theme.font.size = 22
-        pyvista.global_theme.font.label_size = 22
-        pyvista.global_theme.font.title_size = 22
-        pyvista.global_theme.return_cpos = False
-        # necessary when building the sphinx gallery
-        pyvista.BUILDING_GALLERY = True
-        pyvista.set_jupyter_backend(None)
-    if plotly is not None:
-        plotly.io.renderers.default = "sphinx_gallery"
-        examples_dirs.append("../plotly_examples")
-        gallery_dirs.append("auto_plotly_examples")
-
+else:
+    examples_dirs.append("../plotly_examples")
+    gallery_dirs.append("auto_plotly_examples")
 
 min_reported_time = 0
 if "SOURCE_DATE_EPOCH" in os.environ:
@@ -405,7 +387,7 @@ def reset_others(gallery_conf, fname):
     "examples_dirs": examples_dirs,
     "gallery_dirs": gallery_dirs,
     "image_scrapers": image_scrapers,
-    "reset_modules": ("matplotlib", "seaborn", reset_others),
+    "reset_modules": ("matplotlib", "seaborn", "sg_doc_build.reset_others"),
     "compress_images": ("images", "thumbnails"),
     # specify the order of examples to be according to filename
     "within_subsection_order": "FileNameSortKey",
@@ -436,7 +418,7 @@ def reset_others(gallery_conf, fname):
     "image_srcset": ["2x"],
     "nested_sections": True,
     "show_api_usage": True,
-    "parallel": True,
+    "parallel": True,  # can run with -j2 for example for speed
 }
 
 # Remove matplotlib agg warnings from generated doc when using plt.show
diff --git a/doc/sphinxext/sg_doc_build.py b/doc/sphinxext/sg_doc_build.py
index 3f9f08630..3bfb93594 100644
--- a/doc/sphinxext/sg_doc_build.py
+++ b/doc/sphinxext/sg_doc_build.py
@@ -56,3 +56,29 @@ def notebook_modification_function(notebook_content, notebook_filename):
     notebook_content["cells"] = (
         dummy_notebook_content["cells"] + notebook_content["cells"]
     )
+
+
+def reset_others(gallery_conf, fname):
+    """Reset plotting functions."""
+    try:
+        import pyvista
+    except Exception:
+        pass
+    else:
+        pyvista.OFF_SCREEN = True
+        # Preferred plotting style for documentation
+        pyvista.set_plot_theme("document")
+        pyvista.global_theme.window_size = [1024, 768]
+        pyvista.global_theme.font.size = 22
+        pyvista.global_theme.font.label_size = 22
+        pyvista.global_theme.font.title_size = 22
+        pyvista.global_theme.return_cpos = False
+        # necessary when building the sphinx gallery
+        pyvista.BUILDING_GALLERY = True
+        pyvista.set_jupyter_backend(None)
+    try:
+        import plotly.io
+    except Exception:
+        pass
+    else:
+        plotly.io.renderers.default = "sphinx_gallery"
diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
index e1e19a074..aff0e8174 100644
--- a/sphinx_gallery/gen_gallery.py
+++ b/sphinx_gallery/gen_gallery.py
@@ -226,8 +226,23 @@ def _fill_gallery_conf_defaults(sphinx_gallery_conf, app=None, check_keys=True):
             + type(gallery_conf["ignore_repr_types"])
         )
 
+    if not isinstance(gallery_conf["parallel"], (bool, int)):
+        raise TypeError(
+            'gallery_conf["parallel"] must be bool or int, got '
+            f'{type(gallery_conf["parallel"])}'
+        )
+    if gallery_conf["parallel"] is True:
+        gallery_conf["parallel"] = app.parallel
+    if gallery_conf["parallel"] == 1:
+        gallery_conf["parallel"] = False
+    if gallery_conf["parallel"]:
+        try:
+            import joblib  # noqa
+        except Exception:
+            raise ValueError("joblib must be importable when parallel mode is enabled")
+
     # deal with show_memory
-    _get_call_memory_and_base(gallery_conf)
+    _get_call_memory_and_base(gallery_conf, update=True)
 
     # check callables
     for key in (
@@ -460,19 +475,6 @@ def _fill_gallery_conf_defaults(sphinx_gallery_conf, app=None, check_keys=True):
 
     _update_gallery_conf_exclude_implicit_doc(gallery_conf)
 
-    if not isinstance(gallery_conf["parallel"], (bool, int)):
-        raise TypeError(
-            'gallery_conf["parallel"] must be bool or int, got '
-            f'{type(gallery_conf["parallel"])}'
-        )
-    if gallery_conf["parallel"] is True:
-        gallery_conf["parallel"] = app.parallel
-    if gallery_conf["parallel"]:
-        try:
-            import joblib  # noqa
-        except Exception:
-            raise ValueError("joblib must be importable when parallel mode is enabled")
-
     return gallery_conf
 
 
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 5fd108a23..7e69ed471 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -26,7 +26,6 @@
 from textwrap import indent
 import warnings
 from shutil import copyfile
-import subprocess
 import sys
 import traceback
 import codeop
@@ -562,6 +561,8 @@ def generate_dir_rst(
             gallery_conf["failing_examples"][src_file] = out_vars["formatted_exception"]
         if "passing" in out_vars:
             gallery_conf["passing_examples"].append(src_file)
+        if "stale" in out_vars:
+            gallery_conf["stale_examples"].append(out_vars["stale"])
         costs.append(dict(t=t, mem=mem, src_file=src_file, target_dir=target_dir))
         gallery_item_filename = (
             (Path(build_target_dir) / fname).with_suffix("").as_posix()
@@ -765,24 +766,13 @@ def __call__(self):
                         sys.modules["__main__"] = old_main
 
 
+@lru_cache()
 def _get_memory_base():
-    """Get the base amount of memory used by running a Python process."""
+    """Get the base amount of memory used by the current Python process."""
     # There might be a cleaner way to do this at some point
     from memory_profiler import memory_usage
 
-    if sys.platform in ("win32", "darwin"):
-        sleep, timeout = (1, 2)
-    else:
-        sleep, timeout = (0.5, 1)
-    proc = subprocess.Popen(
-        [sys.executable, "-c", f"import time, sys; time.sleep({sleep}); sys.exit(0)"],
-        close_fds=True,
-    )
-    memories = memory_usage(proc, interval=1e-3, timeout=timeout)
-    proc.communicate(timeout=timeout)
-    # On OSX sometimes the last entry can be None
-    memories = [mem for mem in memories if mem is not None] + [0.0]
-    memory_base = max(memories)
+    memory_base = memory_usage(max_usage=True)
     return memory_base
 
 
@@ -1194,6 +1184,7 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
         Variables used to run the script.
     """
     src_file = os.path.normpath(os.path.join(src_dir, fname))
+    out_vars = dict()
     target_file = Path(target_dir) / fname
     _replace_md5(src_file, target_file, "copy", mode="t")
 
@@ -1218,9 +1209,9 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
             if gallery_conf["run_stale_examples"]:
                 do_return = False
             else:
-                gallery_conf["stale_examples"].append(str(target_file))
+                out_vars["stale"] = str(target_file)
         if do_return:
-            return intro, title, (0, 0), {}
+            return intro, title, (0, 0), out_vars
 
     image_dir = os.path.join(target_dir, "images")
     os.makedirs(image_dir, exist_ok=True)
@@ -1338,7 +1329,7 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
     # This can help with garbage collection in some instances
     if global_variables is not None and "___" in global_variables:
         del global_variables["___"]
-    out_vars = dict(backrefs=backrefs)
+    out_vars["backrefs"] = backrefs
     for key in ("passing", "formatted_exception"):
         if key in script_vars:
             out_vars[key] = script_vars[key]
@@ -1638,21 +1629,24 @@ def _sg_call_memory_noop(func):
     return 0.0, func()
 
 
-def _get_call_memory_and_base(gallery_conf):
-    show_memory = gallery_conf["show_memory"]
-
+def _get_call_memory_and_base(gallery_conf, *, update=False):
     # Default to no-op version
     call_memory = _sg_call_memory_noop
     memory_base = 0.0
 
-    if show_memory:
-        if callable(show_memory):
-            call_memory = show_memory
-        elif gallery_conf["plot_gallery"]:  # True-like
-            out = _get_memprof_call_memory()
+    if gallery_conf["show_memory"] and gallery_conf["plot_gallery"]:
+        if gallery_conf["parallel"]:
+            if update:
+                logger.warning(
+                    f"{gallery_conf['show_memory']=} disabled due to "
+                    f"{gallery_conf['parallel']=}."
+                )
+                gallery_conf["show_memory"] = False
+        else:
+            out = _get_memprof_call_memory(warn=update)
             if out is not None:
                 call_memory, memory_base = out
-            else:
+            elif update:
                 gallery_conf["show_memory"] = False
 
     assert callable(call_memory)
@@ -1671,14 +1665,14 @@ def _sg_call_memory_memprof(func):
     return mem, out
 
 
-@lru_cache()
-def _get_memprof_call_memory():
+def _get_memprof_call_memory(*, warn=False):
     try:
         from memory_profiler import memory_usage  # noqa
     except ImportError:
-        logger.warning(
-            "Please install 'memory_profiler' to enable peak memory measurements."
-        )
+        if warn:
+            logger.warning(
+                "Please install 'memory_profiler' to enable peak memory measurements."
+            )
         return None
     else:
         return _sg_call_memory_memprof, _get_memory_base()
diff --git a/sphinx_gallery/scrapers.py b/sphinx_gallery/scrapers.py
index 6fe8d450c..9bd11ff7f 100644
--- a/sphinx_gallery/scrapers.py
+++ b/sphinx_gallery/scrapers.py
@@ -265,7 +265,7 @@ def _anim_rst(anim, image_path, gallery_conf):
         dpi = fig.dpi
     video_uri = video.relative_to(gallery_conf["src_dir"]).as_posix()
     html = _ANIMATION_VIDEO_RST.format(
-        video=f"/{video_uri}",
+        video=video_uri,
         width=int(fig_size[0] * dpi),
         height=int(fig_size[1] * dpi),
         options="".join(f"   :{opt}:\n" for opt in options),
diff --git a/sphinx_gallery/tests/tinybuild/doc/conf.py b/sphinx_gallery/tests/tinybuild/doc/conf.py
index 2bd853b83..101340dce 100644
--- a/sphinx_gallery/tests/tinybuild/doc/conf.py
+++ b/sphinx_gallery/tests/tinybuild/doc/conf.py
@@ -83,7 +83,7 @@
     "show_api_usage": True,
     "copyfile_regex": r".*\.rst",
     "recommender": {"enable": True, "n_examples": 3},
-    "parallel": 2,
+    "parallel": False,
 }
 nitpicky = True
 highlight_language = "python3"

From c446f01e7fb2513c88a26f6f8a82b45c539924f9 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Tue, 4 Jun 2024 17:21:13 -0700
Subject: [PATCH 05/40] FIX: Cleanup

---
 .circleci/config.yml          |  2 +-
 doc/configuration.rst         |  2 ++
 sphinx_gallery/gen_gallery.py |  1 -
 sphinx_gallery/gen_rst.py     | 20 +++++++++++---------
 sphinx_gallery/scrapers.py    |  2 +-
 5 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3a2c99a3d..aa11a4cc8 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -70,7 +70,7 @@ jobs:
       - attach_workspace:
           at: ~/
       - bash_env
-      - run: sphinx-build doc doc/_build/html -nW --keep-going -b html -j2 2>&1 | tee sphinx_log.txt
+      - run: sphinx-build doc doc/_build/html -nW --keep-going -b html 2>&1 | tee sphinx_log.txt
       - run:
           name: Check sphinx log for warnings (which are treated as errors)
           when: always
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 3c49f0409..cf1fd77c5 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -2115,6 +2115,8 @@ Sphinx. If an ``int``, then that number of jobs will be passed to
     set variables in a :ref:`custom resetter <custom_reset>` for example to ensure
     that all spawned processes are properly set up and torn down.
 
+    Using parallel building will also disable memory measurements.
+
 .. _recommend_examples:
 
 Enabling the example recommender system
diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
index aff0e8174..760857c0a 100644
--- a/sphinx_gallery/gen_gallery.py
+++ b/sphinx_gallery/gen_gallery.py
@@ -1374,7 +1374,6 @@ def _expected_failing_examples(gallery_conf):
 
 def _parse_failures(gallery_conf):
     """Split the failures."""
-    print(gallery_conf["failing_examples"])
     failing_examples = set(gallery_conf["failing_examples"])
     expected_failing_examples = _expected_failing_examples(gallery_conf)
     failing_as_expected = failing_examples.intersection(expected_failing_examples)
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 7e69ed471..d2a458a02 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -558,10 +558,13 @@ def generate_dir_rst(
         src_file = os.path.normpath(os.path.join(src_dir, fname))
         gallery_conf["titles"][src_file] = title
         if "formatted_exception" in out_vars:
+            assert "passing" not in out_vars
+            assert "stale" not in out_vars
             gallery_conf["failing_examples"][src_file] = out_vars["formatted_exception"]
-        if "passing" in out_vars:
+        elif "passing" in out_vars:
+            assert "stale" not in out_vars
             gallery_conf["passing_examples"].append(src_file)
-        if "stale" in out_vars:
+        else:  # should be guaranteed stale is in out_vars
             gallery_conf["stale_examples"].append(out_vars["stale"])
         costs.append(dict(t=t, mem=mem, src_file=src_file, target_dir=target_dir))
         gallery_item_filename = (
@@ -766,7 +769,6 @@ def __call__(self):
                         sys.modules["__main__"] = old_main
 
 
-@lru_cache()
 def _get_memory_base():
     """Get the base amount of memory used by the current Python process."""
     # There might be a cleaner way to do this at some point
@@ -1643,7 +1645,7 @@ def _get_call_memory_and_base(gallery_conf, *, update=False):
                 )
                 gallery_conf["show_memory"] = False
         else:
-            out = _get_memprof_call_memory(warn=update)
+            out = _get_memprof_call_memory()
             if out is not None:
                 call_memory, memory_base = out
             elif update:
@@ -1665,14 +1667,14 @@ def _sg_call_memory_memprof(func):
     return mem, out
 
 
-def _get_memprof_call_memory(*, warn=False):
+@lru_cache()
+def _get_memprof_call_memory():
     try:
         from memory_profiler import memory_usage  # noqa
     except ImportError:
-        if warn:
-            logger.warning(
-                "Please install 'memory_profiler' to enable peak memory measurements."
-            )
+        logger.warning(
+            "Please install 'memory_profiler' to enable peak memory measurements."
+        )
         return None
     else:
         return _sg_call_memory_memprof, _get_memory_base()
diff --git a/sphinx_gallery/scrapers.py b/sphinx_gallery/scrapers.py
index 9bd11ff7f..6fe8d450c 100644
--- a/sphinx_gallery/scrapers.py
+++ b/sphinx_gallery/scrapers.py
@@ -265,7 +265,7 @@ def _anim_rst(anim, image_path, gallery_conf):
         dpi = fig.dpi
     video_uri = video.relative_to(gallery_conf["src_dir"]).as_posix()
     html = _ANIMATION_VIDEO_RST.format(
-        video=video_uri,
+        video=f"/{video_uri}",
         width=int(fig_size[0] * dpi),
         height=int(fig_size[1] * dpi),
         options="".join(f"   :{opt}:\n" for opt in options),

From 4c2247d4f86d3bb91c173535d530e546849b1e0b Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Tue, 4 Jun 2024 17:32:29 -0700
Subject: [PATCH 06/40] FIX: More

---
 sphinx_gallery/gen_rst.py                  | 4 ++--
 sphinx_gallery/tests/tinybuild/doc/conf.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index d2a458a02..7393ad2c1 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -397,7 +397,7 @@ def save_thumbnail(image_path_template, src_file, script_vars, file_conf, galler
     base_image_name = os.path.splitext(os.path.basename(src_file))[0]
     thumb_file = os.path.join(thumb_dir, f"sphx_glr_{base_image_name}_thumb.{ext}")
 
-    if script_vars.get("formatted_exception", None):
+    if "formatted_exception" in script_vars:
         img = os.path.join(glr_path_static(), "broken_example.png")
     elif os.path.exists(thumbnail_image_path):
         img = thumbnail_image_path
@@ -564,7 +564,7 @@ def generate_dir_rst(
         elif "passing" in out_vars:
             assert "stale" not in out_vars
             gallery_conf["passing_examples"].append(src_file)
-        else:  # should be guaranteed stale is in out_vars
+        elif "stale" in out_vars:  # non-executable files have none of these three
             gallery_conf["stale_examples"].append(out_vars["stale"])
         costs.append(dict(t=t, mem=mem, src_file=src_file, target_dir=target_dir))
         gallery_item_filename = (
diff --git a/sphinx_gallery/tests/tinybuild/doc/conf.py b/sphinx_gallery/tests/tinybuild/doc/conf.py
index 101340dce..220fc6ffc 100644
--- a/sphinx_gallery/tests/tinybuild/doc/conf.py
+++ b/sphinx_gallery/tests/tinybuild/doc/conf.py
@@ -83,7 +83,7 @@
     "show_api_usage": True,
     "copyfile_regex": r".*\.rst",
     "recommender": {"enable": True, "n_examples": 3},
-    "parallel": False,
+    "parallel": True,  # can be built with -j2 for example
 }
 nitpicky = True
 highlight_language = "python3"

From fc3b8fc7348f4d4cd1cf8771a41ef92c5c3793b2 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Tue, 4 Jun 2024 17:42:15 -0700
Subject: [PATCH 07/40] FIX: Deps

---
 pyproject.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index b6f0cefdc..eb59696ee 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,6 +39,7 @@ optional-dependencies.animations = [
 optional-dependencies.dev = [
   "absl-py",
   "graphviz",
+  "intersphinx-registry",
   "joblib",
   "jupyterlite-sphinx",
   "lxml",
@@ -56,6 +57,9 @@ optional-dependencies.dev = [
 optional-dependencies.jupyterlite = [
   "jupyterlite-sphinx",
 ]
+optional-dependencies.parallel = [
+  "joblib",
+]
 optional-dependencies.recommender = [
   "numpy",
 ]

From c205cca2f30fdf32462f56f5329d4bef65406f37 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Tue, 4 Jun 2024 17:49:59 -0700
Subject: [PATCH 08/40] FIX: Patch

---
 sphinx_gallery/gen_rst.py                | 4 +++-
 sphinx_gallery/tests/test_gen_gallery.py | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 7393ad2c1..b179097ac 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -1637,7 +1637,9 @@ def _get_call_memory_and_base(gallery_conf, *, update=False):
     memory_base = 0.0
 
     if gallery_conf["show_memory"] and gallery_conf["plot_gallery"]:
-        if gallery_conf["parallel"]:
+        if callable(gallery_conf["show_memory"]):
+            call_memory = gallery_conf["show_memory"]
+        elif gallery_conf["parallel"]:
             if update:
                 logger.warning(
                     f"{gallery_conf['show_memory']=} disabled due to "
diff --git a/sphinx_gallery/tests/test_gen_gallery.py b/sphinx_gallery/tests/test_gen_gallery.py
index cd70c1777..93b4d9639 100644
--- a/sphinx_gallery/tests/test_gen_gallery.py
+++ b/sphinx_gallery/tests/test_gen_gallery.py
@@ -538,7 +538,7 @@ def test_examples_not_expected_to_pass(sphinx_app_wrapper):
 def test_show_memory_callable(sphinx_app_wrapper):
     sphinx_app = sphinx_app_wrapper.build_sphinx_app()
     status = sphinx_app._status.getvalue()
-    assert "0.0 MB" in status
+    assert "0.0 MB" in status, status
 
 
 @pytest.mark.parametrize(

From 6916dd77ab4945a2b2aaaf528bb2bc52507db93b Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Tue, 4 Jun 2024 17:51:21 -0700
Subject: [PATCH 09/40] FIX: Revert

---
 sphinx_gallery/gen_rst.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index b179097ac..a9e93ed63 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -1632,13 +1632,15 @@ def _sg_call_memory_noop(func):
 
 
 def _get_call_memory_and_base(gallery_conf, *, update=False):
+    show_memory = gallery_conf["show_memory"]
+
     # Default to no-op version
     call_memory = _sg_call_memory_noop
     memory_base = 0.0
 
-    if gallery_conf["show_memory"] and gallery_conf["plot_gallery"]:
-        if callable(gallery_conf["show_memory"]):
-            call_memory = gallery_conf["show_memory"]
+    if show_memory and gallery_conf["plot_gallery"]:
+        if callable(show_memory):
+            call_memory = show_memory
         elif gallery_conf["parallel"]:
             if update:
                 logger.warning(

From 03dd5a7e0bfe098e188c3975aca7367bb8bfdde3 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Tue, 4 Jun 2024 22:24:36 -0700
Subject: [PATCH 10/40] FIX: Deps

---
 .circleci/config.yml        | 2 +-
 .github/workflows/tests.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index aa11a4cc8..e6384ed3f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -43,7 +43,7 @@ jobs:
                 traits pyvista memory_profiler "ipython!=8.7.0" plotly graphviz \
                 "docutils>=0.18" imageio pydata-sphinx-theme \
                 "jupyterlite-sphinx>=0.8.0,<0.9.0" "jupyterlite-pyodide-kernel<0.1.0" \
-                libarchive-c "sphinxcontrib-video>=0.2.1rc0"
+                libarchive-c "sphinxcontrib-video>=0.2.1rc0" intersphinx_registry
             pip uninstall -yq vtk  # pyvista installs vtk above
             pip install --upgrade --only-binary ":all" --extra-index-url https://wheels.vtk.org vtk-osmesa
       - save_cache:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 59efa7b74..a09321721 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -79,7 +79,7 @@ jobs:
             python=${{ env.PYTHON_VERSION }} pip numpy setuptools matplotlib pillow
             pytest pytest-cov coverage seaborn statsmodels plotly joblib wheel libiconv
             pygraphviz memory_profiler ipython pypandoc lxml conda-libmamba-solver mamba
-            ffmpeg
+            ffmpeg intersphinx-registry
         if: matrix.distrib == 'mamba'
       # Make sure that things work even if the locale is set to C (which
       # effectively means ASCII). Some of the input rst files have unicode

From 6d23446964ea1cbec490d04322936cd0025b326b Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Wed, 5 Jun 2024 08:27:34 -0700
Subject: [PATCH 11/40] TST: Try it

---
 sphinx_gallery/gen_rst.py                  | 6 +++++-
 sphinx_gallery/tests/tinybuild/doc/conf.py | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index a9e93ed63..0574ce756 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -548,7 +548,11 @@ def generate_dir_rst(
         from joblib import Parallel, delayed
 
         p_fun = delayed(generate_file_rst)
-        parallel = Parallel(n_jobs=gallery_conf["parallel"])
+        parallel = Parallel(
+            n_jobs=gallery_conf["parallel"],
+            pre_dispatch="n_jobs",
+            prefer="processes",
+        )
 
     results = parallel(
         p_fun(fname, target_dir, src_dir, gallery_conf) for fname in iterator
diff --git a/sphinx_gallery/tests/tinybuild/doc/conf.py b/sphinx_gallery/tests/tinybuild/doc/conf.py
index 220fc6ffc..2bd853b83 100644
--- a/sphinx_gallery/tests/tinybuild/doc/conf.py
+++ b/sphinx_gallery/tests/tinybuild/doc/conf.py
@@ -83,7 +83,7 @@
     "show_api_usage": True,
     "copyfile_regex": r".*\.rst",
     "recommender": {"enable": True, "n_examples": 3},
-    "parallel": True,  # can be built with -j2 for example
+    "parallel": 2,
 }
 nitpicky = True
 highlight_language = "python3"

From e6694d78a0aa9b4fe765944c2512f5bc002d104b Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Wed, 5 Jun 2024 08:28:22 -0700
Subject: [PATCH 12/40] FIX: Explicit

---
 sphinx_gallery/gen_rst.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 0574ce756..6d906e4d9 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -551,7 +551,7 @@ def generate_dir_rst(
         parallel = Parallel(
             n_jobs=gallery_conf["parallel"],
             pre_dispatch="n_jobs",
-            prefer="processes",
+            backend="loky",
         )
 
     results = parallel(

From 503f9598346dbfac07bc84a6a68fd396817658a8 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Wed, 5 Jun 2024 12:44:21 -0700
Subject: [PATCH 13/40] FIX: More

---
 sphinx_gallery/gen_gallery.py         |  6 ++----
 sphinx_gallery/gen_rst.py             | 21 +++++++++++++++------
 sphinx_gallery/interactive_example.py | 24 +++++++++++++-----------
 sphinx_gallery/tests/test_full.py     | 12 ++++--------
 4 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
index 760857c0a..0b8352dae 100644
--- a/sphinx_gallery/gen_gallery.py
+++ b/sphinx_gallery/gen_gallery.py
@@ -545,8 +545,7 @@ def _prepare_sphx_glr_dirs(gallery_conf, srcdir):
 
     if bool(gallery_conf["backreferences_dir"]):
         backreferences_dir = os.path.join(srcdir, gallery_conf["backreferences_dir"])
-        if not os.path.exists(backreferences_dir):
-            os.makedirs(backreferences_dir)
+        os.makedirs(backreferences_dir, exist_ok=True)
 
     return list(zip(examples_dirs, gallery_dirs))
 
@@ -1339,8 +1338,7 @@ def write_junit_xml(gallery_conf, target_dir, costs):
     # Actually write it
     fname = os.path.normpath(os.path.join(target_dir, gallery_conf["junit"]))
     junit_dir = os.path.dirname(fname)
-    if not os.path.isdir(junit_dir):
-        os.makedirs(junit_dir)
+    os.makedirs(junit_dir, exist_ok=True)
     with codecs.open(fname, "w", encoding="utf-8") as fid:
         fid.write(output)
 
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 6d906e4d9..0822cd6b9 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -63,8 +63,11 @@
 from .block_parser import BlockParser
 
 from .notebook import jupyter_notebook, save_notebook
-from .interactive_example import gen_binder_rst
-from .interactive_example import gen_jupyterlite_rst
+from .interactive_example import (
+    _add_jupyterlite_badge_logo,
+    gen_binder_rst,
+    gen_jupyterlite_rst,
+)
 
 
 logger = sphinx.util.logging.getLogger("sphinx-gallery")
@@ -369,8 +372,7 @@ def save_thumbnail(image_path_template, src_file, script_vars, file_conf, galler
         Sphinx-Gallery configuration dictionary
     """
     thumb_dir = os.path.join(os.path.dirname(image_path_template), "thumb")
-    if not os.path.exists(thumb_dir):
-        os.makedirs(thumb_dir)
+    os.makedirs(thumb_dir, exist_ok=True)
 
     # read specification of the figure to display as thumbnail from main text
     thumbnail_number = file_conf.get("thumbnail_number", None)
@@ -505,8 +507,15 @@ def generate_dir_rst(
     # Add empty lines to avoid bug in issue #165
     subsection_index_content += "\n\n"
 
-    if not os.path.exists(target_dir):
-        os.makedirs(target_dir)
+    # Make all dirs ahead of time to avoid collisions in parallel processing
+    os.makedirs(target_dir, exist_ok=True)
+    image_dir = os.path.join(target_dir, "images")
+    os.makedirs(image_dir, exist_ok=True)
+    thumb_dir = os.path.join(image_dir, "thumb")
+    os.makedirs(thumb_dir, exist_ok=True)
+    if gallery_conf["jupyterlite"] is not None:
+        _add_jupyterlite_badge_logo(image_dir)
+
     # get filenames
     listdir = [
         fname
diff --git a/sphinx_gallery/interactive_example.py b/sphinx_gallery/interactive_example.py
index 95e487f64..6b87d1472 100644
--- a/sphinx_gallery/interactive_example.py
+++ b/sphinx_gallery/interactive_example.py
@@ -168,8 +168,7 @@ def _copy_binder_reqs(app, binder_conf):
             )
 
     binder_folder = os.path.join(app.outdir, "binder")
-    if not os.path.isdir(binder_folder):
-        os.makedirs(binder_folder)
+    os.makedirs(binder_folder, exist_ok=True)
 
     # Copy over the requirements to the output directory
     for path in path_reqs:
@@ -207,7 +206,7 @@ def _copy_binder_notebooks(app):
     binder_conf = gallery_conf["binder"]
     notebooks_dir = os.path.join(app.outdir, binder_conf["notebooks_dir"])
     shutil.rmtree(notebooks_dir, ignore_errors=True)
-    os.makedirs(notebooks_dir)
+    os.makedirs(notebooks_dir, exist_ok=True)
 
     if not isinstance(gallery_dirs, (list, tuple)):
         gallery_dirs = [gallery_dirs]
@@ -425,14 +424,8 @@ def gen_jupyterlite_rst(fpath, gallery_conf):
 
     # Similar work-around for badge file as in
     # gen_binder_rst
-    physical_path = os.path.join(
-        os.path.dirname(fpath), "images", "jupyterlite_badge_logo.svg"
-    )
-    os.makedirs(os.path.dirname(physical_path), exist_ok=True)
-    if not os.path.isfile(physical_path):
-        shutil.copyfile(
-            os.path.join(glr_path_static(), "jupyterlite_badge_logo.svg"), physical_path
-        )
+    image_dir = os.path.join(os.path.dirname(fpath), "images")
+    _add_jupyterlite_badge_logo(image_dir)
     rst = (
         "\n"
         "  .. container:: lite-badge\n\n"
@@ -444,6 +437,15 @@ def gen_jupyterlite_rst(fpath, gallery_conf):
     return rst
 
 
+def _add_jupyterlite_badge_logo(image_dir):
+    os.makedirs(image_dir, exist_ok=True)
+    physical_path = os.path.join(image_dir, "jupyterlite_badge_logo.svg")
+    if not os.path.isfile(physical_path):
+        shutil.copyfile(
+            os.path.join(glr_path_static(), "jupyterlite_badge_logo.svg"), physical_path
+        )
+
+
 def check_jupyterlite_conf(jupyterlite_conf, app):
     """Return full JupyterLite configuration with defaults."""
     # app=None can happen for testing
diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index e129731d0..adf36857b 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -460,12 +460,8 @@ def test_embed_links_and_styles(sphinx_app):
     )  # noqa: E501
     assert dummy_class_prop.search(lines) is not None
 
-    try:
-        import memory_profiler  # noqa: F401
-    except ImportError:
-        assert "memory usage" not in lines
-    else:
-        assert "memory usage" in lines
+    # We do a parallel build so there should not be memory usage reported
+    assert "memory usage" not in lines
 
     # CSS styles
     assert 'class="sphx-glr-signature"' in lines
@@ -613,7 +609,7 @@ def _assert_mtimes(list_orig, list_new, different=(), ignore=()):
                 op.getmtime(new),
                 atol=1e-3,
                 rtol=1e-20,
-                err_msg=op.basename(orig),
+                err_msg=f"{op.basename(orig)} was updated but should not have been",
             )
 
 
@@ -979,7 +975,7 @@ def test_error_messages(sphinx_app, name, want):
     """Test that informative error messages are added."""
     src_dir = Path(sphinx_app.srcdir)
     rst = (src_dir / "auto_examples" / (name + ".rst")).read_text("utf-8")
-    assert re.match(want, rst, re.DOTALL) is not None
+    assert re.match(want, rst, re.DOTALL) is not None, f"{name} should have had: {want}"
 
 
 @pytest.mark.parametrize(

From 492c81481faacea01d322007248ce587b593afb1 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Wed, 5 Jun 2024 21:31:53 -0700
Subject: [PATCH 14/40] MAINT: Switch to JSON for codeobj

---
 doc/getting_started.rst                    |  2 +-
 sphinx_gallery/backreferences.py           | 30 +++++++--------
 sphinx_gallery/docs_resolv.py              | 44 ++++++++++++++--------
 sphinx_gallery/gen_gallery.py              |  9 ++++-
 sphinx_gallery/gen_rst.py                  | 15 +++-----
 sphinx_gallery/tests/test_full.py          | 30 +++++++--------
 sphinx_gallery/tests/tinybuild/doc/conf.py |  2 +-
 sphinx_gallery/utils.py                    |  8 ++++
 8 files changed, 78 insertions(+), 62 deletions(-)

diff --git a/doc/getting_started.rst b/doc/getting_started.rst
index 2eb6c4432..9cafe3a55 100644
--- a/doc/getting_started.rst
+++ b/doc/getting_started.rst
@@ -172,7 +172,7 @@ generated:
 * ``.py`` - to enable the user to download a ``.py`` version of the example.
 * ``.py.md5`` - a md5 hash of the ``.py`` file, used to determine if changes
   have been made to the file and thus if new output files need to be generated.
-* ``_codeobj.pickle`` - used to identify function names and to which module
+* ``.codeobj.json`` - used to identify function names and to which module
   they belong (more details in
   :ref:`sphx_glr_auto_examples_plot_6_function_identifier.py`)
 
diff --git a/sphinx_gallery/backreferences.py b/sphinx_gallery/backreferences.py
index aff501ced..3f198c8a9 100644
--- a/sphinx_gallery/backreferences.py
+++ b/sphinx_gallery/backreferences.py
@@ -6,8 +6,6 @@
 """
 
 import ast
-import codecs
-import collections
 from html import escape
 import inspect
 import os
@@ -250,9 +248,9 @@ def identify_names(script_blocks, ref_regex, global_variables=None, node=""):
 
     Returns
     -------
-    example_code_obj : OrderedDict[str, Any]
-        OrderedDict with information about all code object references found in an
-        example. OrderedDict contains the following keys:
+    example_code_obj : Dict[str, Any]
+        Dict with information about all code object references found in an
+        example. Dict contains the following keys:
 
             - example_code_obj['name'] : function or class name (str)
             - example_code_obj['module'] : module name (str)
@@ -272,7 +270,7 @@ def identify_names(script_blocks, ref_regex, global_variables=None, node=""):
     # Get matches from docstring inspection (explicit matches)
     text = "\n".join(block.content for block in script_blocks if block.type == "text")
     names.extend((x, x, False, False, True) for x in re.findall(ref_regex, text))
-    example_code_obj = collections.OrderedDict()  # order is important
+    example_code_obj = dict()  # native dict preserves order nowadays
     # Make a list of all guesses, in `_embed_code_links` we will break
     # when we find a match
     for name, full_name, class_like, is_class, is_explicit in names:
@@ -293,13 +291,13 @@ def identify_names(script_blocks, ref_regex, global_variables=None, node=""):
 
         # get shortened module name
         module_short = _get_short_module_name(module, attribute)
-        cobj = {
-            "name": attribute,
-            "module": module,
-            "module_short": module_short or module,
-            "is_class": is_class,
-            "is_explicit": is_explicit,
-        }
+        cobj = dict(
+            name=attribute,
+            module=module,
+            module_short=module_short or module,
+            is_class=is_class,
+            is_explicit=is_explicit,
+        )
         example_code_obj[name].append(cobj)
     return example_code_obj
 
@@ -392,9 +390,7 @@ def _write_backreferences(
             f"{backref}.examples.new",
         )
         seen = backref in seen_backrefs
-        with codecs.open(
-            include_path, "a" if seen else "w", encoding="utf-8"
-        ) as ex_file:
+        with open(include_path, "a" if seen else "w", encoding="utf-8") as ex_file:
             if not seen:
                 # Be aware that if the number of lines of this heading changes,
                 #   the minigallery directive should be modified accordingly
@@ -433,7 +429,7 @@ def _finalize_backreferences(seen_backrefs, gallery_conf):
         if os.path.isfile(path):
             # Close div containing all thumbnails
             # (it was open in _write_backreferences)
-            with codecs.open(path, "a", encoding="utf-8") as ex_file:
+            with open(path, "a", encoding="utf-8") as ex_file:
                 ex_file.write(THUMBNAIL_PARENT_DIV_CLOSE)
             _replace_md5(path, mode="t")
         else:
diff --git a/sphinx_gallery/docs_resolv.py b/sphinx_gallery/docs_resolv.py
index 8ac63a2c6..f0b304782 100644
--- a/sphinx_gallery/docs_resolv.py
+++ b/sphinx_gallery/docs_resolv.py
@@ -2,11 +2,10 @@
 # License: 3-clause BSD
 """Link resolver objects."""
 
-import codecs
 import gzip
+import json
 from io import BytesIO
 import os
-import pickle
 import posixpath
 import re
 import shelve
@@ -19,7 +18,7 @@
 from sphinx.search import js_index
 import sphinx.util
 
-from .utils import status_iterator
+from .utils import status_iterator, _replace_md5
 
 
 logger = sphinx.util.logging.getLogger("sphinx-gallery")
@@ -40,7 +39,7 @@ def _get_data(url):
             raise ExtensionError(f"unknown encoding {encoding!r}")
         data = data.decode("utf-8")
     else:
-        with codecs.open(url, mode="r", encoding="utf-8") as fid:
+        with open(url, mode="r", encoding="utf-8") as fid:
             data = fid.read()
 
     return data
@@ -244,8 +243,8 @@ def resolve(self, cobj, this_url, return_type=False):
 
         Parameters
         ----------
-        cobj : OrderedDict[str, Any]
-            OrderedDict with information about the "code object" for which we are
+        cobj : Dict[str, Any]
+            Dict with information about the "code object" for which we are
             resolving a link.
 
                 - cobj['name'] : function or class name (str)
@@ -254,7 +253,6 @@ def resolve(self, cobj, this_url, return_type=False):
                 - cobj['is_class'] : whether object is class (bool)
                 - cobj['is_explicit'] : whether object is an explicit
                   backreference (referred to by sphinx markup) (bool)
-
         this_url: str
             URL of the current page. Needed to construct relative URLs
             (only used if relative=True in constructor).
@@ -335,6 +333,21 @@ def _get_intersphinx_inventory(app):
     return intersphinx_inv
 
 
+# Whatever mechanism is used for writing here should be paired with reading in
+# _embed_code_links
+def _write_code_obj(target_file, example_code_obj):
+    codeobj_fname = target_file.with_name(target_file.stem + ".codeobj.json.new")
+    with open(codeobj_fname, "w", encoding="utf-8") as fid:
+        json.dump(
+            example_code_obj,
+            fid,
+            ensure_ascii=False,
+            indent=2,
+            check_circular=False,
+        )
+    _replace_md5(codeobj_fname)
+
+
 def _embed_code_links(app, gallery_conf, gallery_dir):
     """Add resolvers for the packages for which we want to show links."""
     doc_resolvers = {}
@@ -369,6 +382,7 @@ def _embed_code_links(app, gallery_conf, gallery_dir):
         [dirpath, filename]
         for dirpath, _, filenames in os.walk(html_gallery_dir)
         for filename in filenames
+        if filename.endswith(".html")
     ]
     iterator = status_iterator(
         flat,
@@ -381,15 +395,15 @@ def _embed_code_links(app, gallery_conf, gallery_dir):
     for dirpath, fname in iterator:
         full_fname = os.path.join(html_gallery_dir, dirpath, fname)
         subpath = dirpath[len(html_gallery_dir) + 1 :]
-        pickle_fname = os.path.join(
-            src_gallery_dir, subpath, fname[:-5] + "_codeobj.pickle"
+        json_fname = os.path.join(
+            src_gallery_dir, subpath, fname[:-5] + ".codeobj.json"
         )
-        if not os.path.exists(pickle_fname):
+        if not os.path.exists(json_fname):
             continue
 
-        # we have a pickle file with the objects to embed links for
-        with open(pickle_fname, "rb") as fid:
-            example_code_obj = pickle.load(fid)
+        # we have a json file with the objects to embed links for
+        with open(json_fname, "r", encoding="utf-8") as fid:
+            example_code_obj = json.load(fid)
         # generate replacement strings with the links
         str_repl = {}
         for name in sorted(example_code_obj):
@@ -473,9 +487,9 @@ def substitute_link(match):
             return str_repl[match.group()]
 
         if len(str_repl) > 0:
-            with codecs.open(full_fname, "r", "utf-8") as fid:
+            with open(full_fname, "r", encoding="utf-8") as fid:
                 lines_in = fid.readlines()
-            with codecs.open(full_fname, "w", "utf-8") as fid:
+            with open(full_fname, "w", encoding="utf-8") as fid:
                 for line in lines_in:
                     line_out = regex.sub(substitute_link, line)
                     fid.write(line_out)
diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
index 0b8352dae..3132e0fca 100644
--- a/sphinx_gallery/gen_gallery.py
+++ b/sphinx_gallery/gen_gallery.py
@@ -1284,6 +1284,11 @@ def write_junit_xml(gallery_conf, target_dir, costs):
     failing_as_expected, failing_unexpectedly, passing_unexpectedly = _parse_failures(
         gallery_conf
     )
+    import sys
+
+    print(f"{failing_as_expected=}", file=sys.__stderr__)
+    print(f"{failing_unexpectedly=}", file=sys.__stderr__)
+    print(f"{passing_unexpectedly=}", file=sys.__stderr__)
     n_tests = 0
     n_failures = 0
     n_skips = 0
@@ -1378,11 +1383,11 @@ def _parse_failures(gallery_conf):
     failing_unexpectedly = failing_examples.difference(expected_failing_examples)
     passing_unexpectedly = expected_failing_examples.difference(failing_examples)
     # filter from examples actually run
-    passing_unexpectedly = [
+    passing_unexpectedly = set(
         src_file
         for src_file in passing_unexpectedly
         if re.search(gallery_conf["filename_pattern"], src_file)
-    ]
+    )
     return failing_as_expected, failing_unexpectedly, passing_unexpectedly
 
 
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 0822cd6b9..df00ca629 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -12,10 +12,8 @@
 import copy
 import contextlib
 import ast
-import codecs
 from functools import lru_cache
 import gc
-import pickle
 import importlib
 import inspect
 from io import StringIO
@@ -61,7 +59,7 @@
 )
 from . import py_source_parser
 from .block_parser import BlockParser
-
+from .docs_resolv import _write_code_obj
 from .notebook import jupyter_notebook, save_notebook
 from .interactive_example import (
     _add_jupyterlite_badge_logo,
@@ -498,7 +496,7 @@ def generate_dir_rst(
     subsection_readme_fname = _get_readme(src_dir, gallery_conf)
     have_index_rst = False
     if subsection_readme_fname:
-        with codecs.open(subsection_readme_fname, "r", encoding="utf-8") as fid:
+        with open(subsection_readme_fname, "r", encoding="utf-8") as fid:
             subsection_readme_content = fid.read()
             subsection_index_content += subsection_readme_content
     else:
@@ -612,7 +610,7 @@ def generate_dir_rst(
     subsection_index_path = None
     if gallery_conf["nested_sections"] is True and not have_index_rst:
         subsection_index_path = os.path.join(target_dir, "index.rst.new")
-        with codecs.open(subsection_index_path, "w", encoding="utf-8") as (findex):
+        with open(subsection_index_path, "w", encoding="utf-8") as (findex):
             findex.write(
                 "\n\n.. _sphx_glr_{}:\n\n".format(head_ref.replace(os.path.sep, "_"))
             )
@@ -1324,10 +1322,7 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
     ref_regex = _make_ref_regex(gallery_conf["default_role"])
     example_code_obj = identify_names(script_blocks, ref_regex, global_variables, node)
     if example_code_obj:
-        codeobj_fname = target_file.with_name(target_file.stem + "_codeobj.pickle.new")
-        with open(codeobj_fname, "wb") as fid:
-            pickle.dump(example_code_obj, fid, pickle.HIGHEST_PROTOCOL)
-        _replace_md5(codeobj_fname)
+        _write_code_obj(target_file, example_code_obj)
     exclude_regex = gallery_conf["exclude_implicit_doc_regex"]
     backrefs = {
         "{module_short}.{name}".format(**cobj)
@@ -1539,7 +1534,7 @@ def save_rst_example(
         example_rst += SPHX_GLR_SIG
 
     write_file_new = example_file.with_suffix(".rst.new")
-    with codecs.open(write_file_new, "w", encoding="utf-8") as f:
+    with open(write_file_new, "w", encoding="utf-8") as f:
         f.write(example_rst)
     # make it read-only so that people don't try to edit it
     mode = os.stat(write_file_new).st_mode
diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index adf36857b..fcd249439 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -487,9 +487,7 @@ def test_embed_links_and_styles(sphinx_app):
     assert re.match(want_warn, lines, re.DOTALL) is not None
     sys.stdout.write(lines)
 
-    example_file = op.join(examples_dir, "plot_pickle.html")
-    with codecs.open(example_file, "r", "utf-8") as fid:
-        lines = fid.read()
+    lines = (Path(examples_dir) / "plot_pickle.html").read_text("utf-8")
     assert "joblib.Parallel.html" in lines
 
 
@@ -645,10 +643,10 @@ def test_rebuild(tmpdir_factory, sphinx_app):
         for f in os.listdir(op.join(old_src_dir, "auto_examples"))
         if f.endswith(".rst")
     )
-    generated_pickle_0 = sorted(
+    generated_json_0 = sorted(
         op.join(old_src_dir, "auto_examples", f)
         for f in os.listdir(op.join(old_src_dir, "auto_examples"))
-        if f.endswith(".pickle")
+        if f.endswith(".json")
     )
     copied_py_0 = sorted(
         op.join(old_src_dir, "auto_examples", f)
@@ -663,7 +661,7 @@ def test_rebuild(tmpdir_factory, sphinx_app):
     assert len(generated_modules_0) > 0
     assert len(generated_backrefs_0) > 0
     assert len(generated_rst_0) > 0
-    assert len(generated_pickle_0) > 0
+    assert len(generated_json_0) > 0
     assert len(copied_py_0) > 0
     assert len(copied_ipy_0) > 0
     assert len(sphinx_app.config.sphinx_gallery_conf["stale_examples"]) == 0
@@ -737,10 +735,10 @@ def test_rebuild(tmpdir_factory, sphinx_app):
         for f in os.listdir(op.join(new_app.srcdir, "auto_examples"))
         if f.endswith(".rst")
     )
-    generated_pickle_1 = sorted(
+    generated_json_1 = sorted(
         op.join(new_app.srcdir, "auto_examples", f)
         for f in os.listdir(op.join(new_app.srcdir, "auto_examples"))
-        if f.endswith(".pickle")
+        if f.endswith(".json")
     )
     copied_py_1 = sorted(
         op.join(new_app.srcdir, "auto_examples", f)
@@ -771,8 +769,8 @@ def test_rebuild(tmpdir_factory, sphinx_app):
     )
     _assert_mtimes(generated_rst_0, generated_rst_1, ignore=ignore)
 
-    # mtimes for pickles
-    _assert_mtimes(generated_pickle_0, generated_pickle_1)
+    # mtimes for jsons
+    _assert_mtimes(generated_json_0, generated_json_1)
 
     # mtimes for .py files (gh-395)
     _assert_mtimes(copied_py_0, copied_py_1)
@@ -796,7 +794,7 @@ def test_rebuild(tmpdir_factory, sphinx_app):
             generated_modules_0,
             generated_backrefs_0,
             generated_rst_0,
-            generated_pickle_0,
+            generated_json_0,
             copied_py_0,
             copied_ipy_0,
         )
@@ -811,7 +809,7 @@ def _rerun(
     generated_modules_0,
     generated_backrefs_0,
     generated_rst_0,
-    generated_pickle_0,
+    generated_json_0,
     copied_py_0,
     copied_ipy_0,
 ):
@@ -906,10 +904,10 @@ def _rerun(
         for f in os.listdir(op.join(new_app.srcdir, "auto_examples"))
         if f.endswith(".rst")
     )
-    generated_pickle_1 = sorted(
+    generated_json_1 = sorted(
         op.join(new_app.srcdir, "auto_examples", f)
         for f in os.listdir(op.join(new_app.srcdir, "auto_examples"))
-        if f.endswith(".pickle")
+        if f.endswith(".json")
     )
     copied_py_1 = sorted(
         op.join(new_app.srcdir, "auto_examples", f)
@@ -945,9 +943,9 @@ def _rerun(
     if not bad:
         _assert_mtimes(generated_rst_0, generated_rst_1, different, ignore)
 
-        # mtimes for pickles
+        # mtimes for jsons
         use_different = () if how == "run_stale" else different
-        _assert_mtimes(generated_pickle_0, generated_pickle_1, ignore=ignore)
+        _assert_mtimes(generated_json_0, generated_json_1, ignore=ignore)
 
         # mtimes for .py files (gh-395)
         _assert_mtimes(copied_py_0, copied_py_1, different=use_different)
diff --git a/sphinx_gallery/tests/tinybuild/doc/conf.py b/sphinx_gallery/tests/tinybuild/doc/conf.py
index 2bd853b83..c702bf7de 100644
--- a/sphinx_gallery/tests/tinybuild/doc/conf.py
+++ b/sphinx_gallery/tests/tinybuild/doc/conf.py
@@ -73,7 +73,7 @@
         "../examples/future/plot_future_imports_broken.py",
         "../examples/plot_scraper_broken.py",
     ],
-    "show_memory": True,
+    "show_memory": False,
     "compress_images": ("images", "thumbnails"),
     "junit": op.join("sphinx-gallery", "junit-results.xml"),
     "matplotlib_animations": (True, "mp4"),
diff --git a/sphinx_gallery/utils.py b/sphinx_gallery/utils.py
index f6e08af35..f8f7add74 100644
--- a/sphinx_gallery/utils.py
+++ b/sphinx_gallery/utils.py
@@ -156,11 +156,19 @@ def _replace_md5(fname_new, fname_old=None, method="move", mode="b"):
     if os.path.isfile(fname_old):
         if get_md5sum(fname_old, mode) == get_md5sum(fname_new, mode):
             replace = False
+            if str(fname_old).endswith(".json"):
+                import sys
+
+                print(f"Keeping {fname_old}", file=sys.__stderr__)
             if method == "move":
                 os.remove(fname_new)
         else:
             logger.debug(f"Replacing stale {fname_old} with {fname_new}")
     if replace:
+        if str(fname_old).endswith(".json") and os.path.isfile(fname_old):
+            step = os.path.basename(fname_old)
+            copyfile(fname_old, f"/Users/larsoner/Desktop/old_{step}")
+            copyfile(fname_new, f"/Users/larsoner/Desktop/new_{step}")
         if method == "move":
             move(fname_new, fname_old)
         else:

From 5f29ec6f613edc9382945f3efd55f6863fe3b988 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Wed, 5 Jun 2024 21:35:05 -0700
Subject: [PATCH 15/40] FIX: Debug

---
 sphinx_gallery/gen_gallery.py | 5 -----
 sphinx_gallery/utils.py       | 4 ----
 2 files changed, 9 deletions(-)

diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
index 3132e0fca..91453dc23 100644
--- a/sphinx_gallery/gen_gallery.py
+++ b/sphinx_gallery/gen_gallery.py
@@ -1284,11 +1284,6 @@ def write_junit_xml(gallery_conf, target_dir, costs):
     failing_as_expected, failing_unexpectedly, passing_unexpectedly = _parse_failures(
         gallery_conf
     )
-    import sys
-
-    print(f"{failing_as_expected=}", file=sys.__stderr__)
-    print(f"{failing_unexpectedly=}", file=sys.__stderr__)
-    print(f"{passing_unexpectedly=}", file=sys.__stderr__)
     n_tests = 0
     n_failures = 0
     n_skips = 0
diff --git a/sphinx_gallery/utils.py b/sphinx_gallery/utils.py
index f8f7add74..2420b3008 100644
--- a/sphinx_gallery/utils.py
+++ b/sphinx_gallery/utils.py
@@ -156,10 +156,6 @@ def _replace_md5(fname_new, fname_old=None, method="move", mode="b"):
     if os.path.isfile(fname_old):
         if get_md5sum(fname_old, mode) == get_md5sum(fname_new, mode):
             replace = False
-            if str(fname_old).endswith(".json"):
-                import sys
-
-                print(f"Keeping {fname_old}", file=sys.__stderr__)
             if method == "move":
                 os.remove(fname_new)
         else:

From c628c82a99ed4758cd01ef87914c1936890966c3 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Wed, 5 Jun 2024 21:36:47 -0700
Subject: [PATCH 16/40] FIX: Compact

---
 sphinx_gallery/docs_resolv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sphinx_gallery/docs_resolv.py b/sphinx_gallery/docs_resolv.py
index f0b304782..936c41304 100644
--- a/sphinx_gallery/docs_resolv.py
+++ b/sphinx_gallery/docs_resolv.py
@@ -342,7 +342,7 @@ def _write_code_obj(target_file, example_code_obj):
             example_code_obj,
             fid,
             ensure_ascii=False,
-            indent=2,
+            indent=1,
             check_circular=False,
         )
     _replace_md5(codeobj_fname)

From 209e8039062de9997982d399317582518f36ff2d Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Wed, 5 Jun 2024 21:41:11 -0700
Subject: [PATCH 17/40] FIX: Dont do it

---
 doc/conf.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/conf.py b/doc/conf.py
index 9fc549275..8586907bd 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -418,7 +418,6 @@ def setup(app):
     "image_srcset": ["2x"],
     "nested_sections": True,
     "show_api_usage": True,
-    "parallel": True,  # can run with -j2 for example for speed
 }
 
 # Remove matplotlib agg warnings from generated doc when using plt.show

From ac559f888459b558b8e61aa850e963f07e937fd9 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Wed, 5 Jun 2024 21:47:37 -0700
Subject: [PATCH 18/40] FIX: Cruft

---
 sphinx_gallery/tests/test_full.py | 2 +-
 sphinx_gallery/utils.py           | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index fcd249439..a914c66b5 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -596,7 +596,7 @@ def _assert_mtimes(list_orig, list_new, different=(), ignore=()):
     good_sphinx = Version(sphinx_version) >= Version("4.1")
     for orig, new in zip(list_orig, list_new):
         check_name = op.splitext(op.basename(orig))[0]
-        if check_name.endswith("_codeobj"):
+        if check_name.endswith(".codeobj"):
             check_name = check_name[:-8]
         if check_name in different:
             if good_sphinx:
diff --git a/sphinx_gallery/utils.py b/sphinx_gallery/utils.py
index 2420b3008..f6e08af35 100644
--- a/sphinx_gallery/utils.py
+++ b/sphinx_gallery/utils.py
@@ -161,10 +161,6 @@ def _replace_md5(fname_new, fname_old=None, method="move", mode="b"):
         else:
             logger.debug(f"Replacing stale {fname_old} with {fname_new}")
     if replace:
-        if str(fname_old).endswith(".json") and os.path.isfile(fname_old):
-            step = os.path.basename(fname_old)
-            copyfile(fname_old, f"/Users/larsoner/Desktop/old_{step}")
-            copyfile(fname_new, f"/Users/larsoner/Desktop/new_{step}")
         if method == "move":
             move(fname_new, fname_old)
         else:

From 7ccfda7f4d116e9ee5e64bf4b3b542a63457f153 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Wed, 5 Jun 2024 22:10:17 -0700
Subject: [PATCH 19/40] FIX: Really stabilize

---
 sphinx_gallery/docs_resolv.py |  3 ++-
 sphinx_gallery/gen_rst.py     |  2 +-
 sphinx_gallery/utils.py       | 19 +++++++++++++++++--
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/sphinx_gallery/docs_resolv.py b/sphinx_gallery/docs_resolv.py
index 936c41304..88afeb60e 100644
--- a/sphinx_gallery/docs_resolv.py
+++ b/sphinx_gallery/docs_resolv.py
@@ -341,11 +341,12 @@ def _write_code_obj(target_file, example_code_obj):
         json.dump(
             example_code_obj,
             fid,
+            sort_keys=True,
             ensure_ascii=False,
             indent=1,
             check_circular=False,
         )
-    _replace_md5(codeobj_fname)
+    _replace_md5(codeobj_fname, check="json")
 
 
 def _embed_code_links(app, gallery_conf, gallery_dir):
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index df00ca629..e80a1e19b 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -1199,7 +1199,7 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
     src_file = os.path.normpath(os.path.join(src_dir, fname))
     out_vars = dict()
     target_file = Path(target_dir) / fname
-    _replace_md5(src_file, target_file, "copy", mode="t")
+    _replace_md5(src_file, target_file, method="copy", mode="t")
 
     if fname.endswith(".py"):
         parser = py_source_parser
diff --git a/sphinx_gallery/utils.py b/sphinx_gallery/utils.py
index f6e08af35..2d21ca248 100644
--- a/sphinx_gallery/utils.py
+++ b/sphinx_gallery/utils.py
@@ -6,8 +6,11 @@
 # Author: Eric Larson
 # License: 3-clause BSD
 
+from functools import partial
 import hashlib
+import json
 import os
+from pathlib import Path
 import re
 from shutil import move, copyfile
 import subprocess
@@ -146,7 +149,7 @@ def get_md5sum(src_file, mode="b"):
         return hashlib.md5(src_content).hexdigest()
 
 
-def _replace_md5(fname_new, fname_old=None, method="move", mode="b"):
+def _replace_md5(fname_new, fname_old=None, *, method="move", mode="b", check="md5"):
     fname_new = str(fname_new)  # convert possible Path
     assert method in ("move", "copy")
     if fname_old is None:
@@ -154,7 +157,19 @@ def _replace_md5(fname_new, fname_old=None, method="move", mode="b"):
         fname_old = os.path.splitext(fname_new)[0]
     replace = True
     if os.path.isfile(fname_old):
-        if get_md5sum(fname_old, mode) == get_md5sum(fname_new, mode):
+        if check == "md5":  # default
+            func = partial(get_md5sum, mode=mode)
+        else:
+            assert check == "json"
+
+            def func(x):
+                return json.loads(Path(x).read_text("utf-8"))
+
+        try:
+            equiv = func(fname_old) == func(fname_new)
+        except Exception:  # e.g., old JSON file is a problem
+            equiv = False
+        if equiv:
             replace = False
             if method == "move":
                 os.remove(fname_new)

From f06214b1cd60b0375064f1be5af55ec1b92a7282 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Thu, 20 Jun 2024 10:45:22 -0400
Subject: [PATCH 20/40] FIX: What

---
 sphinx_gallery/tests/test_gen_rst.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sphinx_gallery/tests/test_gen_rst.py b/sphinx_gallery/tests/test_gen_rst.py
index a85071c5b..c6cb2c9cd 100644
--- a/sphinx_gallery/tests/test_gen_rst.py
+++ b/sphinx_gallery/tests/test_gen_rst.py
@@ -488,7 +488,7 @@ def _generate_rst(gallery_conf, fname, content):
         gallery_conf["gallery_dir"],
         gallery_conf,
         set(),
-        include_toctree=False,
+        is_subsection=False,
     )
     # read rst file and check if it contains code output
     rst_fname = os.path.splitext(fname)[0] + ".rst"

From 966e27171459cb7d017905a1cf1f193fa43844fc Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Thu, 20 Jun 2024 12:46:17 -0400
Subject: [PATCH 21/40] FIX: more complete

---
 sphinx_gallery/tests/test_full.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index 944906491..ebf595530 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -15,6 +15,7 @@
 import json
 
 import lxml.html
+import lxml.etree
 from packaging.version import Version
 
 from sphinx import __version__ as sphinx_version
@@ -186,11 +187,20 @@ def test_junit(sphinx_app, tmpdir):
     out_dir = sphinx_app.outdir
     junit_file = op.join(out_dir, "sphinx-gallery", "junit-results.xml")
     assert op.isfile(junit_file)
-    with codecs.open(junit_file, "r", "utf-8") as fid:
+    with open(junit_file, "rb") as fid:
         contents = fid.read()
-    assert contents.startswith("<?xml")
-    assert 'errors="0" failures="0"' in contents
-    assert f'tests="{N_EXAMPLES}"' in contents
+    suite = lxml.etree.fromstring(contents)
+    want = dict(
+        errors="0",
+        failures="0",
+        skipped="2",
+        tests=f"{N_EXAMPLES}",
+        name="sphinx-gallery",
+    )
+    got = dict(suite.attrib)
+    del got["time"]
+    assert got == want
+    contents = contents.decode("utf-8")
     assert "local_module" not in contents  # it's not actually run as an ex
     assert "expected example failure" in contents
     assert "<failure message" not in contents
@@ -224,11 +234,15 @@ def test_junit(sphinx_app, tmpdir):
             app.build(False, [])
     junit_file = op.join(new_out_dir, "sphinx-gallery", "junit-results.xml")
     assert op.isfile(junit_file)
-    with codecs.open(junit_file, "r", "utf-8") as fid:
+    with open(junit_file, "rb") as fid:
         contents = fid.read()
-    assert 'errors="0" failures="2"' in contents
+    suite = lxml.etree.fromstring(contents)
     # this time we only ran the stale files
-    assert f'tests="{N_FAILING + 1}"' in contents
+    want.update(failures="2", skipped="1", tests="3")
+    got = dict(suite.attrib)
+    del got["time"]
+    assert got == want
+    contents = contents.decode("utf-8")
     assert '<failure message="RuntimeError: Forcing' in contents
     assert "Passed even though it was marked to fail" in contents
 

From 0db2af6ecf199772f63335f2771d51a386a4ed3c Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Thu, 20 Jun 2024 13:37:57 -0400
Subject: [PATCH 22/40] FIX: More mode

---
 sphinx_gallery/backreferences.py     |  7 ++++---
 sphinx_gallery/docs_resolv.py        |  6 +++---
 sphinx_gallery/gen_rst.py            | 22 ++++++++++----------
 sphinx_gallery/tests/test_gen_rst.py | 30 +++++++++++-----------------
 sphinx_gallery/utils.py              |  7 ++++++-
 5 files changed, 37 insertions(+), 35 deletions(-)

diff --git a/sphinx_gallery/backreferences.py b/sphinx_gallery/backreferences.py
index ff73e0bb7..722044df2 100644
--- a/sphinx_gallery/backreferences.py
+++ b/sphinx_gallery/backreferences.py
@@ -17,7 +17,7 @@
 import sphinx.util
 
 from .scrapers import _find_image_ext
-from .utils import _replace_md5
+from .utils import _replace_md5, _W_KW
 
 
 THUMBNAIL_PARENT_DIV = """
@@ -390,7 +390,8 @@ def _write_backreferences(
             f"{backref}.examples.new",
         )
         seen = backref in seen_backrefs
-        with open(include_path, "a" if seen else "w", encoding="utf-8") as ex_file:
+        mode = "a" if seen else "w"
+        with open(include_path, mode, **_W_KW) as ex_file:
             if not seen:
                 # Be aware that if the number of lines of this heading changes,
                 #   the minigallery directive should be modified accordingly
@@ -429,7 +430,7 @@ def _finalize_backreferences(seen_backrefs, gallery_conf):
         if os.path.isfile(path):
             # Close div containing all thumbnails
             # (it was open in _write_backreferences)
-            with open(path, "a", encoding="utf-8") as ex_file:
+            with open(path, "a", **_W_KW) as ex_file:
                 ex_file.write(THUMBNAIL_PARENT_DIV_CLOSE)
             _replace_md5(path, mode="t")
         else:
diff --git a/sphinx_gallery/docs_resolv.py b/sphinx_gallery/docs_resolv.py
index 88afeb60e..853d5acbc 100644
--- a/sphinx_gallery/docs_resolv.py
+++ b/sphinx_gallery/docs_resolv.py
@@ -18,7 +18,7 @@
 from sphinx.search import js_index
 import sphinx.util
 
-from .utils import status_iterator, _replace_md5
+from .utils import status_iterator, _replace_md5, _W_KW
 
 
 logger = sphinx.util.logging.getLogger("sphinx-gallery")
@@ -337,7 +337,7 @@ def _get_intersphinx_inventory(app):
 # _embed_code_links
 def _write_code_obj(target_file, example_code_obj):
     codeobj_fname = target_file.with_name(target_file.stem + ".codeobj.json.new")
-    with open(codeobj_fname, "w", encoding="utf-8") as fid:
+    with open(codeobj_fname, "w", **_W_KW) as fid:
         json.dump(
             example_code_obj,
             fid,
@@ -490,7 +490,7 @@ def substitute_link(match):
         if len(str_repl) > 0:
             with open(full_fname, "r", encoding="utf-8") as fid:
                 lines_in = fid.readlines()
-            with open(full_fname, "w", encoding="utf-8") as fid:
+            with open(full_fname, "w", **_W_KW) as fid:
                 for line in lines_in:
                     line_out = regex.sub(substitute_link, line)
                     fid.write(line_out)
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 50f20f8dc..4773ff9c2 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -49,6 +49,7 @@
     _replace_md5,
     optipng,
     status_iterator,
+    _W_KW,
 )
 from . import glr_path_static, py_source_parser
 from .backreferences import (
@@ -341,16 +342,16 @@ def extract_intro_and_title(filename, docstring):
 
 def md5sum_is_current(src_file, mode="b"):
     """Checks whether src_file has the same md5 hash as the one on disk."""
-    src_md5 = get_md5sum(src_file, mode)
+    src_md5 = get_md5sum(src_file, mode=mode)
 
     src_md5_file = str(src_file) + ".md5"
-    if os.path.exists(src_md5_file):
-        with open(src_md5_file) as file_checksum:
-            ref_md5 = file_checksum.read()
+    if not os.path.exists(src_md5_file):
+        return False
 
-        return src_md5 == ref_md5
+    with open(src_md5_file) as file_cs:
+        ref_md5 = file_cs.read()
 
-    return False
+    return src_md5 == ref_md5
 
 
 def save_thumbnail(image_path_template, src_file, script_vars, file_conf, gallery_conf):
@@ -472,7 +473,7 @@ def _write_subsection_index(
     if gallery_conf["nested_sections"] and not user_index_rst and is_subsection:
         index_path = os.path.join(target_dir, "index.rst.new")
         head_ref = os.path.relpath(target_dir, gallery_conf["src_dir"])
-        with open(index_path, "w", encoding="utf-8") as (findex):
+        with open(index_path, "w", **_W_KW) as findex:
             findex.write(
                 "\n\n.. _sphx_glr_{}:\n\n".format(head_ref.replace(os.sep, "_"))
             )
@@ -1172,8 +1173,8 @@ def execute_script(script_blocks, script_vars, gallery_conf, file_conf):
         script_vars["memory_delta"] -= memory_start
         # Write md5 checksum if the example was meant to run (no-plot
         # shall not cache md5sum) and has built correctly
-        with open(script_vars["target_file"] + ".md5", "w") as file_checksum:
-            file_checksum.write(get_md5sum(script_vars["target_file"], "t"))
+        with open(script_vars["target_file"] + ".md5", "w") as file_cs:
+            file_cs.write(get_md5sum(script_vars["target_file"], mode="t"))
         script_vars["passing"] = True
 
     return output_blocks, time_elapsed
@@ -1298,6 +1299,7 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
 
     if md5sum_is_current(target_file, mode="t"):
         do_return = True
+        logger.debug(f"md5sum is current: {target_file}")
         if executable:
             if gallery_conf["run_stale_examples"]:
                 do_return = False
@@ -1574,7 +1576,7 @@ def save_rst_example(
         example_rst += SPHX_GLR_SIG
 
     write_file_new = example_file.with_suffix(".rst.new")
-    with open(write_file_new, "w", encoding="utf-8") as f:
+    with open(write_file_new, "w", **_W_KW) as f:
         f.write(example_rst)
     # make it read-only so that people don't try to edit it
     mode = os.stat(write_file_new).st_mode
diff --git a/sphinx_gallery/tests/test_gen_rst.py b/sphinx_gallery/tests/test_gen_rst.py
index c6cb2c9cd..22517d288 100644
--- a/sphinx_gallery/tests/test_gen_rst.py
+++ b/sphinx_gallery/tests/test_gen_rst.py
@@ -376,26 +376,20 @@ def test_extract_intro_and_title():
         ["t", "ea8a570e9f3afc0a7c3f2a17a48b8047"],
     ),
 )
-def test_md5sums(mode, expected_md5):
+def test_md5sums(mode, expected_md5, tmp_path):
     """Test md5sum check functions work on know file content."""
     file_content = b"Local test\r\n"
-    with tempfile.NamedTemporaryFile("wb", delete=False) as f:
-        f.write(file_content)
-    try:
-        file_md5 = sg.get_md5sum(f.name, mode)
-        # verify correct md5sum
-        assert file_md5 == expected_md5
-        # False because is a new file
-        assert not sg.md5sum_is_current(f.name)
-        # Write md5sum to file to check is current
-        with open(f.name + ".md5", "w") as file_checksum:
-            file_checksum.write(file_md5)
-        try:
-            assert sg.md5sum_is_current(f.name, mode)
-        finally:
-            os.remove(f.name + ".md5")
-    finally:
-        os.remove(f.name)
+    fname = tmp_path / "test"
+    fname.write_bytes(file_content)
+    file_md5 = sg.get_md5sum(fname, mode)
+    # verify correct md5sum
+    assert file_md5 == expected_md5, mode
+    # False because is a new file
+    assert not sg.md5sum_is_current(fname), mode
+    # Write md5sum to file to check is current
+    with open(str(fname) + ".md5", "w") as file_checksum:
+        file_checksum.write(file_md5)
+    assert sg.md5sum_is_current(fname, mode), mode
 
 
 @pytest.mark.parametrize(
diff --git a/sphinx_gallery/utils.py b/sphinx_gallery/utils.py
index d511176dc..b038d0111 100644
--- a/sphinx_gallery/utils.py
+++ b/sphinx_gallery/utils.py
@@ -28,6 +28,10 @@
 logger = sphinx.util.logging.getLogger("sphinx-gallery")
 
 
+# Text writing kwargs for builtins.open
+_W_KW = dict(encoding="utf-8", newline="\n")
+
+
 def _get_image():
     try:
         from PIL import Image
@@ -142,7 +146,8 @@ def get_md5sum(src_file, mode="b"):
         kwargs = {"errors": "surrogateescape", "encoding": "utf-8"}
     else:
         kwargs = {}
-    with open(src_file, "r" + mode, **kwargs) as src_data:
+    # Universal newline mode is intentional here
+    with open(src_file, f"r{mode}", **kwargs) as src_data:
         src_content = src_data.read()
         if mode == "t":
             src_content = src_content.encode(**kwargs)

From 95041e64d109f7b197ad6f045b2dafa63a0001d6 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Thu, 20 Jun 2024 19:02:09 -0400
Subject: [PATCH 23/40] FIX: Closer

---
 sphinx_gallery/tests/test_full.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index ebf595530..f51d32e81 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -241,6 +241,13 @@ def test_junit(sphinx_app, tmpdir):
     want.update(failures="2", skipped="1", tests="3")
     got = dict(suite.attrib)
     del got["time"]
+    assert len(suite) == 3
+    assert suite[0].attrib["classname"] == "plot_numpy_matplotlib"
+    assert suite[0][0].tag == "failure", suite[0].attrib["classname"]
+    assert suite[1].attrib["classname"] == "plot_scraper_broken"
+    assert suite[1][0].tag == "skipped", suite[1].attrib["classname"]
+    assert suite[2].attrib["classname"] == "plot_future_imports_broken"
+    assert suite[2][0].tag == "failure", suite[2].attrib["classname"]
     assert got == want
     contents = contents.decode("utf-8")
     assert '<failure message="RuntimeError: Forcing' in contents

From 1114b83e6a1f90519e87d2bb96c785ea722a33b0 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Thu, 20 Jun 2024 21:12:53 -0400
Subject: [PATCH 24/40] WIP: Debug

---
 sphinx_gallery/tests/test_full.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index f51d32e81..be57611ab 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -183,7 +183,7 @@ def test_optipng(sphinx_app):
     assert "optipng version" not in status.lower()  # catch the --version
 
 
-def test_junit(sphinx_app, tmpdir):
+def test_junit(sphinx_app, tmp_path):
     out_dir = sphinx_app.outdir
     junit_file = op.join(out_dir, "sphinx-gallery", "junit-results.xml")
     assert op.isfile(junit_file)
@@ -205,7 +205,7 @@ def test_junit(sphinx_app, tmpdir):
     assert "expected example failure" in contents
     assert "<failure message" not in contents
     src_dir = sphinx_app.srcdir
-    new_src_dir = op.join(str(tmpdir), "src")
+    new_src_dir = op.join(str(tmp_path), "src")
     shutil.copytree(op.join(src_dir, "../"), new_src_dir)
     del src_dir
     new_src_dir = op.join(new_src_dir, "doc")
@@ -226,7 +226,7 @@ def test_junit(sphinx_app, tmpdir):
             new_out_dir,
             new_toctree_dir,
             buildername="html",
-            status=StringIO(),
+            verbosity=1,
         )
         # need to build within the context manager
         # for automodule and backrefs to work
@@ -238,20 +238,25 @@ def test_junit(sphinx_app, tmpdir):
         contents = fid.read()
     suite = lxml.etree.fromstring(contents)
     # this time we only ran the stale files
+    from pprint import pprint
+    from sphinx_gallery.gen_gallery import _parse_failures
+
+    pprint(list(app.config.sphinx_gallery_conf["failing_examples"]))
+    pprint(_parse_failures(app.config.sphinx_gallery_conf))
     want.update(failures="2", skipped="1", tests="3")
     got = dict(suite.attrib)
     del got["time"]
     assert len(suite) == 3
     assert suite[0].attrib["classname"] == "plot_numpy_matplotlib"
     assert suite[0][0].tag == "failure", suite[0].attrib["classname"]
+    assert suite[0][0].attrib["message"].startswith("RuntimeError: Forcing")
     assert suite[1].attrib["classname"] == "plot_scraper_broken"
     assert suite[1][0].tag == "skipped", suite[1].attrib["classname"]
     assert suite[2].attrib["classname"] == "plot_future_imports_broken"
     assert suite[2][0].tag == "failure", suite[2].attrib["classname"]
+    assert suite[2][0].attrib["message"] == "Passed even though it was marked to fail"
     assert got == want
-    contents = contents.decode("utf-8")
-    assert '<failure message="RuntimeError: Forcing' in contents
-    assert "Passed even though it was marked to fail" in contents
+    raise RuntimeError
 
 
 def test_run_sphinx(sphinx_app):

From 13c65e9996c525936bfa43873d428654ab35e61b Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Fri, 21 Jun 2024 10:18:02 -0400
Subject: [PATCH 25/40] WIP: More

---
 sphinx_gallery/gen_rst.py          | 2 ++
 sphinx_gallery/py_source_parser.py | 6 ++----
 sphinx_gallery/tests/test_full.py  | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 4773ff9c2..39f012da5 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -1003,6 +1003,8 @@ def execute_code_block(
     )
     need_save_figures = match is None
 
+    print(f'{script_vars["src_file"]}\n{block}\n', file=sys.__stderr__)
+
     try:
         # The "compile" step itself can fail on a SyntaxError, so just prepend
         # newlines to get the correct failing line to show up in the traceback
diff --git a/sphinx_gallery/py_source_parser.py b/sphinx_gallery/py_source_parser.py
index df456ea0f..d8655cb7d 100644
--- a/sphinx_gallery/py_source_parser.py
+++ b/sphinx_gallery/py_source_parser.py
@@ -4,7 +4,6 @@
 # Author: Óscar Nájera
 
 from collections import namedtuple
-import codecs
 import ast
 from io import BytesIO
 import re
@@ -57,10 +56,9 @@ def parse_source_file(filename):
     node : AST node
     content : utf-8 encoded string
     """
-    with codecs.open(filename, "r", "utf-8") as fid:
+    # builtin open automatically converts \r\n to \n
+    with open(filename, "r", encoding="utf-8") as fid:
         content = fid.read()
-    # change from Windows format to UNIX for uniformity
-    content = content.replace("\r\n", "\n")
 
     try:
         node = ast.parse(content)
diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index be57611ab..fece77fa0 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -219,6 +219,7 @@ def test_junit(sphinx_app, tmp_path):
     shutil.move(passing_fname, passing_fname + ".temp")
     shutil.move(failing_fname, passing_fname)
     shutil.move(passing_fname + ".temp", failing_fname)
+    print(sphinx_app._status.getvalue())
     with docutils_namespace():
         app = Sphinx(
             new_src_dir,
@@ -235,8 +236,7 @@ def test_junit(sphinx_app, tmp_path):
     junit_file = op.join(new_out_dir, "sphinx-gallery", "junit-results.xml")
     assert op.isfile(junit_file)
     with open(junit_file, "rb") as fid:
-        contents = fid.read()
-    suite = lxml.etree.fromstring(contents)
+        suite = lxml.etree.fromstring(fid.read())
     # this time we only ran the stale files
     from pprint import pprint
     from sphinx_gallery.gen_gallery import _parse_failures

From c7a22cd31cbfab70d3872242aaced5a242a898ba Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Fri, 21 Jun 2024 10:28:44 -0400
Subject: [PATCH 26/40] FIX: More debug

---
 sphinx_gallery/gen_rst.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 39f012da5..492d82f09 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -1003,8 +1003,6 @@ def execute_code_block(
     )
     need_save_figures = match is None
 
-    print(f'{script_vars["src_file"]}\n{block}\n', file=sys.__stderr__)
-
     try:
         # The "compile" step itself can fail on a SyntaxError, so just prepend
         # newlines to get the correct failing line to show up in the traceback
@@ -1029,8 +1027,14 @@ def execute_code_block(
             images_rst = save_figures(block, script_vars, gallery_conf)
         else:
             images_rst = ""
-    except Exception:
+    except Exception as e:
         logging_tee.restore_std()
+        print("*" * 120, file=sys.__stdout__)
+        print(f'{script_vars["src_file"]}\n{block}\n', file=sys.__stdout__)
+        print(
+            "".join(traceback.format_exception(type(e), e, e.__traceback__)),
+            file=sys.__stdout__,
+        )
         except_rst = handle_exception(
             sys.exc_info(), src_file, script_vars, gallery_conf
         )

From f5535d5b562c7a1ee2554e61eb691b03381b1283 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Fri, 21 Jun 2024 10:44:16 -0400
Subject: [PATCH 27/40] WIP: More

---
 sphinx_gallery/gen_rst.py         | 3 ++-
 sphinx_gallery/tests/test_full.py | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 492d82f09..c1513c4e0 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -1030,7 +1030,8 @@ def execute_code_block(
     except Exception as e:
         logging_tee.restore_std()
         print("*" * 120, file=sys.__stdout__)
-        print(f'{script_vars["src_file"]}\n{block}\n', file=sys.__stdout__)
+        # print(f'{script_vars["src_file"]}\n{block}\n', file=sys.__stdout__)
+        print(f'{script_vars["src_file"]}\n{sys.path=}\n', file=sys.__stdout__)
         print(
             "".join(traceback.format_exception(type(e), e, e.__traceback__)),
             file=sys.__stdout__,
diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index fece77fa0..eba4ebeb7 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -211,6 +211,7 @@ def test_junit(sphinx_app, tmp_path):
     new_src_dir = op.join(new_src_dir, "doc")
     new_out_dir = op.join(new_src_dir, "_build", "html")
     new_toctree_dir = op.join(new_src_dir, "_build", "toctrees")
+    # swap numpy_matplotlib (passing) with future_imports_broken (failing)
     passing_fname = op.join(new_src_dir, "../examples", "plot_numpy_matplotlib.py")
     failing_fname = op.join(
         new_src_dir, "../examples", "future", "plot_future_imports_broken.py"

From 1dc224d443940ed57a236a36d08d313cc2419761 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Fri, 21 Jun 2024 11:19:14 -0400
Subject: [PATCH 28/40] FIX: Drop Sphinx 4

---
 .github/workflows/tests.yml       |  4 ---
 pyproject.toml                    |  2 +-
 sphinx_gallery/gen_rst.py         |  9 +-----
 sphinx_gallery/tests/test_full.py | 54 ++++++++++++++++---------------
 4 files changed, 30 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8364ff245..54fc9d2a5 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -32,10 +32,6 @@ jobs:
             sphinx_version: dev
             distrib: pip
             locale: C
-          - os: ubuntu-latest  # oldest supported Python and Sphinx
-            python: '3.8'
-            sphinx_version: '4'
-            distrib: mamba
           - os: ubuntu-latest
             python: '3.11'
             sphinx_version: '5'
diff --git a/pyproject.toml b/pyproject.toml
index bcfddb721..e8b2ae5fa 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,7 +31,7 @@ dynamic = [
 ]
 dependencies = [
   "pillow",
-  "sphinx>=4",
+  "sphinx>=5",
 ]
 optional-dependencies.animations = [
   "sphinxcontrib-video",
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index c1513c4e0..4773ff9c2 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -1027,15 +1027,8 @@ def execute_code_block(
             images_rst = save_figures(block, script_vars, gallery_conf)
         else:
             images_rst = ""
-    except Exception as e:
+    except Exception:
         logging_tee.restore_std()
-        print("*" * 120, file=sys.__stdout__)
-        # print(f'{script_vars["src_file"]}\n{block}\n', file=sys.__stdout__)
-        print(f'{script_vars["src_file"]}\n{sys.path=}\n', file=sys.__stdout__)
-        print(
-            "".join(traceback.format_exception(type(e), e, e.__traceback__)),
-            file=sys.__stdout__,
-        )
         except_rst = handle_exception(
             sys.exc_info(), src_file, script_vars, gallery_conf
         )
diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index eba4ebeb7..7bc713884 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -80,8 +80,8 @@ def _sphinx_app(tmpdir_factory, buildername):
     # Skip if numpy not installed
     pytest.importorskip("numpy")
 
-    temp_dir = (tmpdir_factory.getbasetemp() / f"root_{buildername}").strpath
-    src_dir = op.join(op.dirname(__file__), "tinybuild")
+    temp_dir = tmpdir_factory.getbasetemp() / f"root_{buildername}"
+    src_dir = Path(__file__).parent / "tinybuild"
 
     def ignore(src, names):
         return ("_build", "gen_modules", "auto_examples")
@@ -89,9 +89,9 @@ def ignore(src, names):
     shutil.copytree(src_dir, temp_dir, ignore=ignore)
     # For testing iteration, you can get similar behavior just doing `make`
     # inside the tinybuild/doc directory
-    conf_dir = op.join(temp_dir, "doc")
-    out_dir = op.join(conf_dir, "_build", buildername)
-    toctrees_dir = op.join(conf_dir, "_build", "toctrees")
+    conf_dir = temp_dir / "doc"
+    out_dir = conf_dir / "_build" / buildername
+    toctrees_dir = conf_dir / "_build" / "toctrees"
     # Avoid warnings about re-registration, see:
     # https://github.com/sphinx-doc/sphinx/issues/5038
     with docutils_namespace():
@@ -184,9 +184,10 @@ def test_optipng(sphinx_app):
 
 
 def test_junit(sphinx_app, tmp_path):
+    """Test junit output."""
     out_dir = sphinx_app.outdir
-    junit_file = op.join(out_dir, "sphinx-gallery", "junit-results.xml")
-    assert op.isfile(junit_file)
+    junit_file = out_dir / "sphinx-gallery" / "junit-results.xml"
+    assert junit_file.is_file()
     with open(junit_file, "rb") as fid:
         contents = fid.read()
     suite = lxml.etree.fromstring(contents)
@@ -205,22 +206,24 @@ def test_junit(sphinx_app, tmp_path):
     assert "expected example failure" in contents
     assert "<failure message" not in contents
     src_dir = sphinx_app.srcdir
-    new_src_dir = op.join(str(tmp_path), "src")
-    shutil.copytree(op.join(src_dir, "../"), new_src_dir)
+    new_root_dir = tmp_path / "src"
+    shutil.copytree(src_dir.parent, new_root_dir)
     del src_dir
-    new_src_dir = op.join(new_src_dir, "doc")
-    new_out_dir = op.join(new_src_dir, "_build", "html")
-    new_toctree_dir = op.join(new_src_dir, "_build", "toctrees")
+    new_src_dir = new_root_dir / "doc"
+    new_out_dir = new_src_dir / "_build" / "html"
+    new_toctree_dir = new_src_dir / "_build" / "toctrees"
+    new_examples_dir = new_src_dir.parent / "examples"
     # swap numpy_matplotlib (passing) with future_imports_broken (failing)
-    passing_fname = op.join(new_src_dir, "../examples", "plot_numpy_matplotlib.py")
-    failing_fname = op.join(
-        new_src_dir, "../examples", "future", "plot_future_imports_broken.py"
-    )
+    passing_fname = new_examples_dir / "plot_numpy_matplotlib.py"
+    failing_fname = new_examples_dir / "future" / "plot_future_imports_broken.py"
     print("Names", passing_fname, failing_fname)
-    shutil.move(passing_fname, passing_fname + ".temp")
+    shutil.move(passing_fname, passing_fname.with_suffix(".temp"))
     shutil.move(failing_fname, passing_fname)
-    shutil.move(passing_fname + ".temp", failing_fname)
-    print(sphinx_app._status.getvalue())
+    shutil.move(passing_fname.with_suffix(".temp"), failing_fname)
+    shutil.copyfile(
+        new_examples_dir / "local_module.py",
+        new_examples_dir / "future" / "local_module.py",
+    )
     with docutils_namespace():
         app = Sphinx(
             new_src_dir,
@@ -234,8 +237,8 @@ def test_junit(sphinx_app, tmp_path):
         # for automodule and backrefs to work
         with pytest.raises(ExtensionError, match="Here is a summary of the "):
             app.build(False, [])
-    junit_file = op.join(new_out_dir, "sphinx-gallery", "junit-results.xml")
-    assert op.isfile(junit_file)
+    junit_file = new_out_dir / "sphinx-gallery" / "junit-results.xml"
+    assert junit_file.is_file()
     with open(junit_file, "rb") as fid:
         suite = lxml.etree.fromstring(fid.read())
     # this time we only ran the stale files
@@ -257,20 +260,19 @@ def test_junit(sphinx_app, tmp_path):
     assert suite[2][0].tag == "failure", suite[2].attrib["classname"]
     assert suite[2][0].attrib["message"] == "Passed even though it was marked to fail"
     assert got == want
-    raise RuntimeError
 
 
 def test_run_sphinx(sphinx_app):
     """Test basic outputs."""
-    out_dir = str(sphinx_app.outdir)
+    out_dir = sphinx_app.outdir
     out_files = os.listdir(out_dir)
     assert "index.html" in out_files
     assert "auto_examples" in out_files
     assert "auto_examples_with_rst" in out_files
     assert "auto_examples_rst_index" in out_files
     assert "auto_examples_README_header" in out_files
-    generated_examples_dir = op.join(out_dir, "auto_examples")
-    assert op.isdir(generated_examples_dir)
+    generated_examples_dir = out_dir / "auto_examples"
+    assert generated_examples_dir.is_dir()
     # make sure that indices are properly being passed forward...
     files_to_check = [
         "auto_examples_rst_index/examp_subdir1/index.html",
@@ -278,7 +280,7 @@ def test_run_sphinx(sphinx_app):
         "auto_examples_rst_index/index.html",
     ]
     for f in files_to_check:
-        assert op.isfile(out_dir + "/" + f)
+        assert (out_dir / f).is_file()
     status = sphinx_app._status.getvalue()
     assert f"executed {N_GOOD} out of {N_EXAMPLES}" in status
     assert "after excluding 0" in status

From a9acc6571465f7a081115ed510ac9469fe79a314 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Fri, 21 Jun 2024 11:23:05 -0400
Subject: [PATCH 29/40] FIX: Path

---
 sphinx_gallery/tests/test_full.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index 7bc713884..88031af5b 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -186,7 +186,7 @@ def test_optipng(sphinx_app):
 def test_junit(sphinx_app, tmp_path):
     """Test junit output."""
     out_dir = sphinx_app.outdir
-    junit_file = out_dir / "sphinx-gallery" / "junit-results.xml"
+    junit_file = Path(out_dir) / "sphinx-gallery" / "junit-results.xml"
     assert junit_file.is_file()
     with open(junit_file, "rb") as fid:
         contents = fid.read()
@@ -205,7 +205,7 @@ def test_junit(sphinx_app, tmp_path):
     assert "local_module" not in contents  # it's not actually run as an ex
     assert "expected example failure" in contents
     assert "<failure message" not in contents
-    src_dir = sphinx_app.srcdir
+    src_dir = Path(sphinx_app.srcdir)
     new_root_dir = tmp_path / "src"
     shutil.copytree(src_dir.parent, new_root_dir)
     del src_dir
@@ -264,7 +264,7 @@ def test_junit(sphinx_app, tmp_path):
 
 def test_run_sphinx(sphinx_app):
     """Test basic outputs."""
-    out_dir = sphinx_app.outdir
+    out_dir = Path(sphinx_app.outdir)
     out_files = os.listdir(out_dir)
     assert "index.html" in out_files
     assert "auto_examples" in out_files

From dfc39cb635a4a9e0826abe24f15f12e2ad3f754c Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Fri, 21 Jun 2024 11:47:05 -0400
Subject: [PATCH 30/40] FIX: Remove cruft

---
 sphinx_gallery/tests/test_full.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/sphinx_gallery/tests/test_full.py b/sphinx_gallery/tests/test_full.py
index 88031af5b..4652f1a78 100644
--- a/sphinx_gallery/tests/test_full.py
+++ b/sphinx_gallery/tests/test_full.py
@@ -242,11 +242,6 @@ def test_junit(sphinx_app, tmp_path):
     with open(junit_file, "rb") as fid:
         suite = lxml.etree.fromstring(fid.read())
     # this time we only ran the stale files
-    from pprint import pprint
-    from sphinx_gallery.gen_gallery import _parse_failures
-
-    pprint(list(app.config.sphinx_gallery_conf["failing_examples"]))
-    pprint(_parse_failures(app.config.sphinx_gallery_conf))
     want.update(failures="2", skipped="1", tests="3")
     got = dict(suite.attrib)
     del got["time"]

From 2775e40fb6e42d8ac0a659e89a38586ff32079ee Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Fri, 21 Jun 2024 11:55:27 -0400
Subject: [PATCH 31/40] TST: Third one


From 304258ecddd67baa9a79b45a2b3177e2534c2c07 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Fri, 21 Jun 2024 12:06:27 -0400
Subject: [PATCH 32/40] TST: Fourth one


From 6d92c7bb29a98021229503f0c5d47e2e88bd5be2 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Fri, 21 Jun 2024 12:27:49 -0400
Subject: [PATCH 33/40] TST: Fifth one


From 27f51827b4b309fe4d2c59e104f9cfcdb40ce465 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Sat, 22 Jun 2024 12:06:48 -0400
Subject: [PATCH 34/40] FIX: Batch size

---
 sphinx_gallery/gen_rst.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 4773ff9c2..6ae84dc7a 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -589,6 +589,7 @@ def generate_dir_rst(
         parallel = Parallel(
             n_jobs=gallery_conf["parallel"],
             pre_dispatch="n_jobs",
+            batch_size=1,
             backend="loky",
         )
 

From 9fee1ceec9d2e97df087fe1a916554e18fc59aa3 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Mon, 24 Jun 2024 10:51:05 -0400
Subject: [PATCH 35/40] DOC: Document considerations

---
 doc/configuration.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/configuration.rst b/doc/configuration.rst
index a15c37ebd..c76a96358 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -2114,7 +2114,10 @@ Sphinx. If an ``int``, then that number of jobs will be passed to
 .. warning::
     Some packages might not play nicely with parallel processing. You might need to
     set variables in a :ref:`custom resetter <custom_reset>` for example to ensure
-    that all spawned processes are properly set up and torn down.
+    that all spawned processes are properly set up and torn down. Parallelism is
+    achieved through the Loky backend of joblib, see :ref:`joblib:parallel` for
+    documentation of many relevant conisderations (e.g., pickling, oversubscription
+    of CPU resources, etc.).
 
     Using parallel building will also disable memory measurements.
 

From 05caa49c1f90749dee5c1e4d5b357c12f87664cb Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Mon, 24 Jun 2024 10:52:04 -0400
Subject: [PATCH 36/40] DOC: Better wording

---
 doc/configuration.rst | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/doc/configuration.rst b/doc/configuration.rst
index c76a96358..ce4d1df14 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -2112,12 +2112,14 @@ Sphinx. If an ``int``, then that number of jobs will be passed to
 :class:`joblib.Parallel`.
 
 .. warning::
-    Some packages might not play nicely with parallel processing. You might need to
-    set variables in a :ref:`custom resetter <custom_reset>` for example to ensure
-    that all spawned processes are properly set up and torn down. Parallelism is
-    achieved through the Loky backend of joblib, see :ref:`joblib:parallel` for
-    documentation of many relevant conisderations (e.g., pickling, oversubscription
-    of CPU resources, etc.).
+    Some packages might not play nicely with parallel processing, so this feature
+    is considered **experimental**!
+
+    For example, you might need to set variables or call functions in a
+    :ref:`custom resetter <custom_reset>` to ensure that all spawned processes are
+    properly set up and torn down. Parallelism is achieved through the Loky backend of
+    joblib, see :ref:`joblib:parallel` for documentation of many relevant conisderations
+    (e.g., pickling, oversubscription of CPU resources, etc.).
 
     Using parallel building will also disable memory measurements.
 

From 554e5679a461c9e8b1aca479dcec97e62b6ca4cb Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Mon, 24 Jun 2024 10:52:41 -0400
Subject: [PATCH 37/40] DOC: Better wording

---
 doc/configuration.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/configuration.rst b/doc/configuration.rst
index ce4d1df14..6fe3608d8 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -2104,12 +2104,12 @@ Sphinx-Gallery can be configured to run examples simultaneously using
 
     sphinx_gallery_conf = {
         ...
-        'parallel': True,
+        'parallel': 2,
     }
 
+If an ``int``, then that number of jobs will be passed to :class:`joblib.Parallel`.
 If ``True``, then the same number of jobs will be used as the ``-j`` flag for
-Sphinx. If an ``int``, then that number of jobs will be passed to
-:class:`joblib.Parallel`.
+Sphinx.
 
 .. warning::
     Some packages might not play nicely with parallel processing, so this feature

From cdc7a93969f8b34e74bc0709f98051d63f00fdc0 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Mon, 24 Jun 2024 14:17:13 -0400
Subject: [PATCH 38/40] ENH: Add count

---
 sphinx_gallery/gen_gallery.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
index d9fd7ca03..684253aec 100644
--- a/sphinx_gallery/gen_gallery.py
+++ b/sphinx_gallery/gen_gallery.py
@@ -1470,7 +1470,9 @@ def summarize_failing_examples(app, exception):
 
     idt = "    "
     if failing_as_expected:
-        logger.info(bold("Examples failing as expected:"), color="blue")
+        logger.info(
+            bold(blue(f"Examples failing as expected ({len(failing_as_expected)}):"))
+        )
         for fail_example in failing_as_expected:
             path = os.path.relpath(fail_example, gallery_conf["src_dir"])
             logger.info(
@@ -1480,7 +1482,9 @@ def summarize_failing_examples(app, exception):
 
     fail_msgs = []
     if failing_unexpectedly:
-        fail_msgs.append(bold(red("Unexpected failing examples:\n")))
+        fail_msgs.append(
+            bold(red(f"Unexpected failing examples ({len(failing_unexpectedly)}):\n"))
+        )
         for fail_example in failing_unexpectedly:
             path = os.path.relpath(fail_example, gallery_conf["src_dir"])
             fail_msgs.append(
@@ -1493,7 +1497,7 @@ def summarize_failing_examples(app, exception):
             os.path.relpath(p, gallery_conf["src_dir"]) for p in passing_unexpectedly
         ]
         fail_msgs.append(
-            bold(red("Examples expected to fail, but not failing:\n\n"))
+            bold(red(f"Examples expected to fail, but not failing ({len(paths)}):\n\n"))
             + red("\n".join(indent(p, idt) for p in paths))
             + "\n\nPlease remove these examples from "
             + "sphinx_gallery_conf['expected_failing_examples'] "

From 3cb6893ee16945206cf67d57d8c83ad95ffeeed0 Mon Sep 17 00:00:00 2001
From: Lucy Liu <jliu176@gmail.com>
Date: Fri, 28 Jun 2024 16:23:51 +1000
Subject: [PATCH 39/40] add ver drop to changes

---
 CHANGES.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index e55734322..4fa0a8592 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,6 +1,12 @@
 Changelog
 =========
 
+v0.17.0
+-------
+
+Support for Python 3.8 and Sphinx 4 dropped in this release.
+Requirement is now Python >= 3.9 and Sphinx >= 5.
+
 v0.16.0
 -------
 Sphinx 7.3.0 and above changed caching and serialization checks. Now instead of passing

From f557d6467c5b9c09e3f0a40817704c1e3fc6afb0 Mon Sep 17 00:00:00 2001
From: Eric Larson <larson.eric.d@gmail.com>
Date: Sat, 29 Jun 2024 12:59:00 -0400
Subject: [PATCH 40/40] FIX: Updates

---
 sphinx_gallery/gen_gallery.py |  7 ++-----
 sphinx_gallery/gen_rst.py     | 18 +++++++++++++-----
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
index 684253aec..fcdffc048 100644
--- a/sphinx_gallery/gen_gallery.py
+++ b/sphinx_gallery/gen_gallery.py
@@ -401,11 +401,8 @@ def _fill_gallery_conf_defaults(sphinx_gallery_conf, app=None, check_keys=True):
     # Check ignore_repr_types
     _check_config_type(gallery_conf, "ignore_repr_types", str)
 
-    if not isinstance(gallery_conf["parallel"], (bool, int)):
-        raise TypeError(
-            'gallery_conf["parallel"] must be bool or int, got '
-            f'{type(gallery_conf["parallel"])}'
-        )
+    # Check parallel
+    _check_config_type(gallery_conf, "parallel", (bool, int))
     if gallery_conf["parallel"] is True:
         gallery_conf["parallel"] = app.parallel
     if gallery_conf["parallel"] == 1:
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 2e510cfbe..3d1d559a4 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -599,6 +599,8 @@ def generate_dir_rst(
         fname = sorted_listdir[fi]
         src_file = os.path.normpath(os.path.join(src_dir, fname))
         gallery_conf["titles"][src_file] = title
+        # n.b. non-executable files have none of these three variables defined,
+        # so the last conditional must be "elif" not just "else"
         if "formatted_exception" in out_vars:
             assert "passing" not in out_vars
             assert "stale" not in out_vars
@@ -606,7 +608,7 @@ def generate_dir_rst(
         elif "passing" in out_vars:
             assert "stale" not in out_vars
             gallery_conf["passing_examples"].append(src_file)
-        elif "stale" in out_vars:  # non-executable files have none of these three
+        elif "stale" in out_vars:
             gallery_conf["stale_examples"].append(out_vars["stale"])
         costs.append(dict(t=t, mem=mem, src_file=src_file, target_dir=target_dir))
         gallery_item_filename = (
@@ -783,7 +785,6 @@ def __call__(self):
 
 def _get_memory_base():
     """Get the base amount of memory used by the current Python process."""
-    # There might be a cleaner way to do this at some point
     from memory_profiler import memory_usage
 
     memory_base = memory_usage(max_usage=True)
@@ -1277,10 +1278,17 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
     cost : tuple
         A tuple containing the ``(time_elapsed, memory_used)`` required to run the
         script.
-    backrefs : set
-        The backrefs seen in this example.
     out_vars : dict
-        Variables used to run the script.
+        Variables used to run the script, possibly with entries:
+
+        "stale"
+            True if the example was stale.
+        "backrefs"
+            The backreferences.
+        "passing"
+            True if the example passed.
+        "formatted_exception"
+            Formatted string of the exception.
     """
     src_file = os.path.normpath(os.path.join(src_dir, fname))
     out_vars = dict()