Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1b4aecd

Browse files
jschuellerlarsonerlucyleeow
authored
ENH: Parallel gallery generation (#877)
Co-authored-by: Eric Larson <[email protected]> Co-authored-by: Lucy Liu <[email protected]>
1 parent cf81205 commit 1b4aecd

19 files changed

+420
-246
lines changed

‎.circleci/config.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
"git+https://github.com/pyvista/pyvista" \
4545
"docutils>=0.18" imageio pydata-sphinx-theme \
4646
"jupyterlite-sphinx>=0.8.0,<0.9.0" "jupyterlite-pyodide-kernel<0.1.0" \
47-
libarchive-c "sphinxcontrib-video>=0.2.1rc0"
47+
libarchive-c "sphinxcontrib-video>=0.2.1rc0" intersphinx_registry
4848
pip uninstall -yq vtk # pyvista installs vtk above
4949
pip install --upgrade --only-binary ":all" --extra-index-url https://wheels.vtk.org vtk-osmesa
5050
- save_cache:

‎.github/workflows/tests.yml

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,6 @@ jobs:
3232
sphinx_version: dev
3333
distrib: pip
3434
locale: C
35-
- os: ubuntu-latest # oldest supported Python and Sphinx
36-
python: '3.8'
37-
sphinx_version: '4'
38-
distrib: mamba
3935
- os: ubuntu-latest
4036
python: '3.11'
4137
sphinx_version: '5'
@@ -79,7 +75,7 @@ jobs:
7975
python=${{ env.PYTHON_VERSION }} pip numpy setuptools matplotlib pillow
8076
pytest pytest-cov coverage seaborn statsmodels plotly joblib wheel libiconv
8177
pygraphviz memory_profiler ipython pypandoc lxml conda-libmamba-solver mamba
82-
ffmpeg
78+
ffmpeg intersphinx-registry
8379
if: matrix.distrib == 'mamba'
8480
# Make sure that things work even if the locale is set to C (which
8581
# effectively means ASCII). Some of the input rst files have unicode

‎CHANGES.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
Changelog
22
=========
33

4+
v0.17.0
5+
-------
6+
7+
Support for Python 3.8 and Sphinx 4 dropped in this release.
8+
Requirement is now Python >= 3.9 and Sphinx >= 5.
9+
410
v0.16.0
511
-------
612
Sphinx 7.3.0 and above changed caching and serialization checks. Now instead of passing

‎doc/conf.py

Lines changed: 18 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
import warnings
1818
from datetime import date
1919

20+
from intersphinx_registry import get_intersphinx_mapping
21+
2022
import sphinx_gallery
2123

2224
# If extensions (or modules to document with autodoc) are in another directory,
@@ -332,15 +334,18 @@ def setup(app):
332334

333335

334336
# Example configuration for intersphinx: refer to the Python standard library.
335-
intersphinx_mapping = {
336-
"python": (f"https://docs.python.org/{sys.version_info.major}", None),
337-
"numpy": ("https://numpy.org/doc/stable/", None),
338-
"matplotlib": ("https://matplotlib.org/stable", None),
339-
"pyvista": ("https://docs.pyvista.org/version/stable", None),
340-
"sklearn": ("https://scikit-learn.org/stable", None),
341-
"sphinx": ("https://www.sphinx-doc.org/en/master", None),
342-
"pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
343-
}
337+
intersphinx_mapping = get_intersphinx_mapping(
338+
packages={
339+
"joblib",
340+
"matplotlib",
341+
"numpy",
342+
"pandas",
343+
"python",
344+
"pyvista",
345+
"sklearn",
346+
"sphinx",
347+
},
348+
)
344349

345350
examples_dirs = ["../examples", "../tutorials"]
346351
gallery_dirs = ["auto_examples", "tutorials"]
@@ -352,30 +357,19 @@ def setup(app):
352357
# installed
353358
import pyvista
354359
except Exception: # can raise all sorts of errors
355-
pass
360+
pyvista = None
356361
else:
357362
image_scrapers += ("pyvista",)
358363
examples_dirs.append("../pyvista_examples")
359364
gallery_dirs.append("auto_pyvista_examples")
360-
pyvista.OFF_SCREEN = True
361-
# Preferred plotting style for documentation
362-
pyvista.set_plot_theme("document")
363-
pyvista.global_theme.window_size = [1024, 768]
364-
pyvista.global_theme.font.size = 22
365-
pyvista.global_theme.font.label_size = 22
366-
pyvista.global_theme.font.title_size = 22
367-
pyvista.global_theme.return_cpos = False
368-
# necessary when building the sphinx gallery
369-
pyvista.BUILDING_GALLERY = True
370-
pyvista.set_jupyter_backend(None)
371365

372366
# Set plotly renderer to capture _repr_html_ for sphinx-gallery
373367
try:
368+
import plotly
374369
import plotly.io
375370
except ImportError:
376-
pass
371+
plotly = None
377372
else:
378-
plotly.io.renderers.default = "sphinx_gallery"
379373
examples_dirs.append("../plotly_examples")
380374
gallery_dirs.append("auto_plotly_examples")
381375

@@ -393,6 +387,7 @@ def setup(app):
393387
"examples_dirs": examples_dirs,
394388
"gallery_dirs": gallery_dirs,
395389
"image_scrapers": image_scrapers,
390+
"reset_modules": ("matplotlib", "seaborn", "sg_doc_build.reset_others"),
396391
"compress_images": ("images", "thumbnails"),
397392
# specify the order of examples to be according to filename
398393
"within_subsection_order": "FileNameSortKey",

‎doc/configuration.rst

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ file, inside a ``sphinx_gallery_conf`` dictionary.
4444
- ``abort_on_example_error`` (:ref:`abort_on_first`)
4545
- ``expected_failing_examples`` (:ref:`dont_fail_exit`)
4646
- ``only_warn_on_example_error`` (:ref:`warning_on_error`)
47+
- ``parallel`` (:ref:`parallel`)
4748

4849
**Cross-referencing**
4950

@@ -2093,6 +2094,35 @@ flag is passed to ``sphinx-build``. This can be enabled by setting::
20932094
}
20942095

20952096

2097+
.. _parallel:
2098+
2099+
Build examples in parallel
2100+
^^^^^^^^^^^^^^^^^^^^^^^^^^
2101+
2102+
Sphinx-Gallery can be configured to run examples simultaneously using
2103+
:mod:`joblib`. This can be enabled by setting::
2104+
2105+
sphinx_gallery_conf = {
2106+
...
2107+
'parallel': 2,
2108+
}
2109+
2110+
If an ``int``, then that number of jobs will be passed to :class:`joblib.Parallel`.
2111+
If ``True``, then the same number of jobs will be used as the ``-j`` flag for
2112+
Sphinx.
2113+
2114+
.. warning::
2115+
Some packages might not play nicely with parallel processing, so this feature
2116+
is considered **experimental**!
2117+
2118+
For example, you might need to set variables or call functions in a
2119+
:ref:`custom resetter <custom_reset>` to ensure that all spawned processes are
2120+
properly set up and torn down. Parallelism is achieved through the Loky backend of
2121+
joblib, see :ref:`joblib:parallel` for documentation of many relevant conisderations
2122+
(e.g., pickling, oversubscription of CPU resources, etc.).
2123+
2124+
Using parallel building will also disable memory measurements.
2125+
20962126
.. _recommend_examples:
20972127

20982128
Enabling the example recommender system

‎doc/getting_started.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ generated:
178178
* ``.py`` - to enable the user to download a ``.py`` version of the example.
179179
* ``.py.md5`` - a md5 hash of the ``.py`` file, used to determine if changes
180180
have been made to the file and thus if new output files need to be generated.
181-
* ``_codeobj.pickle`` - used to identify function names and to which module
181+
* ``.codeobj.json`` - used to identify function names and to which module
182182
they belong (more details in
183183
:ref:`sphx_glr_auto_examples_plot_6_function_identifier.py`)
184184

‎doc/sphinxext/sg_doc_build.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,29 @@ def notebook_modification_function(notebook_content, notebook_filename):
5656
notebook_content["cells"] = (
5757
dummy_notebook_content["cells"] + notebook_content["cells"]
5858
)
59+
60+
61+
def reset_others(gallery_conf, fname):
62+
"""Reset plotting functions."""
63+
try:
64+
import pyvista
65+
except Exception:
66+
pass
67+
else:
68+
pyvista.OFF_SCREEN = True
69+
# Preferred plotting style for documentation
70+
pyvista.set_plot_theme("document")
71+
pyvista.global_theme.window_size = [1024, 768]
72+
pyvista.global_theme.font.size = 22
73+
pyvista.global_theme.font.label_size = 22
74+
pyvista.global_theme.font.title_size = 22
75+
pyvista.global_theme.return_cpos = False
76+
# necessary when building the sphinx gallery
77+
pyvista.BUILDING_GALLERY = True
78+
pyvista.set_jupyter_backend(None)
79+
try:
80+
import plotly.io
81+
except Exception:
82+
pass
83+
else:
84+
plotly.io.renderers.default = "sphinx_gallery"

‎pyproject.toml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,15 @@ dynamic = [
3131
]
3232
dependencies = [
3333
"pillow",
34-
"sphinx>=4",
34+
"sphinx>=5",
3535
]
3636
optional-dependencies.animations = [
3737
"sphinxcontrib-video",
3838
]
3939
optional-dependencies.dev = [
4040
"absl-py",
4141
"graphviz",
42+
"intersphinx-registry",
4243
"ipython",
4344
"joblib",
4445
"jupyterlite-sphinx",
@@ -57,6 +58,9 @@ optional-dependencies.dev = [
5758
optional-dependencies.jupyterlite = [
5859
"jupyterlite-sphinx",
5960
]
61+
optional-dependencies.parallel = [
62+
"joblib",
63+
]
6064
optional-dependencies.recommender = [
6165
"numpy",
6266
]

‎sphinx_gallery/backreferences.py

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@
66
"""
77

88
import ast
9-
import codecs
10-
import collections
119
import inspect
1210
import os
1311
import re
@@ -19,7 +17,7 @@
1917
from sphinx.errors import ExtensionError
2018

2119
from .scrapers import _find_image_ext
22-
from .utils import _replace_md5
20+
from .utils import _W_KW, _replace_md5
2321

2422
THUMBNAIL_PARENT_DIV = """
2523
.. raw:: html
@@ -249,9 +247,9 @@ def identify_names(script_blocks, ref_regex, global_variables=None, node=""):
249247
250248
Returns
251249
-------
252-
example_code_obj : OrderedDict[str, Any]
253-
OrderedDict with information about all code object references found in an
254-
example. OrderedDict contains the following keys:
250+
example_code_obj : Dict[str, Any]
251+
Dict with information about all code object references found in an
252+
example. Dict contains the following keys:
255253
256254
- example_code_obj['name'] : function or class name (str)
257255
- example_code_obj['module'] : module name (str)
@@ -271,7 +269,7 @@ def identify_names(script_blocks, ref_regex, global_variables=None, node=""):
271269
# Get matches from docstring inspection (explicit matches)
272270
text = "\n".join(block.content for block in script_blocks if block.type == "text")
273271
names.extend((x, x, False, False, True) for x in re.findall(ref_regex, text))
274-
example_code_obj = collections.OrderedDict() # order is important
272+
example_code_obj = dict() # native dict preserves order nowadays
275273
# Make a list of all guesses, in `_embed_code_links` we will break
276274
# when we find a match
277275
for name, full_name, class_like, is_class, is_explicit in names:
@@ -292,13 +290,13 @@ def identify_names(script_blocks, ref_regex, global_variables=None, node=""):
292290

293291
# get shortened module name
294292
module_short = _get_short_module_name(module, attribute)
295-
cobj = {
296-
"name": attribute,
297-
"module": module,
298-
"module_short": module_short or module,
299-
"is_class": is_class,
300-
"is_explicit": is_explicit,
301-
}
293+
cobj = dict(
294+
name=attribute,
295+
module=module,
296+
module_short=module_short or module,
297+
is_class=is_class,
298+
is_explicit=is_explicit,
299+
)
302300
example_code_obj[name].append(cobj)
303301
return example_code_obj
304302

@@ -391,9 +389,8 @@ def _write_backreferences(
391389
f"{backref}.examples.new",
392390
)
393391
seen = backref in seen_backrefs
394-
with codecs.open(
395-
include_path, "a" if seen else "w", encoding="utf-8"
396-
) as ex_file:
392+
mode = "a" if seen else "w"
393+
with open(include_path, mode, **_W_KW) as ex_file:
397394
if not seen:
398395
# Be aware that if the number of lines of this heading changes,
399396
# the minigallery directive should be modified accordingly
@@ -432,7 +429,7 @@ def _finalize_backreferences(seen_backrefs, gallery_conf):
432429
if os.path.isfile(path):
433430
# Close div containing all thumbnails
434431
# (it was open in _write_backreferences)
435-
with codecs.open(path, "a", encoding="utf-8") as ex_file:
432+
with open(path, "a", **_W_KW) as ex_file:
436433
ex_file.write(THUMBNAIL_PARENT_DIV_CLOSE)
437434
_replace_md5(path, mode="t")
438435
else:

0 commit comments

Comments
 (0)