Fix trailing text in doctest-syntax plot_directive.

anntzer · anntzer · commit f900c34799ea · 2021-04-29T14:58:24.000+02:00
The problem was that an input like
```
Some text.

&gt;&gt;&gt; python_code()

The end.
```
would get split to
```
Some text.

&gt;&gt;&gt; python_code()
```
and
```
The end.
```
and that unescape_doctest would then believe that `The end.` is already
python code (as it doesn't contain a `&gt;&gt;&gt;`) and try to run it as is.

To fix this, instead of repeatedly calling `contains_doctest` everywhere
to guess whether fragments are doctest or python, just do it once at the
beginning, do the escaping once if needed at the beginning and then call
the new _functions (`_split_code_at_show`, `_run_code`) which don't try to
guess anymore.  Because of the new (non-guessing) semantics these must
go to new functions, so let's make them private and just deprecate the old
(public) ones.

The escaping itself was done by `unescape_doctest`, but that had a
separate bug misparsing
```
This is an example...

&gt;&gt;&gt; some_python()

... isn't it?
```
the last line would get incorrectly misparsed as a line continuation,
despite the blank line in between.  Instead of trying to fix that
ourselves, just use `doctest.script_from_examples` which exactly serves
that purpose.
diff --git a/doc/api/next_api_changes/deprecations/20109-AL.rst b/doc/api/next_api_changes/deprecations/20109-AL.rst
@@ -0,0 +1,5 @@
+``plot_directive`` internals deprecations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The following helpers in `matplotlib.sphinxext.plot_directive` are deprecated:
+``unescape_doctest`` (use `doctest.script_from_examples` instead),
+``split_code_at_show``, ``run_code``.
diff --git a/lib/matplotlib/sphinxext/plot_directive.py b/lib/matplotlib/sphinxext/plot_directive.py
@@ -138,6 +138,7 @@
 """
 
 import contextlib
+import doctest
 from io import StringIO
 import itertools
 import os
@@ -301,14 +302,14 @@ def contains_doctest(text):
     return bool(m)
 
 
+@_api.deprecated("3.5", alternative="doctest.script_from_examples")
 def unescape_doctest(text):
     """
     Extract code from a piece of text, which contains either Python code
     or doctests.
     """
     if not contains_doctest(text):
         return text
-
     code = ""
     for line in text.split("\n"):
         m = re.match(r'^\s*(>>>|\.\.\.) (.*)$', line)
@@ -321,11 +322,16 @@ def unescape_doctest(text):
     return code
 
 
+@_api.deprecated("3.5")
 def split_code_at_show(text):
+    """Split code at plt.show()."""
+    return _split_code_at_show(text)[1]
+
+
+def _split_code_at_show(text):
     """Split code at plt.show()."""
     parts = []
     is_doctest = contains_doctest(text)
-
     part = []
     for line in text.split("\n"):
         if (not is_doctest and line.strip() == 'plt.show()') or \
@@ -337,7 +343,7 @@ def split_code_at_show(text):
             part.append(line)
     if "\n".join(part).strip():
         parts.append("\n".join(part))
-    return parts
+    return is_doctest, parts
 
 
 # -----------------------------------------------------------------------------
@@ -437,11 +443,20 @@ class PlotError(RuntimeError):
     pass
 
 
+@_api.deprecated("3.5")
 def run_code(code, code_path, ns=None, function_name=None):
     """
     Import a Python module from a path, and run the function given by
     name, if function_name is not None.
     """
+    _run_code(unescape_doctest(code), code_path, ns, function_name)
+
+
+def _run_code(code, code_path, ns=None, function_name=None):
+    """
+    Import a Python module from a path, and run the function given by
+    name, if function_name is not None.
+    """
 
     # Change the working directory to the directory of the example, so
     # it can get at its data files, if any.  Add its path to sys.path
@@ -466,7 +481,6 @@ def run_code(code, code_path, ns=None, function_name=None):
             sys, argv=[code_path], path=[os.getcwd(), *sys.path]), \
             contextlib.redirect_stdout(StringIO()):
         try:
-            code = unescape_doctest(code)
             if ns is None:
                 ns = {}
             if not ns:
@@ -529,7 +543,7 @@ def render_figures(code, code_path, output_dir, output_base, context,
 
     # Try to determine if all images already exist
 
-    code_pieces = split_code_at_show(code)
+    is_doctest, code_pieces = _split_code_at_show(code)
 
     # Look for single-figure output files first
     all_exists = True
@@ -593,7 +607,9 @@ def render_figures(code, code_path, output_dir, output_base, context,
         elif close_figs:
             plt.close('all')
 
-        run_code(code_piece, code_path, ns, function_name)
+        _run_code(doctest.script_from_examples(code_piece) if is_doctest
+                  else code_piece,
+                  code_path, ns, function_name)
 
         images = []
         fig_managers = _pylab_helpers.Gcf.get_all_fig_managers()
@@ -816,7 +832,9 @@ def run(arguments, content, options, state_machine, state, lineno):
 
     # copy script (if necessary)
     Path(dest_dir, output_base + source_ext).write_text(
-        unescape_doctest(code) if source_file_name == rst_file else code,
+        doctest.script_from_examples(code)
+        if source_file_name == rst_file and is_doctest
+        else code,
         encoding='utf-8')
 
     return errors
diff --git a/lib/matplotlib/tests/tinypages/some_plots.rst b/lib/matplotlib/tests/tinypages/some_plots.rst
@@ -15,11 +15,16 @@ Plot 2 doesn't use context either; has length 6:
 
     plt.plot(range(6))
 
-Plot 3 has length 4:
+Plot 3 has length 4, and uses doctest syntax:
 
 .. plot::
+    :format: doctest
 
-    plt.plot(range(4))
+    This is a doctest...
+
+    >>> plt.plot(range(4))
+
+    ... isn't it?
 
 Plot 4 shows that a new block with context does not see the variable defined
 in the no-context block:
