Reuse single kpsewhich instance for speed.

anntzer · anntzer · commit 9f3da44fe000 · 2021-02-17T01:26:40.000+01:00
On MacOS, where spawing kpsewhich instances is rather slow, this appears
to speed up
```
python -c 'from pylab import *; mpl.use("pdf"); rcParams["text.usetex"] = True; plot(); savefig("/tmp/test.pdf", backend="pdf")'
```
around two-fold (~4s to ~2s).  (There's also a small speedup on Linux,
perhaps ~10%, but the whole thing is already reasonably fast.)

Note that this is assuming that the dvi cache has already been built;
the costly subprocess calls here are due to calls to kpsewhich to
resolve the fonts whose name are listed in the dvi file.

Much of the complexity here comes from the need to force unbuffered
stdin/stdout when interacting with kpsewhich (otherwise, things just
hang); this is also the reason why this is not implemented on Windows
(Windows experts are welcome to look into this...; there, the speedup
should be even more significant).  (On Linux, another solution, which
does not require a third-party dependency, is to call
`stdbuf -oL kpsewhich ...` and pass bufsize=0 to Popen(), but
`ptyprocess` is pure Python so adding a dependency seems reasonable).

The `format` kwarg to `find_tex_file` had never been used before, and
cannot be handled in the single-process case, so just deprecate it.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -141,7 +141,7 @@ jobs:
 
           # Install dependencies from PyPI.
           python -mpip install --upgrade $PRE \
-            cycler kiwisolver numpy pillow pyparsing python-dateutil \
+            cycler kiwisolver numpy pillow ptyprocess pyparsing python-dateutil \
             -r requirements/testing/all.txt \
             ${{ matrix.extra-requirements }}
 
diff --git a/doc/api/next_api_changes/deprecations/19531-AL.rst b/doc/api/next_api_changes/deprecations/19531-AL.rst
@@ -0,0 +1,3 @@
+The *format* kwarg to ``dviread.find_tex_file``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+... is deprecated.
diff --git a/lib/matplotlib/dviread.py b/lib/matplotlib/dviread.py
@@ -1032,7 +1032,50 @@ def _parse_enc(path):
             "Failed to parse {} as Postscript encoding".format(path))
 
 
+class _Kpsewhich:
+    @lru_cache()  # A singleton.
+    def __new__(cls):
+        self = object.__new__(cls)
+        self._proc = self._new_proc()
+        return self
+
+    def _new_proc(self):
+        from ptyprocess import PtyProcess  # Force unbuffered IO.
+        try:
+            # kpsewhich requires being passed at least argument, so query
+            # texmf.cnf and read that out.  This also serves as sentinel below.
+            proc = PtyProcess.spawn(
+                ["kpsewhich", "-interactive", "texmf.cnf"], echo=False)
+            self._sentinel = proc.readline()
+            return proc
+        except FileNotFoundError:
+            return None
+
+    def search(self, filename):
+        if filename == "texmf.cnf":
+            return self._sentinel
+        if self._proc is not None and not self._proc.isalive():
+            self._proc = self._new_proc()  # Give a 2nd chance if it crashed.
+        if self._proc is None or not self._proc.isalive():
+            return ""
+        # kpsewhich prints nothing if the file does not exist.  To detect this
+        # case without setting an arbitrary timeout, we additionally query for
+        # texmf.cnf, which is known to exist at `self._sentinel`.  We can then
+        # check whether the first query exists by comparing the first returned
+        # line to `self._sentinel`.  (This is also why we need to separately
+        # handle the case of querying texmf.cnf above.)
+        self._proc.write(os.fsencode(filename) + b"\ntexmf.cnf\n")
+        out = self._proc.readline()
+        if out == self._sentinel:
+            return ""
+        else:
+            self._proc.readline()  # flush the extra sentinel line.
+            # POSIX ptys actually emit \r\n.
+            return os.fsdecode(out).rstrip("\r\n")
+
+
 @lru_cache()
+@_api.delete_parameter("3.5", "format")
 def find_tex_file(filename, format=None):
     """
     Find a file in the texmf tree.
@@ -1064,25 +1107,36 @@ def find_tex_file(filename, format=None):
     if isinstance(format, bytes):
         format = format.decode('utf-8', errors='replace')
 
-    if os.name == 'nt':
+    if format is not None:  # Deprecated.
+        if os.name == 'nt':  # See below re: encoding.
+            kwargs = {'env': {**os.environ, 'command_line_encoding': 'utf-8'},
+                      'encoding': 'utf-8'}
+        else:  # On POSIX, run through the equivalent of os.fsdecode().
+            kwargs = {'encoding': sys.getfilesystemencoding(),
+                      'errors': 'surrogatescape'}
+        cmd = ['kpsewhich']
+        if format is not None:
+            cmd += ['--format=' + format]
+        cmd += [filename]
+        try:
+            result = cbook._check_and_log_subprocess(cmd, _log, **kwargs)
+        except (FileNotFoundError, RuntimeError):
+            return ''
+        return result.rstrip('\n')
+
+    if os.name == "nt":
         # On Windows only, kpathsea can use utf-8 for cmd args and output.
-        # The `command_line_encoding` environment variable is set to force it
-        # to always use utf-8 encoding.  See Matplotlib issue #11848.
-        kwargs = {'env': {**os.environ, 'command_line_encoding': 'utf-8'},
-                  'encoding': 'utf-8'}
-    else:  # On POSIX, run through the equivalent of os.fsdecode().
-        kwargs = {'encoding': sys.getfilesystemencoding(),
-                  'errors': 'surrogatescape'}
-
-    cmd = ['kpsewhich']
-    if format is not None:
-        cmd += ['--format=' + format]
-    cmd += [filename]
-    try:
-        result = cbook._check_and_log_subprocess(cmd, _log, **kwargs)
-    except (FileNotFoundError, RuntimeError):
-        return ''
-    return result.rstrip('\n')
+        # The `command_line_encoding` environment variable is set to force
+        # it to always use utf-8 encoding.  See Matplotlib issue #11848.
+        try:
+            result = cbook._check_and_log_subprocess(
+                ["kpsewhich", filename], _log, encoding="utf-8",
+                env={**os.environ, "command_line_encoding": "utf-8"})
+        except (FileNotFoundError, RuntimeError):
+            return ""
+        return result.rstrip("\n")
+    else:
+        return _Kpsewhich().search(filename)
 
 
 @lru_cache()
diff --git a/setup.py b/setup.py
@@ -309,6 +309,7 @@ def build_extensions(self):
         "pillow>=6.2.0",
         "pyparsing>=2.2.1",
         "python-dateutil>=2.7",
+        "ptyprocess;os_name=='posix'",
     ],
 
     cmdclass=cmdclass,

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	+The format kwarg to ``dviread.find_tex_file``
	`2`	`+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
	`3`	`+... is deprecated.`