Switch to per-file locking.

anntzer · anntzer · commit e0283a94739b · 2018-03-02T22:35:54.000-08:00
Replace the Locked contextmanager (which locks a directory, preventing
other (cooperative) processes to access it) by a private _lock_path
contextmanager, which locks a single file (or directory).

- The finer grained lock avoids locking out the entire tex cache when
  handling usetex, which is useful when running multiple processes at
  once.

- Python3 implements the `"x"` ("exclusive") mode to open, which we can
  use instead of relying on `makedirs` to achieve a race-free operation
  on the filesystem.

- The previous implementation allowed multiple threads of a single
  process to acquire the same lock, but (for the use cases here, e.g.
  running a tex subprocess) this is actually undesirable.  Removing this
  behavior also simplifies the implementation.

- As far as I can tell, the previous implementation was actually racy:
  in

    retries = 50
    sleeptime = 0.1
    while retries:
        files = glob.glob(self.pattern)
        if files and not files[0].endswith(self.end):
            time.sleep(sleeptime)
            retries -= 1
        else:
            break
    else:
        err_str = _lockstr.format(self.pattern)
        raise self.TimeoutError(err_str)

    # &lt;----- HERE

    if not files:
        try:
            os.makedirs(self.lock_path)
        except OSError:
            pass
    else:  # PID lock already here --- someone else will remove it.
        self.remove = False

  multiple processes can reach "HERE" at the same time and each
  successfully create their own lock.
diff --git a/doc/api/next_api_changes/2018-02-15-AL-deprecations.rst b/doc/api/next_api_changes/2018-02-15-AL-deprecations.rst
@@ -10,6 +10,7 @@ The following functions and classes are deprecated:
 
 - ``cbook.GetRealpathAndStat`` (which is only a helper for
   ``get_realpath_and_stat``),
+- ``cbook.Locked``,
 - ``cbook.is_numlike`` (use ``isinstance(..., numbers.Number)`` instead),
 - ``mathtext.unichr_safe`` (use ``chr`` instead),
 - ``texmanager.dvipng_hack_alpha``,
diff --git a/lib/matplotlib/cbook/__init__.py b/lib/matplotlib/cbook/__init__.py
@@ -24,6 +24,7 @@
 import numbers
 import operator
 import os
+from pathlib import Path
 import re
 import sys
 import time
@@ -2486,6 +2487,7 @@ def get_label(y, default_name):
 """
 
 
+@deprecated("3.0")
 class Locked(object):
     """
     Context manager to handle locks.
@@ -2541,6 +2543,40 @@ def __exit__(self, exc_type, exc_value, traceback):
                     pass
 
 
+@contextlib.contextmanager
+def _lock_path(path):
+    """
+    Context manager for locking a path.
+
+    Usage::
+
+        with _lock_path(path):
+            ...
+
+    Another thread or process that attempts to lock the same path will wait
+    until this context manager is exited.
+
+    The lock is implemented by creating a temporary file in the parent
+    directory, so that directory must exist and be writable.
+    """
+    path = Path(path)
+    lock_path = path.with_name(path.name + ".matplotlib-lock")
+    retries = 50
+    sleeptime = 0.1
+    for _ in range(retries):
+        try:
+            with lock_path.open("xb"):
+                break
+        except FileExistsError:
+            time.sleep(sleeptime)
+    else:
+        raise TimeoutError(_lockstr.format(lock_path))
+    try:
+        yield
+    finally:
+        lock_path.unlink()
+
+
 def _topmost_artist(
         artists,
         _cached_max=functools.partial(max, key=operator.attrgetter("zorder"))):
diff --git a/lib/matplotlib/font_manager.py b/lib/matplotlib/font_manager.py
@@ -1446,7 +1446,7 @@ def _rebuild():
         fontManager = FontManager()
 
         if _fmcache:
-            with cbook.Locked(cachedir):
+            with cbook._lock_path(_fmcache):
                 json_dump(fontManager, _fmcache)
         _log.info("generated new fontManager")
 
@@ -1459,9 +1459,9 @@ def _rebuild():
             else:
                 fontManager.default_size = None
                 _log.debug("Using fontManager instance from %s", _fmcache)
-        except cbook.Locked.TimeoutError:
+        except TimeoutError:
             raise
-        except:
+        except Exception:
             _rebuild()
     else:
         _rebuild()
diff --git a/lib/matplotlib/texmanager.py b/lib/matplotlib/texmanager.py
@@ -33,29 +33,25 @@
 
 """
 
-from __future__ import absolute_import, division, print_function
-
 import six
 
 import copy
+import distutils.version
 import glob
 import hashlib
 import logging
 import os
 from pathlib import Path
+import re
 import shutil
 import subprocess
 import sys
 import warnings
 
-import distutils.version
 import numpy as np
+
 import matplotlib as mpl
-from matplotlib import rcParams
-from matplotlib._png import read_png
-from matplotlib.cbook import Locked
-import matplotlib.dviread as dviread
-import re
+from matplotlib import _png, cbook, dviread, rcParams
 
 _log = logging.getLogger(__name__)
 
@@ -340,7 +336,7 @@ def make_dvi(self, tex, fontsize):
         dvifile = '%s.dvi' % basefile
         if not os.path.exists(dvifile):
             texfile = self.make_tex(tex, fontsize)
-            with Locked(self.texcache):
+            with cbook._lock_path(texfile):
                 self._run_checked_subprocess(
                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
                      texfile], tex)
@@ -436,7 +432,7 @@ def get_grey(self, tex, fontsize=None, dpi=None):
         alpha = self.grey_arrayd.get(key)
         if alpha is None:
             pngfile = self.make_png(tex, fontsize, dpi)
-            X = read_png(os.path.join(self.texcache, pngfile))
+            X = _png.read_png(os.path.join(self.texcache, pngfile))
             self.grey_arrayd[key] = alpha = X[:, :, -1]
         return alpha
 
