8000 Switch to per-file locking. · matplotlib/matplotlib@5a47356 · GitHub
[go: up one dir, main page]
Skip to content

Commit 5a47356

Browse files
anntzeranntzer
committed
Switch to per-file locking.
Replace the Locked contextmanager (which locks a directory, preventing other (cooperative) processes to access it) by a private _lock_path contextmanager, which locks a single file (or directory). - The finer grained lock avoids locking out the entire tex cache when handling usetex, which is useful when running multiple processes at once. - Python3 implements the `"x"` ("exclusive") mode to open, which we can use instead of relying on `makedirs` to achieve a race-free operation on the filesystem. - The previous implementation allowed multiple threads of a single process to acquire the same lock, but (for the use cases here, e.g. running a tex subprocess) this is actually undesirable. Removing this behavior also simplifies the implementation. - As far as I can tell, the previous implementation was actually racy: in retries = 50 sleeptime = 0.1 while retries: files = glob.glob(self.pattern) if files and not files[0].endswith(self.end): time.sleep(sleeptime) retries -= 1 else: break else: err_str = _lockstr.format(self.pattern) raise self.TimeoutError(err_str) # <----- HERE if not files: try: os.makedirs(self.lock_path) except OSError: pass else: # PID lock already here --- someone else will remove it. self.remove = False multiple processes can reach "HERE" at the same time and each successfully create their own lock.
1 parent 59c39ed commit 5a47356

File tree

4 files changed

+45
-6
lines changed

4 files changed

+45
-6
lines changed

‎doc/api/next_api_changes/2018-02-15-AL-deprecations.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,6 @@ The following functions and classes are deprecated:
44

55
- ``cbook.GetRealpathAndStat`` (which is only a helper for
66
``get_realpath_and_stat``),
7+
- ``cbook.Locked``,
78
- ``cbook.is_numlike`` (use ``isinstance(..., numbers.Number)`` instead),
89
- ``mathtext.unichr_safe`` (use ``chr`` instead),

‎lib/matplotlib/cbook/__init__.py

Lines changed: 39 additions & 0 deletions
< 10000 td data-grid-cell-id="diff-22d7b6f88a379f8ed094611e4034b673df1af718cd74a4b2981a192ff08e446f-2545-2548-2" data-line-anchor="diff-22d7b6f88a379f8ed094611e4034b673df1af718cd74a4b2981a192ff08e446fR2548" data-selected="false" role="gridcell" style="background-color:var(--diffBlob-additionLine-bgColor, var(--diffBlob-addition-bgColor-line));padding-right:24px" tabindex="-1" valign="top" class="focusable-grid-cell diff-text-cell right-side-diff-cell left-side">+
@contextlib.contextmanager
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import numbers
2525
import operator
2626
import os
27+
from pathlib import Path
2728
import re
2829
import sys
2930
import time
@@ -2488,6 +2489,7 @@ def get_label(y, default_name):
24882489
"""
24892490

24902491

2492+
@deprecated("3.0")
24912493
class Locked(object):
24922494
"""
24932495
Context manager to handle locks.
@@ -2543,6 +2545,43 @@ def __exit__(self, exc_type, exc_value, traceback):
25432545
pass
25442546

25452547

2548
2549+
def _lock_path(path):
2550+
"""
2551+
Context manager for locking a path.
2552+
2553+
Usage::
2554+
2555+
with _lock_path(path):
2556+
...
2557+
2558+
Another thread or process that attempts to lock the same path will wait
2559+
until this context manager is exited.
2560+
2561+
The lock is implemented by creating a temporary file in the parent
2562+
directory, so that directory must exist and be writable.
2563+
"""
2564+
path = Path(path)
2565+
lock_path = path.with_name(path.name + ".matplotlib-lock")
2566+
retries = 50
2567+
sleeptime = 0.1
2568+
for _ in range(retries):
2569+
try:
2570+
with lock_path.open("xb"):
2571+
pass
2572+
except FileExistsError:
2573+
time.sleep(sleeptime)
2574+
continue
2575+
else:
2576+
break
2577+
else:
2578+
raise TimeoutError(_lockstr.format(lock_path))
2579+
try:
2580+
yield
2581+
finally:
2582+
lock_path.unlink()
2583+
2584+
25462585
def _topmost_artist(
25472586
artists,
25482587
_cached_max=functools.partial(max, key=operator.attrgetter("zorder"))):

‎lib/matplotlib/font_manager.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1446,7 +1446,7 @@ def _rebuild():
14461446
fontManager = FontManager()
14471447

14481448
if _fmcache:
1449-
with cbook.Locked(cachedir):
1449+
with cbook._lock_path(_fmcache):
14501450
json_dump(fontManager, _fmcache)
14511451
_log.info("generated new fontManager")
14521452

@@ -1459,9 +1459,9 @@ def _rebuild():
14591459
else:
14601460
fontManager.default_size = None
14611461
_log.debug("Using fontManager instance from %s", _fmcache)
1462-
except cbook.Locked.TimeoutError:
1462+
except TimeoutError:
14631463
raise
1464-
except:
1464+
except Exception:
14651465
_rebuild()
14661466
else:
14671467
_rebuild()

‎lib/matplotlib/texmanager.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,8 @@
5050
import distutils.version
5151
import numpy as np
5252
import matplotlib as mpl
53-
from matplotlib import rcParams
53+
from matplotlib import cbook, rcParams
5454
from matplotlib._png import read_png
55-
from matplotlib.cbook import Locked
5655
from matplotlib.compat.subprocess import subprocess, Popen, PIPE, STDOUT
5756
import matplotlib.dviread as dviread
5857
import re
@@ -360,7 +359,7 @@ def make_dvi(self, tex, fontsize):
360359
dvifile = '%s.dvi' % basefile
361360
if not os.path.exists(dvifile):
362361
texfile = self.make_tex(tex, fontsize)
363-
with Locked(self.texcache):
362+
with cbook._lock_path(texfile):
364363
self._run_checked_subprocess(
365364
["latex", "-interaction=nonstopmode", "--halt-on-error",
366365
texfile], tex)

0 commit comments

Comments
 (0)
0