Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ea3eb88

Browse files
pitroupitrou
committed
Issue #9260: A finer-grained import lock.
Most of the import sequence now uses per-module locks rather than the global import lock, eliminating well-known issues with threads and imports.
1 parent 5cec9d2 commit ea3eb88

12 files changed

Lines changed: 3746 additions & 3010 deletions

File tree

‎Doc/c-api/import.rst‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@ Importing Modules
3030
3131
.. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name)
3232
33-
This version of :c:func:`PyImport_ImportModule` does not block. It's intended
34-
to be used in C functions that import other modules to execute a function.
35-
The import may block if another thread holds the import lock. The function
36-
:c:func:`PyImport_ImportModuleNoBlock` never blocks. It first tries to fetch
37-
the module from sys.modules and falls back to :c:func:`PyImport_ImportModule`
38-
unless the lock is held, in which case the function will raise an
39-
:exc:`ImportError`.
33+
This function is a deprecated alias of :c:func:`PyImport_ImportModule`.
34+
35+
.. versionchanged:: 3.3
36+
This function used to fail immediately when the import lock was held
37+
by another thread. In Python 3.3 though, the locking scheme switched
38+
to per-module locks for most purposes, so this function's special
39+
behaviour isn't needed anymore.
4040
4141
4242
.. c:function:: PyObject* PyImport_ImportModuleEx(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist)

‎Doc/library/imp.rst‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -112,18 +112,29 @@ This module provides an interface to the mechanisms used to implement the
112112
Return ``True`` if the import lock is currently held, else ``False``. On
113113
platforms without threads, always return ``False``.
114114

115-
On platforms with threads, a thread executing an import holds an internal lock
116-
until the import is complete. This lock blocks other threads from doing an
117-
import until the original import completes, which in turn prevents other threads
118-
from seeing incomplete module objects constructed by the original thread while
119-
in the process of completing its import (and the imports, if any, triggered by
120-
that).
115+
On platforms with threads, a thread executing an import first holds a
116+
global import lock, then sets up a per-module lock for the rest of the
117+
import. This blocks other threads from importing the same module until
118+
the original import completes, preventing other threads from seeing
119+
incomplete module objects constructed by the original thread. An
120+
exception is made for circular imports, which by construction have to
121+
expose an incomplete module object at some point.
122+
123+
.. note::
124+
Locking semantics of imports are an implementation detail which may
125+
vary from release to release. However, Python ensures that circular
126+
imports work without any deadlocks.
127+
128+
.. versionchanged:: 3.3
129+
In Python 3.3, the locking scheme has changed to per-module locks for
130+
the most part.
121131

122132

123133
.. function:: acquire_lock()
124134

125-
Acquire the interpreter's import lock for the current thread. This lock should
126-
be used by import hooks to ensure thread-safety when importing modules.
135+
Acquire the interpreter's global import lock for the current thread.
136+
This lock should be used by import hooks to ensure thread-safety when
137+
importing modules.
127138

128139
Once a thread has acquired the import lock, the same thread may acquire it
129140
again without blocking; the thread must release it once for each time it has
@@ -134,8 +145,8 @@ This module provides an interface to the mechanisms used to implement the
134145

135146
.. function:: release_lock()
136147

137-
Release the interpreter's import lock. On platforms without threads, this
138-
function does nothing.
148+
Release the interpreter's global import lock. On platforms without
149+
threads, this function does nothing.
139150

140151

141152
.. function:: reload(module)

‎Lib/importlib/_bootstrap.py‎

Lines changed: 178 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,145 @@ def new_module(name):
159159
return type(_io)(name)
160160

161161

162+
# Module-level locking ########################################################
163+
164+
# A dict mapping module names to weakrefs of _ModuleLock instances
165+
_module_locks = {}
166+
# A dict mapping thread ids to _ModuleLock instances
167+
_blocking_on = {}
168+
169+
170+
class _DeadlockError(RuntimeError):
171+
pass
172+
173+
174+
class _ModuleLock:
175+
"""A recursive lock implementation which is able to detect deadlocks
176+
(e.g. thread 1 trying to take locks A then B, and thread 2 trying to
177+
take locks B then A).
178+
"""
179+
180+
def __init__(self, name):
181+
self.lock = _thread.allocate_lock()
182+
self.wakeup = _thread.allocate_lock()
183+
self.name = name
184+
self.owner = None
185+
self.count = 0
186+
self.waiters = 0
187+
188+
def has_deadlock(self):
189+
# Deadlock avoidance for concurrent circular imports.
190+
me = _thread.get_ident()
191+
tid = self.owner
192+
while True:
193+
lock = _blocking_on.get(tid)
194+
if lock is None:
195+
return False
196+
tid = lock.owner
197+
if tid == me:
198+
return True
199+
200+
def acquire(self):
201+
"""
202+
Acquire the module lock. If a potential deadlock is detected,
203+
a _DeadlockError is raised.
204+
Otherwise, the lock is always acquired and True is returned.
205+
"""
206+
tid = _thread.get_ident()
207+
_blocking_on[tid] = self
208+
try:
209+
while True:
210+
with self.lock:
211+
if self.count == 0 or self.owner == tid:
212+
self.owner = tid
213+
self.count += 1
214+
return True
215+
if self.has_deadlock():
216+
raise _DeadlockError("deadlock detected by %r" % self)
217+
if self.wakeup.acquire(False):
218+
self.waiters += 1
219+
# Wait for a release() call
220+
self.wakeup.acquire()
221+
self.wakeup.release()
222+
finally:
223+
del _blocking_on[tid]
224+
225+
def release(self):
226+
tid = _thread.get_ident()
227+
with self.lock:
228+
if self.owner != tid:
229+
raise RuntimeError("cannot release un-acquired lock")
230+
assert self.count > 0
231+
self.count -= 1
232+
if self.count == 0:
233+
self.owner = None
234+
if self.waiters:
235+
self.waiters -= 1
236+
self.wakeup.release()
237+
238+
def __repr__(self):
239+
return "_ModuleLock(%r) at %d" % (self.name, id(self))
240+
241+
242+
class _DummyModuleLock:
243+
"""A simple _ModuleLock equivalent for Python builds without
244+
multi-threading support."""
245+
246+
def __init__(self, name):
247+
self.name = name
248+
self.count = 0
249+
250+
def acquire(self):
251+
self.count += 1
252+
return True
253+
254+
def release(self):
255+
if self.count == 0:
256+
raise RuntimeError("cannot release un-acquired lock")
257+
self.count -= 1
258+
259+
def __repr__(self):
260+
return "_DummyModuleLock(%r) at %d" % (self.name, id(self))
261+
262+
263+
# The following two functions are for consumption by Python/import.c.
264+
265+
def _get_module_lock(name):
266+
"""Get or create the module lock for a given module name.
267+
268+
Should only be called with the import lock taken."""
269+
lock = None
270+
if name in _module_locks:
271+
lock = _module_locks[name]()
272+
if lock is None:
273+
if _thread is None:
274+
lock = _DummyModuleLock(name)
275+
else:
276+
lock = _ModuleLock(name)
277+
def cb(_):
278+
del _module_locks[name]
279+
_module_locks[name] = _weakref.ref(lock, cb)
280+
return lock
281+
282+
def _lock_unlock_module(name):
283+
"""Release the global import lock, and acquires then release the
284+
module lock for a given module name.
285+
This is used to ensure a module is completely initialized, in the
286+
event it is being imported by another thread.
287+
288+
Should only be called with the import lock taken."""
289+
lock = _get_module_lock(name)
290+
_imp.release_lock()
291+
try:
292+
lock.acquire()
293+
except _DeadlockError:
294+
# Concurrent circular import, we'll accept a partially initialized
295+
# module object.
296+
pass
297+
else:
298+
lock.release()
299+
300+
162301
# Finder/loader utility code ##################################################
163302

164303
_PYCACHE = '__pycache__'
@@ -264,12 +403,15 @@ def module_for_loader_wrapper(self, fullname, *args, **kwargs):
264403
else:
265404
module.__package__ = fullname.rpartition('.')[0]
266405
try:
406+
module.__initializing__ = True
267407
# If __package__ was not set above, __import__() will do it later.
268408
return fxn(self, module, *args, **kwargs)
269409
except:
270410
if not is_reload:
271411
del sys.modules[fullname]
272412
raise
413+
finally:
414+
module.__initializing__ = False
273415
_wrap(module_for_loader_wrapper, fxn)
274416
return module_for_loader_wrapper
275417

@@ -932,7 +1074,8 @@ def _find_module(name, path):
9321074
if not sys.meta_path:
9331075
_warnings.warn('sys.meta_path is empty', ImportWarning)
9341076
for finder in sys.meta_path:
935-
loader = finder.find_module(name, path)
1077+
with _ImportLockContext():
1078+
loader = finder.find_module(name, path)
9361079
if loader is not None:
9371080
# The parent import may have already imported this module.
9381081
if name not in sys.modules:
@@ -962,8 +1105,7 @@ def _sanity_check(name, package, level):
9621105

9631106
_ERR_MSG = 'No module named {!r}'
9641107

965-
def _find_and_load(name, import_):
966-
"""Find and load the module."""
1108+
def _find_and_load_unlocked(name, import_):
9671109
path = None
9681110
parent = name.rpartition('.')[0]
9691111
if parent:
@@ -1009,6 +1151,19 @@ def _find_and_load(name, import_):
10091151
return module
10101152

10111153

1154+
def _find_and_load(name, import_):
1155+
"""Find and load the module, and release the import lock."""
1156+
try:
1157+
lock = _get_module_lock(name)
1158+
finally:
1159+
_imp.release_lock()
1160+
lock.acquire()
1161+
try:
1162+
return _find_and_load_unlocked(name, import_)
1163+
finally:
1164+
lock.release()
1165+
1166+
10121167
def _gcd_import(name, package=None, level=0):
10131168
"""Import and return the module based on its name, the package the call is
10141169
being made from, and the level adjustment.
@@ -1021,17 +1176,17 @@ def _gcd_import(name, package=None, level=0):
10211176
_sanity_check(name, package, level)
10221177
if level > 0:
10231178
name = _resolve_name(name, package, level)
1024-
with _ImportLockContext():
1025-
try:
1026-
module = sys.modules[name]
1027-
if module is None:
1028-
message = ("import of {} halted; "
1029-
"None in sys.modules".format(name))
1030-
raise ImportError(message, name=name)
1031-
return module
1032-
except KeyError:
1033-
pass # Don't want to chain the exception
1179+
_imp.acquire_lock()
1180+
if name not in sys.modules:
10341181
return _find_and_load(name, _gcd_import)
1182+
module = sys.modules[name]
1183+
if module is None:
1184+
_imp.release_lock()
1185+
message = ("import of {} halted; "
1186+
"None in sys.modules".format(name))
1187+
raise ImportError(message, name=name)
1188+
_lock_unlock_module(name)
1189+
return module
10351190

10361191

10371192
def _handle_fromlist(module, fromlist, import_):
@@ -1149,7 +1304,17 @@ def _setup(sys_module, _imp_module):
11491304
continue
11501305
else:
11511306
raise ImportError('importlib requires posix or nt')
1307+
1308+
try:
1309+
thread_module = BuiltinImporter.load_module('_thread')
1310+
except ImportError:
1311+
# Python was built without threads
1312+
thread_module = None
1313+
weakref_module = BuiltinImporter.load_module('_weakref')
1314+
11521315
setattr(self_module, '_os', os_module)
1316+
setattr(self_module, '_thread', thread_module)
1317+
setattr(self_module, '_weakref', weakref_module)
11531318
setattr(self_module, 'path_sep', path_sep)
11541319
setattr(self_module, 'path_separators', set(path_separators))
11551320
# Constants

0 commit comments

Comments
 (0)