From 2a4d4f3d4b48bbfa12f1a461d1a8422ef4d99d1d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Jun 2022 10:00:24 -0400 Subject: [PATCH 01/33] Add some implementation documentation This might make it a little bit easier for a new reader/maintainer to understand what this code is doing. Also, link to a couple application-level bug reports about probable misbehaviors related to the deadlock-detection code. --- Lib/importlib/_bootstrap.py | 70 ++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index afb95f4e1df869..ef627642c51329 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -71,9 +71,30 @@ class _ModuleLock: def __init__(self, name): self.lock = _thread.allocate_lock() self.wakeup = _thread.allocate_lock() + # The name of the module for which this is a lock. self.name = name + + # Either None if this lock is not owned by any thread or the thread + # identifier for the owning thread. self.owner = None + + # This is a count of the number of times the owning thread has + # acquired this lock. This supports RLock-like ("re-entrant lock") + # behavior, necessary in case a single thread is following a circular + # import dependency and needs to take the lock for a single module + # more than once. self.count = 0 + + # This is a count of the number of threads that are blocking on + # `self.wakeup.acquire()` to try to get their turn holding this module + # lock. When the module lock is released, if this is greater than + # zero, it is decremented and `self.wakeup` is released one time. The + # intent is that this will let one other thread make more progress on + # acquiring this module lock. This repeats until all the threads have + # gotten a turn. + # + # This is incremented in `self.acquire` when a thread notices it is + # going to have to wait for another thread to finish. self.waiters = 0 def has_deadlock(self): @@ -107,17 +128,64 @@ def acquire(self): _blocking_on[tid] = self try: while True: + # Protect interaction with state on self with a per-module + # lock. This makes it safe for more than one thread to try to + # acquire the lock for a single module at the same time. with self.lock: if self.count == 0 or self.owner == tid: + # If the lock for this module is unowned then we can + # take the lock immediately and succeed. If the lock + # for this module is owned by the running thread then + # we can also allow the acquire to succeed. This + # supports circular imports (thread T imports module A + # which imports module B which imports module A). self.owner = tid self.count += 1 return True + + + # At this point we know the lock is held (because count != + # 0) by another thread (because owner != tid). We'll have + # to get in line to take the module lock. + + # But first, check to see if this thread would create a + # deadlock by acquiring this module lock. If it would + # then just stop with an error. + # + # XXX It's not clear who is expected to handle this error. + # There is one handler in _lock_unlock_module but many + # times this method is called when entering the context + # manager _ModuleLockManager instead - so _DeadlockError + # will just propagate up to application code. + # + # This seems to be more than just a hypothetical - + # https://stackoverflow.com/questions/59509154 + # https://github.com/encode/django-rest-framework/issues/7078 if self.has_deadlock(): raise _DeadlockError('deadlock detected by %r' % self) + + + # Check to see if we're going to be able to acquire the + # lock. If we are going to have to wait then increment + # the waiters so `self.release` will know to unblock us + # later on. We do this part non-blockingly so we don't + # get stuck here before we increment waiters. We have + # this extra acquire call (in addition to the one below, + # outside the self.lock context manager) to make sure + # self.wakeup is held when the next acquire is called (so + # we block). This is probably needlessly complex and we + # should just take self.wakeup in the return codepath + # above. if self.wakeup.acquire(False): self.waiters += 1 - # Wait for a release() call + + # Now blockingly take the lock. This won't complete until the + # thread holding this lock (self.owner) calls self.release. self.wakeup.acquire() + + # Taking it has served its purpose (making us wait) so we can + # give it up now. We'll take it non-blockingly again on the + # next iteration around this while loop. self.wakeup.release() finally: del _blocking_on[tid] From c9e33c796160df6d5817b6fa6e887b0455c361ea Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Jun 2022 10:18:12 -0400 Subject: [PATCH 02/33] Move _blocking_on management into a context manager This more clearly separates the logic for that management from the application code that runs in this context. It will also make subsequent changes to improve that logic more clear. --- Lib/importlib/_bootstrap.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index ef627642c51329..4ac378a365db58 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -58,6 +58,33 @@ def _new_module(name): _blocking_on = {} +class _BlockingOnManager: + """ + A context manager responsible to updating ``_blocking_on`` to track which + threads are likely blocked on taking the import locks for which modules. + """ + def __init__(self, tid, lock): + # The id of the thread in which this manager is being used. + self.tid = tid + # The _ModuleLock for a certain module which the running thread wants + # to take. + self.lock = lock + + def __enter__(self): + """ + Mark the running thread as waiting for the lock this manager knows + about. + """ + _blocking_on[self.tid] = self.lock + + def __exit__(self, *args, **kwargs): + """ + Mark the running thread as no longer waiting for the lock this manager + knows about. + """ + del _blocking_on[self.tid] + + class _DeadlockError(RuntimeError): pass @@ -125,8 +152,7 @@ def acquire(self): Otherwise, the lock is always acquired and True is returned. """ tid = _thread.get_ident() - _blocking_on[tid] = self - try: + with _BlockingOnManager(tid, self): while True: # Protect interaction with state on self with a per-module # lock. This makes it safe for more than one thread to try to @@ -187,8 +213,6 @@ def acquire(self): # give it up now. We'll take it non-blockingly again on the # next iteration around this while loop. self.wakeup.release() - finally: - del _blocking_on[tid] def release(self): tid = _thread.get_ident() From c5912bf5a949fb5086406d5015895e78653ae06b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 30 Jun 2022 19:55:11 -0400 Subject: [PATCH 03/33] Support re-entrant imports in _BlockingOnManager Also update the deadlock detection to work with the new _blocking_on Switch to a recursive implementation so it can easily follow the branching path through the "blocking on" graph that is now possible thanks to re-entrancy. --- Lib/importlib/_bootstrap.py | 152 +++++++++++++++++++++++++++--------- 1 file changed, 117 insertions(+), 35 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 4ac378a365db58..ed9a602d5bc678 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -54,7 +54,14 @@ def _new_module(name): # A dict mapping module names to weakrefs of _ModuleLock instances # Dictionary protected by the global import lock _module_locks = {} -# A dict mapping thread ids to _ModuleLock instances + +# A dict mapping thread ids to lists of _ModuleLock instances. This maps a +# thread to the module locks it is blocking on acquiring. The values are +# lists because a single thread could perform a re-entrant import and be "in +# the process" of blocking on locks for more than one module. "in the +# process" because a thread cannot actually block on acquiring more than one +# lock but it can have set up bookkeeping that reflects that it intends to +# block on acquiring more than one lock. _blocking_on = {} @@ -75,20 +82,75 @@ def __enter__(self): Mark the running thread as waiting for the lock this manager knows about. """ - _blocking_on[self.tid] = self.lock + # Interactions with _blocking_on are *not* protected by the global + # import lock here because each thread only touches the state that it + # owns (state keyed on its thread id). The global import lock is + # re-entrant (ie, a single thread may take it more than once) so it + # wouldn't help us be correct in the face of re-entrancy either. + + # First look up the module locks the running thread already intends to + # take. If this thread hasn't done an import before, it may not be + # present in the dict so be sure to initialize it in this case. + self.blocked_on = _blocking_on.setdefault(self.tid, []) + + # Whether we are re-entering or not, add this lock to the list because + # now this thread is going to be blocked on it. + self.blocked_on.append(self.lock) def __exit__(self, *args, **kwargs): """ Mark the running thread as no longer waiting for the lock this manager knows about. """ - del _blocking_on[self.tid] + self.blocked_on.remove(self.lock) class _DeadlockError(RuntimeError): pass + +def _has_deadlock(seen, subject, tids, _blocking_on): + """ + Considering a graph where nodes are threads (represented by their id + as keys in ``blocking_on``) and edges are "blocked on" relationships + (represented by values in ``_blocking_on``), determine whether ``subject`` + is reachable starting from any of the threads given by ``tids``. + + :param seen: A set of threads that have already been visited. + :param subject: The thread id to try to reach. + :param tids: The thread ids from which to begin. + :param blocking_on: A dict representing the thread/blocking-on graph. + """ + if subject in tids: + # If we have already reached the subject, we're done - signal that it + # is reachable. + return True + + # Otherwise, try to reach the subject from each of the given tids. + for tid in tids: + blocking_on = _blocking_on.get(tid) + if blocking_on is None: + # There are no edges out from this node, skip it. + continue + + if tid in seen: + # bpo 38091: the chain of tid's we encounter here + # eventually leads to a fixpoint or a cycle, but + # does not reach 'me'. This means we would not + # actually deadlock. This can happen if other + # threads are at the beginning of acquire() below. + return False + seen.add(tid) + + # Follow the edges out from this thread. + edges = [lock.owner for lock in blocking_on] + if _has_deadlock(seen, subject, edges, _blocking_on): + return True + + return False + + class _ModuleLock: """A recursive lock implementation which is able to detect deadlocks (e.g. thread 1 trying to take locks A then B, and thread 2 trying to @@ -96,8 +158,29 @@ class _ModuleLock: """ def __init__(self, name): - self.lock = _thread.allocate_lock() + # Create an RLock for protecting the import process for the + # corresponding module. Since it is an RLock a single thread will be + # able to take it more than once. This is necessary to support + # re-entrancy in the import system that arises from (at least) signal + # handlers and the garbage collector. Consider the case of: + # + # import foo + # -> ... + # -> importlib._bootstrap._ModuleLock.acquire + # -> ... + # -> + # -> __del__ + # -> import foo + # -> ... + # -> importlib._bootstrap._ModuleLock.acquire + # -> _BlockingOnManager.__enter__ + # + # If a different thread than the running thread holds the lock then it + # will have to block on taking it which is just what we want for + # thread safety. + self.lock = _thread.RLock() self.wakeup = _thread.allocate_lock() + # The name of the module for which this is a lock. self.name = name @@ -110,7 +193,11 @@ def __init__(self, name): # behavior, necessary in case a single thread is following a circular # import dependency and needs to take the lock for a single module # more than once. - self.count = 0 + # + # Counts are represented as a list of None because list.append(None) + # and list.pop() are both atomic and thread-safe and it's hard to find + # another primitive with the same properties. + self.count = [] # This is a count of the number of threads that are blocking on # `self.wakeup.acquire()` to try to get their turn holding this module @@ -122,28 +209,25 @@ def __init__(self, name): # # This is incremented in `self.acquire` when a thread notices it is # going to have to wait for another thread to finish. - self.waiters = 0 + # + # See the comment above count for explanation of the representation. + self.waiters = [] def has_deadlock(self): - # Deadlock avoidance for concurrent circular imports. - me = _thread.get_ident() - tid = self.owner - seen = set() - while True: - lock = _blocking_on.get(tid) - if lock is None: - return False - tid = lock.owner - if tid == me: - return True - if tid in seen: - # bpo 38091: the chain of tid's we encounter here - # eventually leads to a fixpoint or a cycle, but - # does not reach 'me'. This means we would not - # actually deadlock. This can happen if other - # threads are at the beginning of acquire() below. - return False - seen.add(tid) + # To avoid deadlocks for concurrent or re-entrant circular imports, + # look at the "blocking on" state to see if any threads are blocking + # on getting the import lock for any module for which the import lock + # is held by this thread. + return _has_deadlock( + seen=set(), + # Try to find this thread + subject=_thread.get_ident(), + # starting from the thread that holds the import lock for this + # module. + tids=[self.owner], + # using the global "blocking on" state. + _blocking_on=_blocking_on, + ) def acquire(self): """ @@ -158,7 +242,7 @@ def acquire(self): # lock. This makes it safe for more than one thread to try to # acquire the lock for a single module at the same time. with self.lock: - if self.count == 0 or self.owner == tid: + if self.count == [] or self.owner == tid: # If the lock for this module is unowned then we can # take the lock immediately and succeed. If the lock # for this module is owned by the running thread then @@ -166,10 +250,9 @@ def acquire(self): # supports circular imports (thread T imports module A # which imports module B which imports module A). self.owner = tid - self.count += 1 + self.count.append(None) return True - # At this point we know the lock is held (because count != # 0) by another thread (because owner != tid). We'll have # to get in line to take the module lock. @@ -190,7 +273,6 @@ def acquire(self): if self.has_deadlock(): raise _DeadlockError('deadlock detected by %r' % self) - # Check to see if we're going to be able to acquire the # lock. If we are going to have to wait then increment # the waiters so `self.release` will know to unblock us @@ -203,7 +285,7 @@ def acquire(self): # should just take self.wakeup in the return codepath # above. if self.wakeup.acquire(False): - self.waiters += 1 + self.waiters.append(None) # Now blockingly take the lock. This won't complete until the # thread holding this lock (self.owner) calls self.release. @@ -219,12 +301,12 @@ def release(self): with self.lock: if self.owner != tid: raise RuntimeError('cannot release un-acquired lock') - assert self.count > 0 - self.count -= 1 - if self.count == 0: + assert len(self.count) > 0 + self.count.pop() + if len(self.count) == 0: self.owner = None - if self.waiters: - self.waiters -= 1 + if len(self.waiters) > 0: + self.waiters.pop() self.wakeup.release() def __repr__(self): From 27ae2f07ceb5e16191bdc8cb46b821edef7b742e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 13:52:27 -0500 Subject: [PATCH 04/33] Apply formatting and comment changes from review Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 54 +++++++++++++------------------------ 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index ed9a602d5bc678..f6087f1b17776c 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -66,42 +66,25 @@ def _new_module(name): class _BlockingOnManager: - """ - A context manager responsible to updating ``_blocking_on`` to track which - threads are likely blocked on taking the import locks for which modules. - """ + """A context manager responsible to updating ``_blocking_on``.""" def __init__(self, tid, lock): # The id of the thread in which this manager is being used. self.tid = tid - # The _ModuleLock for a certain module which the running thread wants - # to take. self.lock = lock def __enter__(self): - """ - Mark the running thread as waiting for the lock this manager knows - about. - """ + """Mark the running thread as waiting for self.lock. via _blocking_on.""" # Interactions with _blocking_on are *not* protected by the global # import lock here because each thread only touches the state that it # owns (state keyed on its thread id). The global import lock is - # re-entrant (ie, a single thread may take it more than once) so it + # re-entrant (i.e., a single thread may take it more than once) so it # wouldn't help us be correct in the face of re-entrancy either. - # First look up the module locks the running thread already intends to - # take. If this thread hasn't done an import before, it may not be - # present in the dict so be sure to initialize it in this case. self.blocked_on = _blocking_on.setdefault(self.tid, []) - - # Whether we are re-entering or not, add this lock to the list because - # now this thread is going to be blocked on it. self.blocked_on.append(self.lock) def __exit__(self, *args, **kwargs): - """ - Mark the running thread as no longer waiting for the lock this manager - knows about. - """ + """Remove self.lock from this thread's _blocking_on list.""" self.blocked_on.remove(self.lock) @@ -111,11 +94,12 @@ class _DeadlockError(RuntimeError): def _has_deadlock(seen, subject, tids, _blocking_on): + """Check if 'subject' is holding the same lock as another thread(s). + + The search within _blocking_on starts with the threads listed in tids. + 'seen' contains any threads that are considered already traversed in the search. + """ - Considering a graph where nodes are threads (represented by their id - as keys in ``blocking_on``) and edges are "blocked on" relationships - (represented by values in ``_blocking_on``), determine whether ``subject`` - is reachable starting from any of the threads given by ``tids``. :param seen: A set of threads that have already been visited. :param subject: The thread id to try to reach. @@ -136,7 +120,7 @@ def _has_deadlock(seen, subject, tids, _blocking_on): if tid in seen: # bpo 38091: the chain of tid's we encounter here - # eventually leads to a fixpoint or a cycle, but + # eventually leads to a fixed point or a cycle, but # does not reach 'me'. This means we would not # actually deadlock. This can happen if other # threads are at the beginning of acquire() below. @@ -159,7 +143,7 @@ class _ModuleLock: def __init__(self, name): # Create an RLock for protecting the import process for the - # corresponding module. Since it is an RLock a single thread will be + # corresponding module. Since it is an RLock, a single thread will be # able to take it more than once. This is necessary to support # re-entrancy in the import system that arises from (at least) signal # handlers and the garbage collector. Consider the case of: @@ -188,15 +172,15 @@ def __init__(self, name): # identifier for the owning thread. self.owner = None - # This is a count of the number of times the owning thread has - # acquired this lock. This supports RLock-like ("re-entrant lock") + # Represent the number of times the owning thread has acquired this lock + # via a list of `True`. This supports RLock-like ("re-entrant lock") # behavior, necessary in case a single thread is following a circular # import dependency and needs to take the lock for a single module # more than once. # # Counts are represented as a list of None because list.append(None) - # and list.pop() are both atomic and thread-safe and it's hard to find - # another primitive with the same properties. + # and list.pop() are both atomic and thread-safe in CPython and it's hard + # to find another primitive with the same properties. self.count = [] # This is a count of the number of threads that are blocking on @@ -261,7 +245,7 @@ def acquire(self): # deadlock by acquiring this module lock. If it would # then just stop with an error. # - # XXX It's not clear who is expected to handle this error. + # It's not clear who is expected to handle this error. # There is one handler in _lock_unlock_module but many # times this method is called when entering the context # manager _ModuleLockManager instead - so _DeadlockError @@ -271,7 +255,7 @@ def acquire(self): # https://stackoverflow.com/questions/59509154 # https://github.com/encode/django-rest-framework/issues/7078 if self.has_deadlock(): - raise _DeadlockError('deadlock detected by %r' % self) + raise _DeadlockError(f'deadlock detected by {self!r}') # Check to see if we're going to be able to acquire the # lock. If we are going to have to wait then increment @@ -291,9 +275,9 @@ def acquire(self): # thread holding this lock (self.owner) calls self.release. self.wakeup.acquire() - # Taking it has served its purpose (making us wait) so we can + # Taking the lock has served its purpose (making us wait), so we can # give it up now. We'll take it non-blockingly again on the - # next iteration around this while loop. + # next iteration around this 'while' loop. self.wakeup.release() def release(self): From 5fd15d836aaf5545236473318c684d9c7d9c3f62 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 13:56:37 -0500 Subject: [PATCH 05/33] Rename _BlockingOnManager.tid as suggested by review --- Lib/importlib/_bootstrap.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 79fb6bf8d17ca5..e02e8218c23583 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -67,9 +67,8 @@ def _new_module(name): class _BlockingOnManager: """A context manager responsible to updating ``_blocking_on``.""" - def __init__(self, tid, lock): - # The id of the thread in which this manager is being used. - self.tid = tid + def __init__(self, thread_id, lock): + self.thread_id = tid self.lock = lock def __enter__(self): @@ -80,7 +79,7 @@ def __enter__(self): # re-entrant (i.e., a single thread may take it more than once) so it # wouldn't help us be correct in the face of re-entrancy either. - self.blocked_on = _blocking_on.setdefault(self.tid, []) + self.blocked_on = _blocking_on.setdefault(self.thread_id, []) self.blocked_on.append(self.lock) def __exit__(self, *args, **kwargs): @@ -95,10 +94,10 @@ class _DeadlockError(RuntimeError): def _has_deadlock(seen, subject, tids, _blocking_on): """Check if 'subject' is holding the same lock as another thread(s). - + The search within _blocking_on starts with the threads listed in tids. 'seen' contains any threads that are considered already traversed in the search. - + """ :param seen: A set of threads that have already been visited. From 7a24f2c50a98aa277fb656de5964f43c57fec946 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:00:21 -0500 Subject: [PATCH 06/33] flip the first two arguments to _has_deadlock as suggested by review Also rename it as suggested by review --- Lib/importlib/_bootstrap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index e02e8218c23583..d885c45f89e106 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -92,7 +92,7 @@ class _DeadlockError(RuntimeError): -def _has_deadlock(seen, subject, tids, _blocking_on): +def _has_deadlocked(subject, seen, tids, _blocking_on): """Check if 'subject' is holding the same lock as another thread(s). The search within _blocking_on starts with the threads listed in tids. @@ -128,7 +128,7 @@ def _has_deadlock(seen, subject, tids, _blocking_on): # Follow the edges out from this thread. edges = [lock.owner for lock in blocking_on] - if _has_deadlock(seen, subject, edges, _blocking_on): + if _has_deadlocked(subject, seen, edges, _blocking_on): return True return False @@ -201,10 +201,10 @@ def has_deadlock(self): # look at the "blocking on" state to see if any threads are blocking # on getting the import lock for any module for which the import lock # is held by this thread. - return _has_deadlock( - seen=set(), + return _has_deadlocked( # Try to find this thread subject=_thread.get_ident(), + seen=set(), # starting from the thread that holds the import lock for this # module. tids=[self.owner], From 08892b4c957eca6e575f7228a282a8836d7e17ef Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:00:48 -0500 Subject: [PATCH 07/33] Mark up parameters following PEP 257 as suggested by review See https://peps.python.org/pep-0257/#multi-line-docstrings --- Lib/importlib/_bootstrap.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index d885c45f89e106..1b8cb7e8bfba01 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -98,12 +98,11 @@ def _has_deadlocked(subject, seen, tids, _blocking_on): The search within _blocking_on starts with the threads listed in tids. 'seen' contains any threads that are considered already traversed in the search. - """ - - :param seen: A set of threads that have already been visited. - :param subject: The thread id to try to reach. - :param tids: The thread ids from which to begin. - :param blocking_on: A dict representing the thread/blocking-on graph. + Keyword arguments: + subject -- The thread id to try to reach. + seen -- A set of threads that have already been visited. + tids -- The thread ids from which to begin. + blocking_on -- A dict representing the thread/blocking-on graph. """ if subject in tids: # If we have already reached the subject, we're done - signal that it From 59b53c01870cf318995381a30b7e63d27ee684a5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:04:29 -0500 Subject: [PATCH 08/33] rename the `_blocking_on` parameter as suggested by review --- Lib/importlib/_bootstrap.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 1b8cb7e8bfba01..dae9fb45a4ab97 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -92,10 +92,10 @@ class _DeadlockError(RuntimeError): -def _has_deadlocked(subject, seen, tids, _blocking_on): +def _has_deadlocked(subject, seen, tids, blocking_on): """Check if 'subject' is holding the same lock as another thread(s). - The search within _blocking_on starts with the threads listed in tids. + The search within blocking_on starts with the threads listed in tids. 'seen' contains any threads that are considered already traversed in the search. Keyword arguments: @@ -111,7 +111,7 @@ def _has_deadlocked(subject, seen, tids, _blocking_on): # Otherwise, try to reach the subject from each of the given tids. for tid in tids: - blocking_on = _blocking_on.get(tid) + blocking_on = blocking_on.get(tid) if blocking_on is None: # There are no edges out from this node, skip it. continue @@ -127,7 +127,7 @@ def _has_deadlocked(subject, seen, tids, _blocking_on): # Follow the edges out from this thread. edges = [lock.owner for lock in blocking_on] - if _has_deadlocked(subject, seen, edges, _blocking_on): + if _has_deadlocked(subject, seen, edges, blocking_on): return True return False @@ -208,7 +208,7 @@ def has_deadlock(self): # module. tids=[self.owner], # using the global "blocking on" state. - _blocking_on=_blocking_on, + blocking_on=_blocking_on, ) def acquire(self): From 20007c598beb4453c4ea9e1d2db5ca3fe03c9ccb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:04:39 -0500 Subject: [PATCH 09/33] further document motivation for `blocking_on` parameter as suggested by review --- Lib/importlib/_bootstrap.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index dae9fb45a4ab97..15f732a58a1f76 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -103,6 +103,9 @@ def _has_deadlocked(subject, seen, tids, blocking_on): seen -- A set of threads that have already been visited. tids -- The thread ids from which to begin. blocking_on -- A dict representing the thread/blocking-on graph. + This may be the same object as the global '_blocking_on' + but it is a parameter to reduce the impact that global + mutable state has on the result of this function. """ if subject in tids: # If we have already reached the subject, we're done - signal that it From bad1d3c5e703d8fe0490076ddcce2f518ee84470 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:08:03 -0500 Subject: [PATCH 10/33] Rename more _has_deadlocked parameters as suggested by review --- Lib/importlib/_bootstrap.py | 43 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 15f732a58a1f76..206b2d70104b19 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -92,45 +92,46 @@ class _DeadlockError(RuntimeError): -def _has_deadlocked(subject, seen, tids, blocking_on): - """Check if 'subject' is holding the same lock as another thread(s). +def _has_deadlocked(target_id, seen_ids, candidate_ids, blocking_on): + """Check if 'target_id' is holding the same lock as another thread(s). - The search within blocking_on starts with the threads listed in tids. - 'seen' contains any threads that are considered already traversed in the search. + The search within 'blocking_on' starts with the threads listed in + 'candidate_ids'. 'seen_ids' contains any threads that are considered + already traversed in the search. Keyword arguments: - subject -- The thread id to try to reach. - seen -- A set of threads that have already been visited. - tids -- The thread ids from which to begin. - blocking_on -- A dict representing the thread/blocking-on graph. - This may be the same object as the global '_blocking_on' - but it is a parameter to reduce the impact that global - mutable state has on the result of this function. + target_id -- The thread id to try to reach. + seen_ids -- A set of threads that have already been visited. + candidate_ids -- The thread ids from which to begin. + blocking_on -- A dict representing the thread/blocking-on graph. This may + be the same object as the global '_blocking_on' but it is + a parameter to reduce the impact that global mutable + state has on the result of this function. """ - if subject in tids: - # If we have already reached the subject, we're done - signal that it + if target_id in candidate_ids: + # If we have already reached the target_id, we're done - signal that it # is reachable. return True - # Otherwise, try to reach the subject from each of the given tids. - for tid in tids: + # Otherwise, try to reach the target_id from each of the given candidate_ids. + for tid in candidate_ids: blocking_on = blocking_on.get(tid) if blocking_on is None: # There are no edges out from this node, skip it. continue - if tid in seen: + if tid in seen_ids: # bpo 38091: the chain of tid's we encounter here # eventually leads to a fixed point or a cycle, but # does not reach 'me'. This means we would not # actually deadlock. This can happen if other # threads are at the beginning of acquire() below. return False - seen.add(tid) + seen_ids.add(tid) # Follow the edges out from this thread. edges = [lock.owner for lock in blocking_on] - if _has_deadlocked(subject, seen, edges, blocking_on): + if _has_deadlocked(target_id, seen_ids, edges, blocking_on): return True return False @@ -205,11 +206,11 @@ def has_deadlock(self): # is held by this thread. return _has_deadlocked( # Try to find this thread - subject=_thread.get_ident(), - seen=set(), + target_id=_thread.get_ident(), + seen_ids=set(), # starting from the thread that holds the import lock for this # module. - tids=[self.owner], + candidate_ids=[self.owner], # using the global "blocking on" state. blocking_on=_blocking_on, ) From 2821fcf6f01d542eda64659e7ab30c719e5e2897 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:09:24 -0500 Subject: [PATCH 11/33] Treat None and [] the same for this case as suggested by review --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 206b2d70104b19..8721f3420e6872 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -116,7 +116,7 @@ def _has_deadlocked(target_id, seen_ids, candidate_ids, blocking_on): # Otherwise, try to reach the target_id from each of the given candidate_ids. for tid in candidate_ids: blocking_on = blocking_on.get(tid) - if blocking_on is None: + if not blocking_on: # There are no edges out from this node, skip it. continue From 95c73cb6269f37058a9fb7f02f7c6a60e7e8b0b3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:10:16 -0500 Subject: [PATCH 12/33] update old comment to refer to new names as suggested by review --- Lib/importlib/_bootstrap.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 8721f3420e6872..fc32611a0c60cd 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -121,11 +121,10 @@ def _has_deadlocked(target_id, seen_ids, candidate_ids, blocking_on): continue if tid in seen_ids: - # bpo 38091: the chain of tid's we encounter here - # eventually leads to a fixed point or a cycle, but - # does not reach 'me'. This means we would not - # actually deadlock. This can happen if other - # threads are at the beginning of acquire() below. + # bpo 38091: the chain of tid's we encounter here eventually leads + # to a fixed point or a cycle, but does not reach 'target_id'. + # This means we would not actually deadlock. This can happen if + # other threads are at the beginning of acquire() below. return False seen_ids.add(tid) From 49ff9ddf03f2b9ae74e06ccc6bfa7e3022dbfad7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:14:33 -0500 Subject: [PATCH 13/33] Make _ModuleLock.count a list of True as suggested by review --- Lib/importlib/_bootstrap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index fc32611a0c60cd..fb4d6d2742e211 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -179,7 +179,7 @@ def __init__(self, name): # import dependency and needs to take the lock for a single module # more than once. # - # Counts are represented as a list of None because list.append(None) + # Counts are represented as a list of True because list.append(True) # and list.pop() are both atomic and thread-safe in CPython and it's hard # to find another primitive with the same properties. self.count = [] @@ -235,7 +235,7 @@ def acquire(self): # supports circular imports (thread T imports module A # which imports module B which imports module A). self.owner = tid - self.count.append(None) + self.count.append(True) return True # At this point we know the lock is held (because count != From cd174a82dbb68ff27d1d6f6e8bb5a54958e20b03 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:16:25 -0500 Subject: [PATCH 14/33] Adjust the check for a module lock being released as suggest by review Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index fb4d6d2742e211..7887390621d8ea 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -288,7 +288,7 @@ def release(self): raise RuntimeError('cannot release un-acquired lock') assert len(self.count) > 0 self.count.pop() - if len(self.count) == 0: + if not len(self.count): self.owner = None if len(self.waiters) > 0: self.waiters.pop() From 719b1812fef19e99fa60dda7fcb2e7442cc18a97 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:19:47 -0500 Subject: [PATCH 15/33] Finish the _BlockingOnManager.tid renaming --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 7887390621d8ea..2db9aaf8775d3f 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -68,7 +68,7 @@ def _new_module(name): class _BlockingOnManager: """A context manager responsible to updating ``_blocking_on``.""" def __init__(self, thread_id, lock): - self.thread_id = tid + self.thread_id = thread_id self.lock = lock def __enter__(self): From 6e809cdabab3b6b240b9a96b450a4b367732a9a6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 28 Nov 2022 14:48:21 -0500 Subject: [PATCH 16/33] Fix renaming of `_blocking_on` parameter to `_has_deadlocked` --- Lib/importlib/_bootstrap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 2db9aaf8775d3f..f3d483a87c73dd 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -115,8 +115,8 @@ def _has_deadlocked(target_id, seen_ids, candidate_ids, blocking_on): # Otherwise, try to reach the target_id from each of the given candidate_ids. for tid in candidate_ids: - blocking_on = blocking_on.get(tid) - if not blocking_on: + candidate_blocking_on = blocking_on.get(tid) + if not candidate_blocking_on: # There are no edges out from this node, skip it. continue @@ -129,7 +129,7 @@ def _has_deadlocked(target_id, seen_ids, candidate_ids, blocking_on): seen_ids.add(tid) # Follow the edges out from this thread. - edges = [lock.owner for lock in blocking_on] + edges = [lock.owner for lock in candidate_blocking_on] if _has_deadlocked(target_id, seen_ids, edges, blocking_on): return True From 74cbccdd305c46402487aeb06620f3a4971c1a53 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:05:40 -0500 Subject: [PATCH 17/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index f3d483a87c73dd..1ee25770c3b405 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -55,13 +55,13 @@ def _new_module(name): # Dictionary protected by the global import lock _module_locks = {} -# A dict mapping thread ids to lists of _ModuleLock instances. This maps a +# A dict mapping thread IDs to lists of _ModuleLock instances. This maps a # thread to the module locks it is blocking on acquiring. The values are # lists because a single thread could perform a re-entrant import and be "in -# the process" of blocking on locks for more than one module. "in the -# process" because a thread cannot actually block on acquiring more than one -# lock but it can have set up bookkeeping that reflects that it intends to -# block on acquiring more than one lock. +# the process" of blocking on locks for more than one module. A thread can +# be "in the process" because a thread cannot actually block on acquiring +# more than one lock but it can have set up bookkeeping that reflects that +# it intends to block on acquiring more than one lock. _blocking_on = {} From c57953362ecf8dd89493c6d5f35aa45e6cc72eae Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:05:56 -0500 Subject: [PATCH 18/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 1ee25770c3b405..ec20e3e40eeded 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -119,8 +119,7 @@ def _has_deadlocked(target_id, seen_ids, candidate_ids, blocking_on): if not candidate_blocking_on: # There are no edges out from this node, skip it. continue - - if tid in seen_ids: + elif tid in seen_ids: # bpo 38091: the chain of tid's we encounter here eventually leads # to a fixed point or a cycle, but does not reach 'target_id'. # This means we would not actually deadlock. This can happen if From decb70b3983d207a33f1187fb0ee3fb26c51711d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:06:08 -0500 Subject: [PATCH 19/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index ec20e3e40eeded..96f7be9ee69c82 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -159,9 +159,9 @@ def __init__(self, name): # -> importlib._bootstrap._ModuleLock.acquire # -> _BlockingOnManager.__enter__ # - # If a different thread than the running thread holds the lock then it - # will have to block on taking it which is just what we want for - # thread safety. + # If a different thread than the running one holds the lock then the + # thread will have to block on taking the lock, which is what we want + # for thread safety. self.lock = _thread.RLock() self.wakeup = _thread.allocate_lock() From 3c91cc32863280383a15b8a1ef571d39c6482a4c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:06:19 -0500 Subject: [PATCH 20/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 96f7be9ee69c82..b2e7b18169a5af 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -168,8 +168,8 @@ def __init__(self, name): # The name of the module for which this is a lock. self.name = name - # Either None if this lock is not owned by any thread or the thread - # identifier for the owning thread. + # Can end up being set to None if this lock is not owned by any thread + # or the thread identifier for the owning thread. self.owner = None # Represent the number of times the owning thread has acquired this lock From e032ae2fc3e95e16d8b1bbbf740d9f70d185205f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:06:29 -0500 Subject: [PATCH 21/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index b2e7b18169a5af..ff5d1b1f8ba9a9 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -184,7 +184,7 @@ def __init__(self, name): self.count = [] # This is a count of the number of threads that are blocking on - # `self.wakeup.acquire()` to try to get their turn holding this module + # self.wakeup.acquire() to try to get their turn holding this module # lock. When the module lock is released, if this is greater than # zero, it is decremented and `self.wakeup` is released one time. The # intent is that this will let one other thread make more progress on From dba393a7cec5e02889745c112f5580a7a611a0c4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:06:43 -0500 Subject: [PATCH 22/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index ff5d1b1f8ba9a9..d299196215f0b2 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -121,7 +121,7 @@ def _has_deadlocked(target_id, seen_ids, candidate_ids, blocking_on): continue elif tid in seen_ids: # bpo 38091: the chain of tid's we encounter here eventually leads - # to a fixed point or a cycle, but does not reach 'target_id'. + # to a fixed point or a cycle, but does not reach target_id. # This means we would not actually deadlock. This can happen if # other threads are at the beginning of acquire() below. return False From 25d554bcac3645099571876f91f639447617154f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:06:54 -0500 Subject: [PATCH 23/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index d299196215f0b2..e54845cc2c0867 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -191,7 +191,7 @@ def __init__(self, name): # acquiring this module lock. This repeats until all the threads have # gotten a turn. # - # This is incremented in `self.acquire` when a thread notices it is + # This is incremented in self.acquire() when a thread notices it is # going to have to wait for another thread to finish. # # See the comment above count for explanation of the representation. From 44157a98aa08e1c6b8e4dcd5abef504e79d8d2c1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:07:05 -0500 Subject: [PATCH 24/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index e54845cc2c0867..578c89a3d9f42b 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -276,7 +276,7 @@ def acquire(self): self.wakeup.acquire() # Taking the lock has served its purpose (making us wait), so we can - # give it up now. We'll take it non-blockingly again on the + # give it up now. We'll take it w/o blocking again on the # next iteration around this 'while' loop. self.wakeup.release() From f99ed46b7f209c5554246d2f779e7fbf0ba9dda0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:07:18 -0500 Subject: [PATCH 25/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 578c89a3d9f42b..4ba0339ce6270b 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -173,7 +173,7 @@ def __init__(self, name): self.owner = None # Represent the number of times the owning thread has acquired this lock - # via a list of `True`. This supports RLock-like ("re-entrant lock") + # via a list of True. This supports RLock-like ("re-entrant lock") # behavior, necessary in case a single thread is following a circular # import dependency and needs to take the lock for a single module # more than once. From b6d21f8e468f65ccf3723ecf1d21d193570f9974 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:07:34 -0500 Subject: [PATCH 26/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 4ba0339ce6270b..78b3c6371063df 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -199,7 +199,7 @@ def __init__(self, name): def has_deadlock(self): # To avoid deadlocks for concurrent or re-entrant circular imports, - # look at the "blocking on" state to see if any threads are blocking + # look at _blocking_on to see if any threads are blocking # on getting the import lock for any module for which the import lock # is held by this thread. return _has_deadlocked( From 5643442f7e69ef8569aed9d0c5b379ace269182e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:07:47 -0500 Subject: [PATCH 27/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 78b3c6371063df..22bbd5d7fdb72d 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -203,7 +203,7 @@ def has_deadlock(self): # on getting the import lock for any module for which the import lock # is held by this thread. return _has_deadlocked( - # Try to find this thread + # Try to find this thread. target_id=_thread.get_ident(), seen_ids=set(), # starting from the thread that holds the import lock for this From 92036a8b11406e9d95c047170b1e929f790ad689 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:08:05 -0500 Subject: [PATCH 28/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 22bbd5d7fdb72d..680cf8367e6413 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -206,7 +206,7 @@ def has_deadlock(self): # Try to find this thread. target_id=_thread.get_ident(), seen_ids=set(), - # starting from the thread that holds the import lock for this + # Start from the thread that holds the import lock for this # module. candidate_ids=[self.owner], # using the global "blocking on" state. From bf14ce28bd056cd1a72f97be931c30984612bde0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:08:17 -0500 Subject: [PATCH 29/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 680cf8367e6413..f298a0ecded8dc 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -209,7 +209,7 @@ def has_deadlock(self): # Start from the thread that holds the import lock for this # module. candidate_ids=[self.owner], - # using the global "blocking on" state. + # Use the global "blocking on" state. blocking_on=_blocking_on, ) From 1cc6033dfbb3f3f92f5a2d44a22c4b0ff1d2d079 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:08:30 -0500 Subject: [PATCH 30/33] Apply review suggestion Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index f298a0ecded8dc..e37622b8be30f1 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -271,8 +271,9 @@ def acquire(self): if self.wakeup.acquire(False): self.waiters.append(None) - # Now blockingly take the lock. This won't complete until the - # thread holding this lock (self.owner) calls self.release. + # Now take the lock in a blocking fashion. This won't + # complete until the thread holding this lock + # (self.owner) calls self.release. self.wakeup.acquire() # Taking the lock has served its purpose (making us wait), so we can From ab4073768fb32dda48f9c4f52a782053654143fc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:09:58 -0500 Subject: [PATCH 31/33] Apply review suggestion --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index e37622b8be30f1..b9765395842021 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -184,7 +184,7 @@ def __init__(self, name): self.count = [] # This is a count of the number of threads that are blocking on - # self.wakeup.acquire() to try to get their turn holding this module + # self.wakeup.acquire() awaiting to get their turn holding this module # lock. When the module lock is released, if this is greater than # zero, it is decremented and `self.wakeup` is released one time. The # intent is that this will let one other thread make more progress on From 36082eb81057fe16edc41831846229ce726b4cde Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 09:23:06 -0500 Subject: [PATCH 32/33] news blurb --- .../2023-01-06-09-22-21.gh-issue-91351.iq2vZ_.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-01-06-09-22-21.gh-issue-91351.iq2vZ_.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-06-09-22-21.gh-issue-91351.iq2vZ_.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-06-09-22-21.gh-issue-91351.iq2vZ_.rst new file mode 100644 index 00000000000000..19de1f8d0fb31e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-01-06-09-22-21.gh-issue-91351.iq2vZ_.rst @@ -0,0 +1,5 @@ +Fix a case where re-entrant imports could corrupt the import deadlock +detection code and cause a :exc:`KeyError` to be raised out of +:mod:`importlib/_bootstrap`. In addition to the straightforward cases, this +could also happen when garbage collection leads to a warning being emitted -- +as happens when it collects an open socket or file) From b35f0a833df3ce93ad335c55e8fcffadf2666431 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 14 Jan 2023 08:09:09 -0500 Subject: [PATCH 33/33] Apply suggestions from code review Co-authored-by: Brett Cannon --- Lib/importlib/_bootstrap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index b9765395842021..bebe7e15cbce67 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -92,7 +92,7 @@ class _DeadlockError(RuntimeError): -def _has_deadlocked(target_id, seen_ids, candidate_ids, blocking_on): +def _has_deadlocked(target_id, *, seen_ids, candidate_ids, blocking_on): """Check if 'target_id' is holding the same lock as another thread(s). The search within 'blocking_on' starts with the threads listed in @@ -115,8 +115,7 @@ def _has_deadlocked(target_id, seen_ids, candidate_ids, blocking_on): # Otherwise, try to reach the target_id from each of the given candidate_ids. for tid in candidate_ids: - candidate_blocking_on = blocking_on.get(tid) - if not candidate_blocking_on: + if not (candidate_blocking_on := blocking_on.get(tid)): # There are no edges out from this node, skip it. continue elif tid in seen_ids: @@ -129,7 +128,8 @@ def _has_deadlocked(target_id, seen_ids, candidate_ids, blocking_on): # Follow the edges out from this thread. edges = [lock.owner for lock in candidate_blocking_on] - if _has_deadlocked(target_id, seen_ids, edges, blocking_on): + if _has_deadlocked(target_id, seen_ids=seen_ids, candidate_ids=edges, + blocking_on=blocking_on): return True return False