From 823f1c77ddb49952b3a659b759893a32ff772176 Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Wed, 25 May 2022 15:34:31 +0000 Subject: [PATCH 01/14] Fix broken :class:`asyncio.Semaphore` and strengthen FIFO guarantee. --- Lib/asyncio/locks.py | 20 ++-- Lib/test/test_asyncio/test_locks.py | 92 +++++++++++++++++++ ...2-05-25-15-57-39.gh-issue-90155.YMstB5.rst | 1 + 3 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-05-25-15-57-39.gh-issue-90155.YMstB5.rst diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index e71130274dd6f3..8b356eb978d4ff 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -348,7 +348,7 @@ def __init__(self, value=1): raise ValueError("Semaphore initial value must be >= 0") self._value = value self._waiters = collections.deque() - self._wakeup_scheduled = False + self._wakeup_scheduled = 0 def __repr__(self): res = super().__repr__() @@ -362,7 +362,7 @@ def _wake_up_next(self): waiter = self._waiters.popleft() if not waiter.done(): waiter.set_result(None) - self._wakeup_scheduled = True + self._wakeup_scheduled += 1 return def locked(self): @@ -378,18 +378,22 @@ async def acquire(self): called release() to make it larger than 0, and then return True. """ - # _wakeup_scheduled is set if *another* task is scheduled to wakeup - # but its acquire() is not resumed yet - while self._wakeup_scheduled or self._value <= 0: + freshman = True + while self._value <= 0 or (freshman and self._wakeup_scheduled): + freshman = False fut = self._get_loop().create_future() self._waiters.append(fut) try: await fut - # reset _wakeup_scheduled *after* waiting for a future - self._wakeup_scheduled = False except exceptions.CancelledError: - self._wake_up_next() + if not fut.cancelled(): + self._wakeup_scheduled -= 1 + self._wake_up_next() raise + else: + self._wakeup_scheduled -= 1 + if self._value > 1 and self._wakeup_scheduled == 0: + self._wake_up_next() self._value -= 1 return True diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index ff25d9edef518f..9c1f89ed20749f 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -907,6 +907,27 @@ async def test_acquire_hang(self): self.assertTrue(sem.locked()) self.assertTrue(t2.done()) + async def test_acquire_no_hang(self): + + sem = asyncio.Semaphore(1) + + async def c1(tasks): + async with sem: + await asyncio.sleep(0) + tasks[1].cancel() + + async def c2(tasks): + async with sem: + await asyncio.sleep(0) + + tasks = [] + tasks.append(asyncio.create_task(c1(tasks))) + tasks.append(asyncio.create_task(c2(tasks))) + + await asyncio.gather(*tasks, return_exceptions=True) + + await asyncio.wait_for(sem.acquire(), timeout=0.01) + def test_release_not_acquired(self): sem = asyncio.BoundedSemaphore() @@ -945,6 +966,77 @@ async def coro(tag): result ) + async def test_acquire_fifo_order_2(self): + sem = asyncio.Semaphore(1) + result = [] + + async def c1(result): + await sem.acquire() + result.append(1) + return True + + async def c2(result): + await sem.acquire() + result.append(2) + sem.release() + await sem.acquire() + result.append(4) + return True + + async def c3(result): + await sem.acquire() + result.append(3) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + + sem.release() + sem.release() + + tasks = [t1, t2, t3] + await asyncio.gather(*tasks) + self.assertEqual([1, 2, 3, 4], result) + + async def test_acquire_fifo_order_3(self): + sem = asyncio.Semaphore(0) + result = [] + + async def c1(result): + await sem.acquire() + result.append(1) + return True + + async def c2(result): + await sem.acquire() + result.append(2) + return True + + async def c3(result): + await sem.acquire() + result.append(3) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + + t1.cancel() + + await asyncio.sleep(0.01) + + sem.release() + sem.release() + + tasks = [t1, t2, t3] + await asyncio.gather(*tasks, return_exceptions=True) + self.assertEqual([2, 3], result) + class BarrierTests(unittest.IsolatedAsyncioTestCase): diff --git a/Misc/NEWS.d/next/Library/2022-05-25-15-57-39.gh-issue-90155.YMstB5.rst b/Misc/NEWS.d/next/Library/2022-05-25-15-57-39.gh-issue-90155.YMstB5.rst new file mode 100644 index 00000000000000..533b130b75f79b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-25-15-57-39.gh-issue-90155.YMstB5.rst @@ -0,0 +1 @@ +Fix broken :class:`asyncio.Semaphore` and strengthen FIFO guarantee. From db55bc5dcc97e729eb0003b300d28e6b200ea357 Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Fri, 10 Jun 2022 17:57:01 +0000 Subject: [PATCH 02/14] Fix `locked` method. --- Lib/asyncio/locks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 8b356eb978d4ff..e5a1adb49a9c0e 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -367,7 +367,7 @@ def _wake_up_next(self): def locked(self): """Returns True if semaphore can not be acquired immediately.""" - return self._value == 0 + return self._value <= 0 or self._wakeup_scheduled async def acquire(self): """Acquire a semaphore. From 0ada9f76db3d36cb68d15c280187fbcbe9f93544 Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Thu, 15 Sep 2022 07:25:30 +0000 Subject: [PATCH 03/14] Response to code review r967894274 and r967894421. --- Lib/asyncio/locks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index e5a1adb49a9c0e..cc57d7d5182958 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -367,7 +367,7 @@ def _wake_up_next(self): def locked(self): """Returns True if semaphore can not be acquired immediately.""" - return self._value <= 0 or self._wakeup_scheduled + return self._value <= 0 or bool(self._wakeup_scheduled) async def acquire(self): """Acquire a semaphore. @@ -378,9 +378,9 @@ async def acquire(self): called release() to make it larger than 0, and then return True. """ - freshman = True - while self._value <= 0 or (freshman and self._wakeup_scheduled): - freshman = False + first = True + while self._value <= 0 or (first and self._wakeup_scheduled): + first = False fut = self._get_loop().create_future() self._waiters.append(fut) try: From 2192dc73e411139e6a7044fb8a5601f71831f3b8 Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Sat, 17 Sep 2022 20:13:13 +0000 Subject: [PATCH 04/14] Implement Semaphore using patched Lock implementation. --- Lib/asyncio/locks.py | 68 +++++++++++++++++------------ Lib/test/test_asyncio/test_locks.py | 10 ++--- 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index cc57d7d5182958..f16e31bd4c609a 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -346,9 +346,8 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin): def __init__(self, value=1): if value < 0: raise ValueError("Semaphore initial value must be >= 0") + self._waiters = None self._value = value - self._waiters = collections.deque() - self._wakeup_scheduled = 0 def __repr__(self): res = super().__repr__() @@ -357,44 +356,44 @@ def __repr__(self): extra = f'{extra}, waiters:{len(self._waiters)}' return f'<{res[1:-1]} [{extra}]>' - def _wake_up_next(self): - while self._waiters: - waiter = self._waiters.popleft() - if not waiter.done(): - waiter.set_result(None) - self._wakeup_scheduled += 1 - return - def locked(self): """Returns True if semaphore can not be acquired immediately.""" - return self._value <= 0 or bool(self._wakeup_scheduled) + return self._value <= 0 async def acquire(self): """Acquire a semaphore. - If the internal counter is larger than zero on entry, decrement it by one and return True immediately. If it is zero on entry, block, waiting until some other coroutine has called release() to make it larger than 0, and then return True. """ - first = True - while self._value <= 0 or (first and self._wakeup_scheduled): - first = False - fut = self._get_loop().create_future() - self._waiters.append(fut) + if (not self.locked() and (self._waiters is None or + all(w.cancelled() for w in self._waiters))): + self._value -= 1 + return True + + if self._waiters is None: + self._waiters = collections.deque() + fut = self._get_loop().create_future() + self._waiters.append(fut) + + # Finally block should be called before the CancelledError + # handling as we don't want CancelledError to call + # _wake_up_first() and attempt to wake up itself. + try: try: await fut - except exceptions.CancelledError: - if not fut.cancelled(): - self._wakeup_scheduled -= 1 - self._wake_up_next() - raise - else: - self._wakeup_scheduled -= 1 - if self._value > 1 and self._wakeup_scheduled == 0: - self._wake_up_next() + finally: + self._waiters.remove(fut) + except exceptions.CancelledError: + if not self.locked(): + self._wake_up_first() + raise + self._value -= 1 + if not self.locked(): + self._wake_up_first() return True def release(self): @@ -403,7 +402,22 @@ def release(self): become larger than zero again, wake up that coroutine. """ self._value += 1 - self._wake_up_next() + self._wake_up_first() + + def _wake_up_first(self): + """Wake up the first waiter if it isn't done.""" + if not self._waiters: + return + try: + fut = next(iter(self._waiters)) + except StopIteration: + return + + # .done() necessarily means that a waiter will wake up later on and + # either take the lock, or, if it was cancelled and lock wasn't + # taken already, will hit this again and wake up a new waiter. + if not fut.done(): + fut.set_result(True) class BoundedSemaphore(Semaphore): diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 9c1f89ed20749f..ca369404289954 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -830,7 +830,7 @@ async def c4(result): t2 = asyncio.create_task(c2(result)) t3 = asyncio.create_task(c3(result)) - await asyncio.sleep(0) + await asyncio.sleep(0.01) self.assertEqual([1], result) self.assertTrue(sem.locked()) self.assertEqual(2, len(sem._waiters)) @@ -842,7 +842,7 @@ async def c4(result): sem.release() self.assertEqual(2, sem._value) - await asyncio.sleep(0) + await asyncio.sleep(0.01) self.assertEqual(0, sem._value) self.assertEqual(3, len(result)) self.assertTrue(sem.locked()) @@ -878,13 +878,13 @@ async def test_acquire_cancel_before_awoken(self): t3 = asyncio.create_task(sem.acquire()) t4 = asyncio.create_task(sem.acquire()) - await asyncio.sleep(0) + await asyncio.sleep(0.01) t1.cancel() t2.cancel() sem.release() - await asyncio.sleep(0) + await asyncio.sleep(0.01) num_done = sum(t.done() for t in [t3, t4]) self.assertEqual(num_done, 1) self.assertTrue(t3.done()) @@ -903,7 +903,7 @@ async def test_acquire_hang(self): t1.cancel() sem.release() - await asyncio.sleep(0) + await asyncio.sleep(0.01) self.assertTrue(sem.locked()) self.assertTrue(t2.done()) From c7513a5eba7eaea98c2444f48addccfe32c01006 Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Sat, 17 Sep 2022 20:37:33 +0000 Subject: [PATCH 05/14] Fix asyncio test. --- Lib/test/test_asyncio/test_locks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index ca369404289954..ad1af78efaf858 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -5,6 +5,7 @@ import re import asyncio +import collections STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' @@ -774,6 +775,9 @@ async def test_repr(self): self.assertTrue('waiters' not in repr(sem)) self.assertTrue(RGX_REPR.match(repr(sem))) + if sem._waiters is None: + sem._waiters = collections.deque() + sem._waiters.append(mock.Mock()) self.assertTrue('waiters:1' in repr(sem)) self.assertTrue(RGX_REPR.match(repr(sem))) From 9932c4b330adc7ec4f6f4dddcbf7cb6e88bd5d94 Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Sat, 17 Sep 2022 21:34:57 +0000 Subject: [PATCH 06/14] Include updates from code review. --- Lib/test/test_asyncio/test_locks.py | 15 ++++++++------- .../2022-05-25-15-57-39.gh-issue-90155.YMstB5.rst | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index ad1af78efaf858..13a48636bbbcf3 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -915,20 +915,21 @@ async def test_acquire_no_hang(self): sem = asyncio.Semaphore(1) - async def c1(tasks): + async def c1(): async with sem: await asyncio.sleep(0) - tasks[1].cancel() + t2.cancel() - async def c2(tasks): + async def c2(): async with sem: await asyncio.sleep(0) - tasks = [] - tasks.append(asyncio.create_task(c1(tasks))) - tasks.append(asyncio.create_task(c2(tasks))) + t1 = asyncio.create_task(c1()) + t2 = asyncio.create_task(c2()) - await asyncio.gather(*tasks, return_exceptions=True) + result = await asyncio.gather(t1, t2, return_exceptions=True) + self.assertTrue(result[0] is None) + self.assertTrue(isinstance(result[1], asyncio.CancelledError)) await asyncio.wait_for(sem.acquire(), timeout=0.01) diff --git a/Misc/NEWS.d/next/Library/2022-05-25-15-57-39.gh-issue-90155.YMstB5.rst b/Misc/NEWS.d/next/Library/2022-05-25-15-57-39.gh-issue-90155.YMstB5.rst index 533b130b75f79b..8def76914eda08 100644 --- a/Misc/NEWS.d/next/Library/2022-05-25-15-57-39.gh-issue-90155.YMstB5.rst +++ b/Misc/NEWS.d/next/Library/2022-05-25-15-57-39.gh-issue-90155.YMstB5.rst @@ -1 +1 @@ -Fix broken :class:`asyncio.Semaphore` and strengthen FIFO guarantee. +Fix broken :class:`asyncio.Semaphore` when acquire is cancelled. From 9a6a0533b8c5c94950c699a43689d68891f59b52 Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Sat, 17 Sep 2022 21:53:06 +0000 Subject: [PATCH 07/14] Include code review changes. --- Lib/test/test_asyncio/test_locks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 13a48636bbbcf3..9939d84f6b7006 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -922,7 +922,7 @@ async def c1(): async def c2(): async with sem: - await asyncio.sleep(0) + self.assertFalse(True) t1 = asyncio.create_task(c1()) t2 = asyncio.create_task(c2()) From 59f0516d413310f0735337222c0f464b8dd23baa Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Sun, 18 Sep 2022 20:50:55 +0000 Subject: [PATCH 08/14] Code review. --- Lib/test/test_asyncio/test_locks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 9939d84f6b7006..e2ee9e2fb27f7f 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -834,7 +834,7 @@ async def c4(result): t2 = asyncio.create_task(c2(result)) t3 = asyncio.create_task(c3(result)) - await asyncio.sleep(0.01) + await asyncio.sleep(0) self.assertEqual([1], result) self.assertTrue(sem.locked()) self.assertEqual(2, len(sem._waiters)) From a90aa699b4fd70fd69750a91d4587babed299a1e Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Sun, 18 Sep 2022 21:05:56 +0000 Subject: [PATCH 09/14] Code review. --- Lib/test/test_asyncio/test_locks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index e2ee9e2fb27f7f..9cd70a4773eeaf 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -927,9 +927,9 @@ async def c2(): t1 = asyncio.create_task(c1()) t2 = asyncio.create_task(c2()) - result = await asyncio.gather(t1, t2, return_exceptions=True) - self.assertTrue(result[0] is None) - self.assertTrue(isinstance(result[1], asyncio.CancelledError)) + r1, r2 = await asyncio.gather(t1, t2, return_exceptions=True) + self.assertTrue(r1 is None) + self.assertTrue(isinstance(r2, asyncio.CancelledError)) await asyncio.wait_for(sem.acquire(), timeout=0.01) From f63d13015a57758253585d66038a5323686cd1db Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Sun, 18 Sep 2022 21:15:07 +0000 Subject: [PATCH 10/14] Update `Semaphore.locked`. `Semaphore` docstring says the counter can never go below zero. --- Lib/asyncio/locks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index f16e31bd4c609a..3c8b2f1721ba8d 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -357,8 +357,8 @@ def __repr__(self): return f'<{res[1:-1]} [{extra}]>' def locked(self): - """Returns True if semaphore can not be acquired immediately.""" - return self._value <= 0 + """Returns True if semaphore counter is zero.""" + return self._value == 0 async def acquire(self): """Acquire a semaphore. From 1ba48c4601240b02a32f416753baab7aa0b9f7fe Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Tue, 20 Sep 2022 16:10:53 +0000 Subject: [PATCH 11/14] Code review. --- Lib/asyncio/locks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 3c8b2f1721ba8d..f8f590304e31dc 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -362,6 +362,7 @@ def locked(self): async def acquire(self): """Acquire a semaphore. + If the internal counter is larger than zero on entry, decrement it by one and return True immediately. If it is zero on entry, block, waiting until some other coroutine has @@ -398,6 +399,7 @@ async def acquire(self): def release(self): """Release a semaphore, incrementing the internal counter by one. + When it was zero on entry and another coroutine is waiting for it to become larger than zero again, wake up that coroutine. """ From ea9b33cb11d3fbfd3fd12785d368462c963e1971 Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Wed, 21 Sep 2022 23:25:51 +0000 Subject: [PATCH 12/14] Code review. --- Lib/test/test_asyncio/test_locks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 9cd70a4773eeaf..d8ac2f7aa836df 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -846,7 +846,8 @@ async def c4(result): sem.release() self.assertEqual(2, sem._value) - await asyncio.sleep(0.01) + await asyncio.sleep(0) + await asyncio.sleep(0) self.assertEqual(0, sem._value) self.assertEqual(3, len(result)) self.assertTrue(sem.locked()) From abbef56cd0622e271ccf6c191f9b290d0d576a5d Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Wed, 21 Sep 2022 23:28:18 +0000 Subject: [PATCH 13/14] Code review. --- Lib/test/test_asyncio/test_locks.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index d8ac2f7aa836df..70b53e37a67f9a 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -883,13 +883,14 @@ async def test_acquire_cancel_before_awoken(self): t3 = asyncio.create_task(sem.acquire()) t4 = asyncio.create_task(sem.acquire()) - await asyncio.sleep(0.01) + await asyncio.sleep(0) t1.cancel() t2.cancel() sem.release() - await asyncio.sleep(0.01) + await asyncio.sleep(0) + await asyncio.sleep(0) num_done = sum(t.done() for t in [t3, t4]) self.assertEqual(num_done, 1) self.assertTrue(t3.done()) @@ -908,7 +909,8 @@ async def test_acquire_hang(self): t1.cancel() sem.release() - await asyncio.sleep(0.01) + await asyncio.sleep(0) + await asyncio.sleep(0) self.assertTrue(sem.locked()) self.assertTrue(t2.done()) @@ -1034,7 +1036,7 @@ async def c3(result): t1.cancel() - await asyncio.sleep(0.01) + await asyncio.sleep(0) sem.release() sem.release() From dc2f9622930fa15cc7851c117c1d5655faab770c Mon Sep 17 00:00:00 2001 From: Cyker Way Date: Wed, 21 Sep 2022 23:32:30 +0000 Subject: [PATCH 14/14] Code review. --- Lib/test/test_asyncio/test_locks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 70b53e37a67f9a..6937a449b8eec4 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -934,7 +934,7 @@ async def c2(): self.assertTrue(r1 is None) self.assertTrue(isinstance(r2, asyncio.CancelledError)) - await asyncio.wait_for(sem.acquire(), timeout=0.01) + await asyncio.wait_for(sem.acquire(), timeout=1.0) def test_release_not_acquired(self): sem = asyncio.BoundedSemaphore()