From da89a69e2ea048149a2fbc2a4949356d8c968229 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 13 Apr 2024 16:34:51 +0100 Subject: [PATCH 01/15] GH-117536: fix athrow().throw(...) unawaited warning --- Lib/test/test_asyncgen.py | 11 +++++++++++ Objects/genobject.c | 7 ++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 39605dca3886c8..4bbdf1e86e3478 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1752,6 +1752,17 @@ async def gen(): g.aclose() gc_collect() + def test_aclose_throw(self): + async def gen(): + return + yield + + class MyException(Exception): + pass + + g = gen() + with self.assertRaises(MyException): + gen.aclose().throw(MyException) if __name__ == "__main__": diff --git a/Objects/genobject.c b/Objects/genobject.c index 8d1dbb72ba9ec2..34f79c0c547b74 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -2208,7 +2208,11 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na retval = gen_throw((PyGenObject*)o->agt_gen, args, nargs); if (o->agt_args) { - return async_gen_unwrap_value(o->agt_gen, retval); + retval = async_gen_unwrap_value(o->agt_gen, retval); + if (retval == NULL) { + o->agt_state = AWAITABLE_STATE_CLOSED; + } + return retval; } else { /* aclose() mode */ if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { @@ -2228,6 +2232,7 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na */ PyErr_Clear(); PyErr_SetNone(PyExc_StopIteration); + o->agt_state = AWAITABLE_STATE_CLOSED; } return retval; } From f1ac5d3b47f9f1a2c4b1f64182832dcfdf840126 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 13 Apr 2024 16:53:16 +0100 Subject: [PATCH 02/15] test re-using aclose().throw(...) --- Lib/test/test_asyncgen.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 4bbdf1e86e3478..dee25550ee35d1 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1671,6 +1671,42 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_throw_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + nxt = it.aclose() + with self.assertRaises(StopIteration): + nxt.throw(GeneratorExit) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(GeneratorExit) + + def test_async_gen_throw_custom_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.aclose() + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + def test_async_gen_aclose_twice_with_different_coros(self): # Regression test for https://bugs.python.org/issue39606 async def async_iterate(): @@ -1762,7 +1798,7 @@ class MyException(Exception): g = gen() with self.assertRaises(MyException): - gen.aclose().throw(MyException) + g.aclose().throw(MyException) if __name__ == "__main__": From 733339f31ace31c123c36e6fe0664f8a37f4fa0a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 13 Apr 2024 17:40:21 +0100 Subject: [PATCH 03/15] set agt_state CLOSED on any null result --- Objects/genobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 34f79c0c547b74..5d7da49cfca166 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -2222,6 +2222,9 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; } + if (retval == NULL) { + o->agt_state = AWAITABLE_STATE_CLOSED; + } if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { @@ -2232,7 +2235,6 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na */ PyErr_Clear(); PyErr_SetNone(PyExc_StopIteration); - o->agt_state = AWAITABLE_STATE_CLOSED; } return retval; } From ac70dc2561263a6e56870f8dbc59d76251295be5 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 13 Apr 2024 17:43:50 +0100 Subject: [PATCH 04/15] remove unawaited coroutine warnings for coroutines that were awaited --- Lib/test/test_asyncgen.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index dee25550ee35d1..3dd7cd51b15aa4 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1572,11 +1572,6 @@ async def main(): self.assertIsInstance(message['exception'], ZeroDivisionError) self.assertIn('unhandled exception during asyncio.run() shutdown', message['message']) - with self.assertWarnsRegex(RuntimeWarning, - f"coroutine method 'aclose' of '{async_iterate.__qualname__}' " - f"was never awaited"): - del message, messages - gc_collect() def test_async_gen_expression_01(self): async def arange(n): @@ -1630,10 +1625,6 @@ async def main(): asyncio.run(main()) self.assertEqual([], messages) - with self.assertWarnsRegex(RuntimeWarning, - f"coroutine method 'aclose' of '{async_iterate.__qualname__}' " - f"was never awaited"): - gc_collect() def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): From e3ba8f087e29656e4d26127a52e3c3984a1b5c02 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 13 Apr 2024 17:52:11 +0100 Subject: [PATCH 05/15] await deprecated athrow(GeneratorExit, GeneratorExit(), None) --- Lib/test/test_asyncgen.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 3dd7cd51b15aa4..c66e10ef60835f 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -399,11 +399,8 @@ async def gen(): with self.assertWarns(DeprecationWarning): x = gen().athrow(GeneratorExit, GeneratorExit(), None) - with self.assertWarnsRegex(RuntimeWarning, - f"coroutine method 'athrow' of '{gen.__qualname__}' " - f"was never awaited"): - del x - gc_collect() + with self.assertRaises(GeneratorExit): + x.send(None) def test_async_gen_api_01(self): async def gen(): From 38157fb2ab4d979ca78d2f24582d43b0f340ab1f Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 13 Apr 2024 16:55:58 +0000 Subject: [PATCH 06/15] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst new file mode 100644 index 00000000000000..97f9be7e9d69c0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst @@ -0,0 +1 @@ +Fix a :exc:`RuntimeWarning` when calling ``agen.aclose().throw(Exception)`` From c3ded3334fc23549c2f18b8fb7d08acdfdfa6067 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 15 Apr 2024 14:52:49 +0100 Subject: [PATCH 07/15] Update Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst --- .../2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst index 97f9be7e9d69c0..56c50d0e49bc9e 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst @@ -1 +1,2 @@ Fix a :exc:`RuntimeWarning` when calling ``agen.aclose().throw(Exception)`` +Prevent ``agen.aclose()`` objects being re-used after ``.throw()`` From e20205843fa2f1a350980cf3255b88b1605ede13 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 13:54:03 +0000 Subject: [PATCH 08/15] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst new file mode 100644 index 00000000000000..74035849b2c52e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst @@ -0,0 +1 @@ +Prevent ``agen.aclose()`` objects being re-used after ``.throw()`` From 11e675d51b6cfc8a4177f7bd12495b856c96801b Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 15 Apr 2024 14:54:31 +0100 Subject: [PATCH 09/15] Update Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst --- .../2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst index 56c50d0e49bc9e..97f9be7e9d69c0 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst @@ -1,2 +1 @@ Fix a :exc:`RuntimeWarning` when calling ``agen.aclose().throw(Exception)`` -Prevent ``agen.aclose()`` objects being re-used after ``.throw()`` From d291fe29975671c952dbff910e3f974b01cff3e6 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 15 Apr 2024 15:58:23 +0100 Subject: [PATCH 10/15] Apply suggestions from code review --- .../2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst | 2 +- .../2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst index 97f9be7e9d69c0..2492fd163cb549 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-16-55-53.gh-issue-117536.xkVbfv.rst @@ -1 +1 @@ -Fix a :exc:`RuntimeWarning` when calling ``agen.aclose().throw(Exception)`` +Fix a :exc:`RuntimeWarning` when calling ``agen.aclose().throw(Exception)``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst index 74035849b2c52e..bd32500a54ee21 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-15-13-53-59.gh-issue-117894.8LpZ6m.rst @@ -1 +1 @@ -Prevent ``agen.aclose()`` objects being re-used after ``.throw()`` +Prevent ``agen.aclose()`` objects being re-used after ``.throw()``. From e501ac8aa6bb0596968591cbcf6a7614b38d84c3 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 17 Apr 2024 22:44:48 +0100 Subject: [PATCH 11/15] also test athrow() --- Lib/test/test_asyncgen.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index c66e10ef60835f..1dc0ecb9e01ece 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1695,6 +1695,26 @@ class MyException(Exception): ): nxt.throw(MyException) + def test_async_gen_throw_custom_same_athrow_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.athrow(MyException) + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + def test_async_gen_aclose_twice_with_different_coros(self): # Regression test for https://bugs.python.org/issue39606 async def async_iterate(): From 7f839a16b1cc3f36d649b478f1c2672bb2c0e050 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 17 Apr 2024 22:46:02 +0100 Subject: [PATCH 12/15] add a gc.collect() to test_aclose_throw --- Lib/test/test_asyncgen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 1dc0ecb9e01ece..8db48d1eb3f892 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1807,6 +1807,7 @@ class MyException(Exception): g = gen() with self.assertRaises(MyException): g.aclose().throw(MyException) + gc.collect() if __name__ == "__main__": From cd4236b09ed346cba9e576acdb2f2a1e0ff03820 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 17 Apr 2024 22:46:55 +0100 Subject: [PATCH 13/15] add gc.collect() to test_async_gen_3_arg_deprecation_warning --- Lib/test/test_asyncgen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 8db48d1eb3f892..7c79406176db8a 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -401,6 +401,7 @@ async def gen(): x = gen().athrow(GeneratorExit, GeneratorExit(), None) with self.assertRaises(GeneratorExit): x.send(None) + gc.collect() def test_async_gen_api_01(self): async def gen(): From 7070ab6cd0607b15351d1d26cb3e49f673556430 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 17 Apr 2024 22:53:31 +0100 Subject: [PATCH 14/15] delete objects that might warn --- Lib/test/test_asyncgen.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 7c79406176db8a..66f98432d68992 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -401,6 +401,7 @@ async def gen(): x = gen().athrow(GeneratorExit, GeneratorExit(), None) with self.assertRaises(GeneratorExit): x.send(None) + del x gc.collect() def test_async_gen_api_01(self): @@ -1808,6 +1809,7 @@ class MyException(Exception): g = gen() with self.assertRaises(MyException): g.aclose().throw(MyException) + del g gc.collect() From 5800ca44811db003a7a862ae15d3f80e75d7c79d Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 17 Apr 2024 22:54:38 +0100 Subject: [PATCH 15/15] gc.collect -> gc_collect --- Lib/test/test_asyncgen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 66f98432d68992..a1e9e1b89c6a2c 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -402,7 +402,7 @@ async def gen(): with self.assertRaises(GeneratorExit): x.send(None) del x - gc.collect() + gc_collect() def test_async_gen_api_01(self): async def gen(): @@ -1810,7 +1810,7 @@ class MyException(Exception): with self.assertRaises(MyException): g.aclose().throw(MyException) del g - gc.collect() + gc_collect() if __name__ == "__main__":