From 0138ff1212dbe2344aa4fa036620676abe177989 Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Fri, 3 Nov 2023 23:17:06 -0400 Subject: [PATCH 01/11] Add sys._set_exception() to expose the `PyErr_SetHandledExecption` function Exposing this function allows Python code to clear/set the current exception context as returned by `sys.exception()`. It is prefixed by an underscore since this functionality is not intended to be accessed by user-written code. In order to allow `sys._set_exception(None)` to clear the current exception `_PyErr_GetTopmostException` was changed to consider `Py_None` a valid exception instead of an indication to keep traversing the stack like `NULL` is. Additionally, `PyErr_SetHandledException` was updated to add `Py_None` to the exception stack instead of `NULL`. Put together, these changes allow `PyErr_SetHandledException(NULL)` to mask the entire exception chain and "clear" the current exception state as documented in `Doc/c-api/exceptions.rst`. Furthermore, since `sys._set_exception()` uses `PyErr_SetHandledException`, this allows `sys._set_exception(None)` to clear the current exception context instead of just exposing the next exception in the stack. --- Lib/test/test_sys.py | 117 ++++++++++++++++++ ...-12-19-17-27-08.gh-issue-111375.XeHOl_.rst | 2 + Python/clinic/sysmodule.c.h | 65 +++++++++- Python/errors.c | 3 +- Python/sysmodule.c | 27 ++++ 5 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2023-12-19-17-27-08.gh-issue-111375.XeHOl_.rst diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index fb1c8492a64d38..d71858f5b256f7 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -161,6 +161,123 @@ def f(): self.assertIsInstance(e, ValueError) self.assertIs(exc, e) +class SetExceptionTests(unittest.TestCase): + + def tearDown(self): + # make sure we don't leave the global exception set + sys._set_exception(None); + + def test_set_exc_invalid_values(self): + for x in (0, "1", b"2"): + with self.assertRaises(TypeError): + sys._set_exception(x); + + def test_clear_exc(self): + try: + raise ValueError() + except ValueError: + self.assertIsInstance(sys.exception(), ValueError) + sys._set_exception(None) + self.assertIsNone(sys.exception()) + + def test_set_exc(self): + exc = ValueError() + self.assertIsNone(sys.exception()) + sys._set_exception(exc) + self.assertIs(sys.exception(), exc) + + def test_set_exc_replaced_by_new_exception_and_restored(self): + exc = ValueError() + sys._set_exception(exc) + self.assertIs(sys.exception(), exc) + try: + raise TypeError() + except TypeError: + self.assertIsInstance(sys.exception(), TypeError) + self.assertIs(sys.exception().__context__, exc) + + self.assertIs(sys.exception(), exc) + + def test_set_exc_popped_on_exit_except(self): + exc = ValueError() + try: + raise TypeError() + except TypeError: + self.assertIsInstance(sys.exception(), TypeError) + sys._set_exception(exc) + self.assertIs(sys.exception(), exc) + self.assertIsNone(sys.exception()) + + def test_cleared_exc_overridden_and_restored(self): + try: + raise ValueError() + except ValueError: + try: + raise TypeError() + except TypeError: + self.assertIsInstance(sys.exception(), TypeError) + sys._set_exception(None) + self.assertIsNone(sys.exception()) + try: + raise IndexError() + except IndexError: + self.assertIsInstance(sys.exception(), IndexError) + self.assertIsNone(sys.exception().__context__) + self.assertIsNone(sys.exception()) + self.assertIsInstance(sys.exception(), ValueError) + self.assertIsNone(sys.exception()) + + def test_clear_exc_in_generator(self): + def inner(): + self.assertIsNone(sys.exception()) + yield + self.assertIsInstance(sys.exception(), ValueError) + sys._set_exception(None) + self.assertIsNone(sys.exception()) + yield + self.assertIsNone(sys.exception()) + + # with a single exception in exc_info stack + g = inner() + next(g) + try: + raise ValueError() + except: + self.assertIsInstance(sys.exception(), ValueError) + next(g) + self.assertIsInstance(sys.exception(), ValueError) + self.assertIsNone(sys.exception()) + with self.assertRaises(StopIteration): + next(g) + self.assertIsNone(sys.exception()) + + # with multiple exceptions in exc_info stack by chaining generators + def outer(): + g = inner() + self.assertIsNone(sys.exception()) + yield next(g) + self.assertIsInstance(sys.exception(), TypeError) + try: + raise ValueError() + except: + self.assertIsInstance(sys.exception(), ValueError) + self.assertIsInstance(sys.exception().__context__, TypeError) + yield next(g) + self.assertIsInstance(sys.exception(), ValueError) + self.assertIsInstance(sys.exception(), TypeError) + + g = outer() + next(g) + try: + raise TypeError() + except: + self.assertIsInstance(sys.exception(), TypeError) + next(g) + self.assertIsInstance(sys.exception(), TypeError) + self.assertIsNone(sys.exception()) + with self.assertRaises(StopIteration): + next(g) + self.assertIsNone(sys.exception()) class ExceptHookTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2023-12-19-17-27-08.gh-issue-111375.XeHOl_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2023-12-19-17-27-08.gh-issue-111375.XeHOl_.rst new file mode 100644 index 00000000000000..037211c1b20f50 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2023-12-19-17-27-08.gh-issue-111375.XeHOl_.rst @@ -0,0 +1,2 @@ +Add sys._set_exception() function that can set/clear the current exception +context. Patch by Carey Metcalfe. diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index a47e4d11b54441..bb9ffaee22fe4e 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -179,6 +179,69 @@ sys_exception(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys_exception_impl(module); } +PyDoc_STRVAR(sys__set_exception__doc__, +"_set_exception($module, /, exception)\n" +"--\n" +"\n" +"Set the current exception.\n" +"\n" +"Subsequent calls to sys.exception()/sys.exc_info() will return\n" +"the provided exception until another exception is raised in the\n" +"current thread or the execution stack returns to a frame where\n" +"another exception is being handled."); + +#define SYS__SET_EXCEPTION_METHODDEF \ + {"_set_exception", _PyCFunction_CAST(sys__set_exception), METH_FASTCALL|METH_KEYWORDS, sys__set_exception__doc__}, + +static PyObject * +sys__set_exception_impl(PyObject *module, PyObject *exception); + +static PyObject * +sys__set_exception(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(exception), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"exception", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_set_exception", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *exception; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + exception = args[0]; + return_value = sys__set_exception_impl(module, exception); + +exit: + return return_value; +} + PyDoc_STRVAR(sys_exc_info__doc__, "exc_info($module, /)\n" "--\n" @@ -1948,4 +2011,4 @@ _jit_is_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=449d16326e69dcf6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=320dffdcb61d9e35 input=a9049054013a1b77]*/ diff --git a/Python/errors.c b/Python/errors.c index 81f267b043afaf..4e7ef757a5f5b8 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -122,7 +122,6 @@ _PyErr_GetTopmostException(PyThreadState *tstate) { exc_info = exc_info->previous_item; } - assert(!Py_IsNone(exc_info->exc_value)); return exc_info; } @@ -598,7 +597,7 @@ PyErr_GetHandledException(void) void _PyErr_SetHandledException(PyThreadState *tstate, PyObject *exc) { - Py_XSETREF(tstate->exc_info->exc_value, Py_XNewRef(exc == Py_None ? NULL : exc)); + Py_XSETREF(tstate->exc_info->exc_value, Py_XNewRef(exc ? exc : Py_None)); } void diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 4ed045e3297bbc..48bf219c4a56d0 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -853,6 +853,32 @@ sys_exception_impl(PyObject *module) Py_RETURN_NONE; } +/*[clinic input] +sys._set_exception + exception: object + +Set the current exception. + +Subsequent calls to sys.exception()/sys.exc_info() will return +the provided exception until another exception is raised in the +current thread or the execution stack returns to a frame where +another exception is being handled. +[clinic start generated code]*/ + +static PyObject * +sys__set_exception_impl(PyObject *module, PyObject *exception) +/*[clinic end generated code: output=39e119ee6b747085 input=46da3b45313a1cfa]*/ +{ + if (!Py_IsNone(exception) && !PyExceptionInstance_Check(exception)){ + PyErr_SetString( + PyExc_TypeError, + "must be an exception/None" + ); + return NULL; + } + PyErr_SetHandledException(exception); + Py_RETURN_NONE; +} /*[clinic input] sys.exc_info @@ -2772,6 +2798,7 @@ static PyMethodDef sys_methods[] = { SYS__CURRENT_EXCEPTIONS_METHODDEF SYS_DISPLAYHOOK_METHODDEF SYS_EXCEPTION_METHODDEF + SYS__SET_EXCEPTION_METHODDEF SYS_EXC_INFO_METHODDEF SYS_EXCEPTHOOK_METHODDEF SYS_EXIT_METHODDEF From b285e9a413649c00228e42dddefa791336d4f588 Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Fri, 3 Nov 2023 01:16:21 -0400 Subject: [PATCH 02/11] Fix exception context in @contextmanagers Previously, when using a `try...yield...except` construct within a `@contextmanager` function, an exception raised by the `yield` wouldn't be properly cleared when handled the `except` block. Instead, calling `sys.exception()` would continue to return the exception, leading to confusing tracebacks and logs. This was happening due to the way that the `@contextmanager` decorator drives its decorated generator function. When an exception occurs, it uses `.throw(exc)` to throw the exception into the generator. However, even if the generator handles this exception, because the exception was thrown into it in the context of the `@contextmanager` decorator handling it (and not being finished yet), `sys.exception()` was not being reset. In order to fix this, the exception context as the `@contextmanager` decorator is `__enter__`'d is stored and set as the current exception context just before throwing the new exception into the generator. Doing this means that after the generator handles the thrown exception, `sys.exception()` reverts back to what it was when the `@contextmanager` function was started. --- Lib/contextlib.py | 12 +++ Lib/test/test_contextlib.py | 82 +++++++++++++++++++ ...-11-03-01-44-51.gh-issue-111375.qDBcCI.rst | 2 + 3 files changed, 96 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-11-03-01-44-51.gh-issue-111375.qDBcCI.rst diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 5b646fabca0225..d8d03f84e85b42 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -106,6 +106,7 @@ class _GeneratorContextManagerBase: """Shared functionality for @contextmanager and @asynccontextmanager.""" def __init__(self, func, args, kwds): + self.exc_context = None self.gen = func(*args, **kwds) self.func, self.args, self.kwds = func, args, kwds # Issue 19330: ensure context manager instances have good docstrings @@ -134,6 +135,8 @@ class _GeneratorContextManager( """Helper for @contextmanager decorator.""" def __enter__(self): + # store the exception context on enter so it can be restored on exit + self.exc_context = sys.exception() # do not keep args and kwds alive unnecessarily # they are only needed for recreation, which is not possible anymore del self.args, self.kwds, self.func @@ -143,6 +146,9 @@ def __enter__(self): raise RuntimeError("generator didn't yield") from None def __exit__(self, typ, value, traceback): + # don't keep the stored exception alive unnecessarily + exc_context = self.exc_context + self.exc_context = None if typ is None: try: next(self.gen) @@ -159,6 +165,12 @@ def __exit__(self, typ, value, traceback): # tell if we get the same exception back value = typ() try: + # If the generator handles the exception thrown into it, the + # exception context will revert to the actual current exception + # context here. In order to make the context manager behave + # like a normal function we set the current exception context + # to what it was during the context manager's __enter__ + sys._set_exception(exc_context) self.gen.throw(value) except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index cf6519598037e9..cd26041e1a6f11 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -306,6 +306,88 @@ def woohoo(): with woohoo(): raise StopIteration + def test_contextmanager_handling_exception_resets_exc_info(self): + # Test that sys.exc_info() is correctly unset after handling the error + # when used within a context manager + + @contextmanager + def ctx(reraise=False): + try: + self.assertIsNone(sys.exception()) + yield + except: + self.assertIsInstance(sys.exception(), ZeroDivisionError) + if reraise: + raise + else: + self.assertIsNone(sys.exception()) + self.assertIsNone(sys.exception()) + + with ctx(): + pass + + with ctx(): + 1/0 + + with self.assertRaises(ZeroDivisionError): + with ctx(reraise=True): + 1/0 + + def test_contextmanager_while_handling(self): + # test that any exceptions currently being handled are preserved + # through the context manager + + @contextmanager + def ctx(reraise=False): + # called while handling an IndexError --> TypeError + self.assertIsInstance(sys.exception(), TypeError) + self.assertIsInstance(sys.exception().__context__, IndexError) + exc_ctx = sys.exception() + try: + # raises a ValueError --> ZeroDivisionError + yield + except: + self.assertIsInstance(sys.exception(), ZeroDivisionError) + self.assertIsInstance(sys.exception().__context__, ValueError) + # original error context is preserved + self.assertIs(sys.exception().__context__.__context__, exc_ctx) + if reraise: + raise + + # inner error handled, context should now be the original context + self.assertIs(sys.exception(), exc_ctx) + + try: + raise IndexError() + except: + try: + raise TypeError() + except: + with ctx(): + try: + raise ValueError() + except: + self.assertIsInstance(sys.exception(), ValueError) + 1/0 + self.assertIsInstance(sys.exception(), TypeError) + self.assertIsInstance(sys.exception(), IndexError) + + try: + raise IndexError() + except: + try: + raise TypeError() + except: + with self.assertRaises(ZeroDivisionError): + with ctx(reraise=True): + try: + raise ValueError() + except: + self.assertIsInstance(sys.exception(), ValueError) + 1/0 + self.assertIsInstance(sys.exception(), TypeError) + self.assertIsInstance(sys.exception(), IndexError) + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): diff --git a/Misc/NEWS.d/next/Library/2023-11-03-01-44-51.gh-issue-111375.qDBcCI.rst b/Misc/NEWS.d/next/Library/2023-11-03-01-44-51.gh-issue-111375.qDBcCI.rst new file mode 100644 index 00000000000000..2519b7e614e9a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-03-01-44-51.gh-issue-111375.qDBcCI.rst @@ -0,0 +1,2 @@ +Fix handling of ``sys.exception()`` within ``@contextlib.contextmanager`` +functions. Patch by Carey Metcalfe. From ea784d7c88543e3f0fe2d4fc5f73fc8d9c09cb4b Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Sun, 24 Dec 2023 14:15:25 -0500 Subject: [PATCH 03/11] Reference GH issue in comment Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Lib/contextlib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index d8d03f84e85b42..5262b2ccaca98a 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -166,10 +166,11 @@ def __exit__(self, typ, value, traceback): value = typ() try: # If the generator handles the exception thrown into it, the - # exception context will revert to the actual current exception + # exception context reverts to the actual current exception # context here. In order to make the context manager behave # like a normal function we set the current exception context # to what it was during the context manager's __enter__ + # (see gh-111676). sys._set_exception(exc_context) self.gen.throw(value) except StopIteration as exc: From bf4ef1e6a149912cab8ca696aaacfd8c2afb5208 Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Sun, 24 Dec 2023 14:27:55 -0500 Subject: [PATCH 04/11] Add more checks for exception context --- Lib/test/test_contextlib.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index cd26041e1a6f11..d687794c371a95 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -364,12 +364,17 @@ def ctx(reraise=False): raise TypeError() except: with ctx(): + self.assertIsInstance(sys.exception(), TypeError) + self.assertIsInstance(sys.exception().__context__, IndexError) try: raise ValueError() except: self.assertIsInstance(sys.exception(), ValueError) + self.assertIsInstance(sys.exception().__context__, TypeError) + self.assertIsInstance(sys.exception().__context__.__context__, IndexError) 1/0 self.assertIsInstance(sys.exception(), TypeError) + self.assertIsInstance(sys.exception().__context__, IndexError) self.assertIsInstance(sys.exception(), IndexError) try: @@ -380,12 +385,17 @@ def ctx(reraise=False): except: with self.assertRaises(ZeroDivisionError): with ctx(reraise=True): + self.assertIsInstance(sys.exception(), TypeError) + self.assertIsInstance(sys.exception().__context__, IndexError) try: raise ValueError() except: self.assertIsInstance(sys.exception(), ValueError) + self.assertIsInstance(sys.exception().__context__, TypeError) + self.assertIsInstance(sys.exception().__context__.__context__, IndexError) 1/0 self.assertIsInstance(sys.exception(), TypeError) + self.assertIsInstance(sys.exception().__context__, IndexError) self.assertIsInstance(sys.exception(), IndexError) def _create_contextmanager_attribs(self): From bbc212b36e0371ac226e5475e487a13efa2cab3e Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Sun, 24 Dec 2023 14:33:50 -0500 Subject: [PATCH 05/11] Fix sys._set_exception() docstring wording --- Python/clinic/sysmodule.c.h | 4 ++-- Python/sysmodule.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index bb9ffaee22fe4e..c4186ebf1155c8 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -186,7 +186,7 @@ PyDoc_STRVAR(sys__set_exception__doc__, "Set the current exception.\n" "\n" "Subsequent calls to sys.exception()/sys.exc_info() will return\n" -"the provided exception until another exception is raised in the\n" +"the provided exception until another exception is caught in the\n" "current thread or the execution stack returns to a frame where\n" "another exception is being handled."); @@ -2011,4 +2011,4 @@ _jit_is_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=320dffdcb61d9e35 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=70d40f64811d9e39 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 48bf219c4a56d0..ff011af7a9da60 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -860,14 +860,14 @@ sys._set_exception Set the current exception. Subsequent calls to sys.exception()/sys.exc_info() will return -the provided exception until another exception is raised in the +the provided exception until another exception is caught in the current thread or the execution stack returns to a frame where another exception is being handled. [clinic start generated code]*/ static PyObject * sys__set_exception_impl(PyObject *module, PyObject *exception) -/*[clinic end generated code: output=39e119ee6b747085 input=46da3b45313a1cfa]*/ +/*[clinic end generated code: output=39e119ee6b747085 input=9c0495269be0821d]*/ { if (!Py_IsNone(exception) && !PyExceptionInstance_Check(exception)){ PyErr_SetString( From 59ef993f3b78f5761e3f4923ba0182ef67e1e021 Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Sun, 24 Dec 2023 15:15:08 -0500 Subject: [PATCH 06/11] Fix test case and add explanation in comment --- Lib/test/test_sys.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index d71858f5b256f7..d0eaa2b02b8c6f 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -263,8 +263,16 @@ def outer(): self.assertIsInstance(sys.exception(), ValueError) self.assertIsInstance(sys.exception().__context__, TypeError) yield next(g) + # at this point the TypeError from the caller has been handled + # by the caller's except block. Even still, it should still be + # referenced as the __context__ of the current exception. self.assertIsInstance(sys.exception(), ValueError) - self.assertIsInstance(sys.exception(), TypeError) + self.assertIsInstance(sys.exception().__context__, TypeError) + # not handling an exception, caller isn't handling one either + self.assertIsNone(sys.exception()) + with self.assertRaises(StopIteration): + next(g) + self.assertIsNone(sys.exception()) g = outer() next(g) From c0e740037dd2fe26e06d90d304cf0736345a37fb Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Mon, 19 May 2025 11:15:58 -0400 Subject: [PATCH 07/11] Add exc_context parameter to {gen/coro}.throw() --- Lib/contextlib.py | 3 +- Objects/genobject.c | 70 +++++++++++++++++++++++++++++++++------------ 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 5262b2ccaca98a..1f6c769cf69557 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -171,8 +171,7 @@ def __exit__(self, typ, value, traceback): # like a normal function we set the current exception context # to what it was during the context manager's __enter__ # (see gh-111676). - sys._set_exception(exc_context) - self.gen.throw(value) + self.gen.throw(value, exc_context=exc_context) except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration diff --git a/Objects/genobject.c b/Objects/genobject.c index 98b2c5004df8ac..5e67538bea4f9a 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -452,8 +452,8 @@ gen_close(PyObject *self, PyObject *args) PyDoc_STRVAR(throw_doc, -"throw(value)\n\ -throw(type[,value[,tb]])\n\ +"throw(value, /, *, exc_context=None)\n\ +throw(type[,value[,tb]], /, *, exc_context=None)\n\ \n\ Raise exception in generator, return next yielded value or raise\n\ StopIteration.\n\ @@ -592,14 +592,22 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, return NULL; } +/* +throw(...) method of builtins.generator instance + throw(value, /, *, exc_context=None) + throw(type[,value[,tb]], /, *, exc_context=None) + Raise exception in generator, return next yielded value or raise + StopIteration. +*/ static PyObject * -gen_throw(PyObject *op, PyObject *const *args, Py_ssize_t nargs) +gen_throw(PyObject *op, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyGenObject *gen = _PyGen_CAST(op); PyObject *typ; PyObject *tb = NULL; PyObject *val = NULL; + PyObject *exc_context = NULL; if (!_PyArg_CheckPositional("throw", nargs, 1, 3)) { return NULL; @@ -612,6 +620,18 @@ gen_throw(PyObject *op, PyObject *const *args, Py_ssize_t nargs) return NULL; } } + + static const char * const _keywords[] = {"", "", "", "exc_context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "throw", + }; + PyObject *argsbuf[4]; + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, 1, argsbuf); + if (!args) { + return NULL; + } + typ = args[0]; if (nargs == 3) { val = args[1]; @@ -620,6 +640,17 @@ gen_throw(PyObject *op, PyObject *const *args, Py_ssize_t nargs) else if (nargs == 2) { val = args[1]; } + + if (kwnames && PyTuple_GET_SIZE(kwnames)){ + exc_context = args[3]; + if (!Py_IsNone(exc_context) && !PyExceptionInstance_Check(exc_context)){ + PyErr_SetString(PyExc_TypeError, "exc_context must be an Exception object or None"); + return NULL; + } + } + // update the generator's current exception context before throwing the + // exception into it + PyErr_SetHandledException(exc_context == NULL ? Py_None : exc_context); return _gen_throw(gen, 1, typ, val, tb); } @@ -841,7 +872,7 @@ PyDoc_STRVAR(sizeof__doc__, static PyMethodDef gen_methods[] = { {"send", gen_send, METH_O, send_doc}, - {"throw", _PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, + {"throw", _PyCFunction_CAST(gen_throw), METH_FASTCALL|METH_KEYWORDS, throw_doc}, {"close", gen_close, METH_NOARGS, close_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, @@ -1187,8 +1218,8 @@ PyDoc_STRVAR(coro_send_doc, return next iterated value or raise StopIteration."); PyDoc_STRVAR(coro_throw_doc, -"throw(value)\n\ -throw(type[,value[,traceback]])\n\ +"throw(value, /, *, exc_context=None)\n\ +throw(type[,value[,tb]], *, exc_context=None)\n\ \n\ Raise exception in coroutine, return next iterated value or raise\n\ StopIteration.\n\ @@ -1201,7 +1232,7 @@ PyDoc_STRVAR(coro_close_doc, static PyMethodDef coro_methods[] = { {"send", gen_send, METH_O, coro_send_doc}, - {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, + {"throw", _PyCFunction_CAST(gen_throw), METH_FASTCALL|METH_KEYWORDS, coro_throw_doc}, {"close", gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, @@ -1291,10 +1322,10 @@ coro_wrapper_send(PyObject *self, PyObject *arg) } static PyObject * -coro_wrapper_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +coro_wrapper_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyCoroWrapper *cw = _PyCoroWrapper_CAST(self); - return gen_throw((PyObject*)cw->cw_coroutine, args, nargs); + return gen_throw((PyObject*)cw->cw_coroutine, args, nargs, kwnames); } static PyObject * @@ -1314,8 +1345,8 @@ coro_wrapper_traverse(PyObject *self, visitproc visit, void *arg) static PyMethodDef coro_wrapper_methods[] = { {"send", coro_wrapper_send, METH_O, coro_send_doc}, - {"throw", _PyCFunction_CAST(coro_wrapper_throw), METH_FASTCALL, - coro_throw_doc}, + {"throw", _PyCFunction_CAST(coro_wrapper_throw), + METH_FASTCALL|METH_KEYWORDS, coro_throw_doc}, {"close", coro_wrapper_close, METH_NOARGS, coro_close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -1827,7 +1858,7 @@ async_gen_asend_iternext(PyObject *ags) static PyObject * -async_gen_asend_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +async_gen_asend_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyAsyncGenASend *o = _PyAsyncGenASend_CAST(self); @@ -1851,7 +1882,7 @@ async_gen_asend_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) o->ags_gen->ag_running_async = 1; } - PyObject *result = gen_throw((PyObject*)o->ags_gen, args, nargs); + PyObject *result = gen_throw((PyObject*)o->ags_gen, args, nargs, kwnames); result = async_gen_unwrap_value(o->ags_gen, result); if (result == NULL) { @@ -1871,7 +1902,7 @@ async_gen_asend_close(PyObject *self, PyObject *args) Py_RETURN_NONE; } - PyObject *result = async_gen_asend_throw(self, &PyExc_GeneratorExit, 1); + PyObject *result = async_gen_asend_throw(self, &PyExc_GeneratorExit, 1, NULL); if (result == NULL) { if (PyErr_ExceptionMatches(PyExc_StopIteration) || PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || @@ -1899,7 +1930,8 @@ async_gen_asend_finalize(PyObject *self) static PyMethodDef async_gen_asend_methods[] = { {"send", async_gen_asend_send, METH_O, send_doc}, - {"throw", _PyCFunction_CAST(async_gen_asend_throw), METH_FASTCALL, throw_doc}, + {"throw", _PyCFunction_CAST(async_gen_asend_throw), + METH_FASTCALL|METH_KEYWORDS, throw_doc}, {"close", async_gen_asend_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ }; @@ -2227,7 +2259,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) static PyObject * -async_gen_athrow_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +async_gen_athrow_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyAsyncGenAThrow *o = _PyAsyncGenAThrow_CAST(self); @@ -2258,7 +2290,7 @@ async_gen_athrow_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) o->agt_gen->ag_running_async = 1; } - PyObject *retval = gen_throw((PyObject*)o->agt_gen, args, nargs); + PyObject *retval = gen_throw((PyObject*)o->agt_gen, args, nargs, kwnames); if (o->agt_args) { retval = async_gen_unwrap_value(o->agt_gen, retval); if (retval == NULL) { @@ -2311,7 +2343,7 @@ async_gen_athrow_close(PyObject *self, PyObject *args) Py_RETURN_NONE; } PyObject *result = async_gen_athrow_throw((PyObject*)agt, - &PyExc_GeneratorExit, 1); + &PyExc_GeneratorExit, 1, NULL); if (result == NULL) { if (PyErr_ExceptionMatches(PyExc_StopIteration) || PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || @@ -2342,7 +2374,7 @@ async_gen_athrow_finalize(PyObject *op) static PyMethodDef async_gen_athrow_methods[] = { {"send", async_gen_athrow_send, METH_O, send_doc}, {"throw", _PyCFunction_CAST(async_gen_athrow_throw), - METH_FASTCALL, throw_doc}, + METH_FASTCALL|METH_KEYWORDS, throw_doc}, {"close", async_gen_athrow_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ }; From 5e8323af3dfd28232b367cd17f6b6af7aabd9679 Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Mon, 19 May 2025 18:08:42 -0400 Subject: [PATCH 08/11] Move exc_context setting into _gen_throw and restore previous exception This avoids accidentally overriding the exception state of the current thread when calling `.throw()` (what happened in the previous commit) --- Objects/genobject.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 5e67538bea4f9a..8ade64837abc2e 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -462,11 +462,13 @@ and may be removed in a future version of Python."); static PyObject * _gen_throw(PyGenObject *gen, int close_on_genexit, - PyObject *typ, PyObject *val, PyObject *tb) + PyObject *typ, PyObject *val, PyObject *tb, + PyObject *exc_context) { PyObject *yf = _PyGen_yf(gen); if (yf) { + assert(exc_context == NULL); _PyInterpreterFrame *frame = &gen->gi_iframe; PyObject *ret; int err; @@ -502,7 +504,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, PyFrameState state = gen->gi_frame_state; gen->gi_frame_state = FRAME_EXECUTING; ret = _gen_throw((PyGenObject *)yf, close_on_genexit, - typ, val, tb); + typ, val, tb, NULL); gen->gi_frame_state = state; tstate->current_frame = prev; frame->previous = NULL; @@ -581,8 +583,17 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, goto failed_throw; } + /* hide the current thread's exception context from the generator */ + PyThreadState *tstate = _PyThreadState_GET(); + assert(tstate != NULL); + PyObject *prev_exc = tstate->exc_info->exc_value; + tstate->exc_info->exc_value = (exc_context == NULL ? Py_None : exc_context); + PyErr_Restore(typ, val, tb); - return gen_send_ex(gen, Py_None, 1, 0); + PyObject* ret = gen_send_ex(gen, Py_None, 1, 0); + + tstate->exc_info->exc_value = prev_exc; + return ret; failed_throw: /* Didn't use our arguments, so restore their original refcounts */ @@ -648,10 +659,7 @@ gen_throw(PyObject *op, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnam return NULL; } } - // update the generator's current exception context before throwing the - // exception into it - PyErr_SetHandledException(exc_context == NULL ? Py_None : exc_context); - return _gen_throw(gen, 1, typ, val, tb); + return _gen_throw(gen, 1, typ, val, tb, exc_context); } @@ -2182,7 +2190,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) retval = _gen_throw((PyGenObject *)gen, 0, /* Do not close generator when PyExc_GeneratorExit is passed */ - PyExc_GeneratorExit, NULL, NULL); + PyExc_GeneratorExit, NULL, NULL, NULL); if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { Py_DECREF(retval); @@ -2201,7 +2209,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) retval = _gen_throw((PyGenObject *)gen, 0, /* Do not close generator when PyExc_GeneratorExit is passed */ - typ, val, tb); + typ, val, tb, NULL); retval = async_gen_unwrap_value(o->agt_gen, retval); } if (retval == NULL) { From a7e2dc9d9a76e11f26906d6bcaf87d3f2f2f6cd6 Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Tue, 20 May 2025 09:22:25 -0400 Subject: [PATCH 09/11] Default .throw()'s exc_context to the current exception --- Objects/genobject.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 8ade64837abc2e..95fb4042784866 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -583,16 +583,20 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, goto failed_throw; } - /* hide the current thread's exception context from the generator */ - PyThreadState *tstate = _PyThreadState_GET(); - assert(tstate != NULL); - PyObject *prev_exc = tstate->exc_info->exc_value; - tstate->exc_info->exc_value = (exc_context == NULL ? Py_None : exc_context); - + /* If an exception context is provided, set it (and restore it after) */ + PyThreadState *tstate; + PyObject *prev_exc = NULL; + if (exc_context != NULL){ + tstate = _PyThreadState_GET(); + assert(tstate != NULL); + prev_exc = tstate->exc_info->exc_value; + tstate->exc_info->exc_value = exc_context; + } PyErr_Restore(typ, val, tb); PyObject* ret = gen_send_ex(gen, Py_None, 1, 0); - - tstate->exc_info->exc_value = prev_exc; + if (prev_exc != NULL){ + tstate->exc_info->exc_value = prev_exc; + } return ret; failed_throw: @@ -659,6 +663,10 @@ gen_throw(PyObject *op, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnam return NULL; } } + /* default to current exception */ + if (exc_context == NULL){ + exc_context = _PyErr_GetTopmostException(_PyThreadState_GET())->exc_value; + } return _gen_throw(gen, 1, typ, val, tb, exc_context); } From 90fdbe0250cfade2f280582af8db56a54fc13ac7 Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Tue, 20 May 2025 09:35:33 -0400 Subject: [PATCH 10/11] Remove overactive assert --- Objects/genobject.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 95fb4042784866..eb94b9ae78556a 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -468,7 +468,6 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, PyObject *yf = _PyGen_yf(gen); if (yf) { - assert(exc_context == NULL); _PyInterpreterFrame *frame = &gen->gi_iframe; PyObject *ret; int err; @@ -504,7 +503,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, PyFrameState state = gen->gi_frame_state; gen->gi_frame_state = FRAME_EXECUTING; ret = _gen_throw((PyGenObject *)yf, close_on_genexit, - typ, val, tb, NULL); + typ, val, tb, exc_context); gen->gi_frame_state = state; tstate->current_frame = prev; frame->previous = NULL; From 5da75d09fc51b8637aec8aa1ac01f3ff18956de3 Mon Sep 17 00:00:00 2001 From: Carey Metcalfe Date: Tue, 10 Jun 2025 09:17:10 -0400 Subject: [PATCH 11/11] Rename test case Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Lib/test/test_contextlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index d687794c371a95..59b3b4d96f1442 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -333,7 +333,7 @@ def ctx(reraise=False): with ctx(reraise=True): 1/0 - def test_contextmanager_while_handling(self): + def test_contextmanager_preserves_handled_exception(self): # test that any exceptions currently being handled are preserved # through the context manager