From 720a58430c2830ae067e8ddc4382495c82f9253b Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 19 Oct 2023 18:53:45 -0700 Subject: [PATCH 01/10] Make closure work on pdb --- Lib/pdb.py | 62 ++++++++++++++++++++++++++++++++++++++++++-- Lib/test/test_pdb.py | 57 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 1e4d0a20515fa3..f0e2799415f11d 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -76,10 +76,12 @@ import dis import code import glob +import types import codeop import pprint import signal import inspect +import textwrap import tokenize import functools import traceback @@ -545,15 +547,68 @@ def _disable_tab_completion(self): else: yield + def _exec_in_closure(self, source, globals, locals): + """ Run source code in closure so code object created within source + can find variables in locals correctly + """ + + # If the source is an expression, we need to print its value + try: + compile(source, "", "eval") + source = "__pdb_eval_result__ = " + source + locals["__pdb_eval_result__"] = None + except SyntaxError: + pass + + # Add write-back to update the locals + locals["__pdb_write_back__"] = {} + source += """\nfor key, val in locals().items():\n __pdb_write_back__[key] = val""" + + try: + local_vars = list(locals.keys()) + + # Build a closure source code with freevars from locals like: + # def outer(): + # var = None + # def __pdb_scope(): # This is the code object we want to execute + # nonlocal var + # + source_with_closure = ("def outer():\n" + + "\n".join(f" {var} = None" for var in local_vars) + "\n" + + " def __pdb_scope():\n" + + "\n".join(f" nonlocal {var}" for var in local_vars) + "\n" + + textwrap.indent(source, " ") + ) + + # Compile the instrumented source code, and get the code object of __pdb_scope() + # This simulates executing the original source code with locals as cellvars + # co_consts[0] -> outer; + # outer.co_consts[1] -> __pdb_scope because outer.co_consts[0] is None + code = compile(source_with_closure, "", "exec").co_consts[0].co_consts[1] + cells = tuple(types.CellType(locals.get(var)) for var in code.co_freevars) + exec(code, globals, locals, closure=cells) + + # Write all local variables back to locals + for var, value in locals["__pdb_write_back__"].items(): + locals[var] = value + locals.pop("__pdb_write_back__", None) + + if (ret := locals.get("__pdb_eval_result__")) is not None: + locals.pop("__pdb_eval_result__", None) + print(repr(ret)) + finally: + locals.pop("__pdb_eval_result__", None) + locals.pop("__pdb_write_back__", None) + def default(self, line): if line[:1] == '!': line = line[1:].strip() locals = self.curframe_locals globals = self.curframe.f_globals try: + buffer = line if (code := codeop.compile_command(line + '\n', '', 'single')) is None: # Multi-line mode with self._disable_tab_completion(): - buffer = line continue_prompt = "... " while (code := codeop.compile_command(buffer, '', 'single')) is None: if self.use_rawinput: @@ -582,7 +637,10 @@ def default(self, line): sys.stdin = self.stdin sys.stdout = self.stdout sys.displayhook = self.displayhook - exec(code, globals, locals) + try: + self._exec_in_closure(buffer, globals, locals) + except Exception: + exec(code, globals, locals) finally: sys.stdout = save_stdout sys.stdin = save_stdin diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 4701fa0cc9656a..6cdf47c149586b 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2028,8 +2028,63 @@ def test_pdb_multiline_statement(): (Pdb) c """ +def test_pdb_closure(): + """Test for all expressions/statements that involve closure + + >>> k = 0 + >>> g = 1 + >>> def test_function(): + ... x = 2 + ... g = 3 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... pass + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'k', + ... 'g', + ... 'y = y', + ... 'global g; g', + ... '(lambda: x)()', + ... '(lambda: g)()', + ... 'lst = [n for n in range(10) if (n % x) == 0]', + ... 'lst', + ... 'sum(n for n in lst if n > x)', + ... 'def f():', + ... ' return x', + ... '', + ... 'f()', + ... 'c' + ... ]): + ... test_function() + > (5)test_function() + -> pass + (Pdb) k + 0 + (Pdb) g + 3 + (Pdb) y = y + *** NameError: name 'y' is not defined + (Pdb) global g; g + 1 + (Pdb) (lambda: x)() + 2 + (Pdb) (lambda: g)() + 3 + (Pdb) lst = [n for n in range(10) if (n % x) == 0] + (Pdb) lst + [0, 2, 4, 6, 8] + (Pdb) sum(n for n in lst if n > x) + 18 + (Pdb) def f(): + ... return x + ... + (Pdb) f() + 2 + (Pdb) c + """ + def test_pdb_show_attribute_and_item(): - """Test for multiline statement + """Test for expressions with command prefix >>> def test_function(): ... n = lambda x: x From 8232457a7c6fcccb2da482d65e99f7697877ac93 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 19 Oct 2023 20:32:45 -0700 Subject: [PATCH 02/10] Always update local variables --- Lib/pdb.py | 5 ++++- Lib/test/test_pdb.py | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index f0e2799415f11d..e8293dbc891fa4 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -562,7 +562,10 @@ def _exec_in_closure(self, source, globals, locals): # Add write-back to update the locals locals["__pdb_write_back__"] = {} - source += """\nfor key, val in locals().items():\n __pdb_write_back__[key] = val""" + source = ("try:\n" + + textwrap.indent(source, " ") + "\n" + + "finally:\n" + + " for key, val in locals().items():\n __pdb_write_back__[key] = val") try: local_vars = list(locals.keys()) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 6cdf47c149586b..89d4a28419d474 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2049,6 +2049,8 @@ def test_pdb_closure(): ... 'lst = [n for n in range(10) if (n % x) == 0]', ... 'lst', ... 'sum(n for n in lst if n > x)', + ... 'x = 1; raise Exception()', + ... 'x', ... 'def f():', ... ' return x', ... '', @@ -2075,11 +2077,15 @@ def test_pdb_closure(): [0, 2, 4, 6, 8] (Pdb) sum(n for n in lst if n > x) 18 + (Pdb) x = 1; raise Exception() + *** Exception + (Pdb) x + 1 (Pdb) def f(): ... return x ... (Pdb) f() - 2 + 1 (Pdb) c """ From 8df6d2b576b772495c77bcf765cdb404fb5bf0e6 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 20 Oct 2023 03:50:19 +0000 Subject: [PATCH 03/10] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst diff --git a/Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst b/Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst new file mode 100644 index 00000000000000..73577574ae0bf1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst @@ -0,0 +1 @@ +Make closure work on :mod:`pdb` From fe5a6f6ae6c4f0ba91b409ecdcbaf78d285f70e5 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 4 May 2024 17:55:02 -0700 Subject: [PATCH 04/10] Update the code for pep 667 --- Lib/pdb.py | 109 ++++++++++++++++++++++++++---------------- Objects/frameobject.c | 1 - 2 files changed, 68 insertions(+), 42 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 3e2e028e96275b..f49d7d0472c360 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -627,58 +627,87 @@ def _disable_command_completion(self): def _exec_in_closure(self, source, globals, locals): """ Run source code in closure so code object created within source can find variables in locals correctly + + returns True if the source is executed, False otherwise """ + # Determine if the source should be executed in closure. Only when the + # source compiled to multiple code objects, we should use this feature. + # Otherwise, we can just raise an exception and normal exec will be used. + + code = compile(source, "", "exec") + for const in code.co_consts: + if isinstance(const, CodeType): + break + else: + return False + + # locals could be a proxy which does not support pop + # copy it first to avoid modifying the original locals + locals_copy = dict(locals) + + locals_copy["__pdb_eval__"] = { + "result": None, + "write_back": {} + } + # If the source is an expression, we need to print its value try: compile(source, "", "eval") - source = "__pdb_eval_result__ = " + source - locals["__pdb_eval_result__"] = None except SyntaxError: pass + else: + source = "__pdb_eval__['result'] = " + source # Add write-back to update the locals - locals["__pdb_write_back__"] = {} source = ("try:\n" + textwrap.indent(source, " ") + "\n" + "finally:\n" + - " for key, val in locals().items():\n __pdb_write_back__[key] = val") + " __pdb_eval__['write_back'].update(locals())") + + local_vars = list(locals_copy.keys()) + + # Build a closure source code with freevars from locals like: + # def outer(): + # var = None + # def __pdb_scope(): # This is the code object we want to execute + # nonlocal var + # + # return __pdb_scope.__code__ + source_with_closure = ("def __pdb_outer():\n" + + "\n".join(f" {var} = None" for var in local_vars) + "\n" + + " def __pdb_scope():\n" + + "\n".join(f" nonlocal {var}" for var in local_vars) + "\n" + + textwrap.indent(source, " ") + "\n" + + " return __pdb_scope.__code__" + ) + + # Get the code object of __pdb_scope() + # The exec fills locals_copy with the __pdb_outer() function and we can call + # that to get the code object of __pdb_scope() + code = exec(source_with_closure, {}, locals_copy) + code = locals_copy.pop("__pdb_outer")() + + cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars) try: - local_vars = list(locals.keys()) - - # Build a closure source code with freevars from locals like: - # def outer(): - # var = None - # def __pdb_scope(): # This is the code object we want to execute - # nonlocal var - # - source_with_closure = ("def outer():\n" + - "\n".join(f" {var} = None" for var in local_vars) + "\n" + - " def __pdb_scope():\n" + - "\n".join(f" nonlocal {var}" for var in local_vars) + "\n" + - textwrap.indent(source, " ") - ) - - # Compile the instrumented source code, and get the code object of __pdb_scope() - # This simulates executing the original source code with locals as cellvars - # co_consts[0] -> outer; - # outer.co_consts[1] -> __pdb_scope because outer.co_consts[0] is None - code = compile(source_with_closure, "", "exec").co_consts[0].co_consts[1] - cells = tuple(types.CellType(locals.get(var)) for var in code.co_freevars) - exec(code, globals, locals, closure=cells) - - # Write all local variables back to locals - for var, value in locals["__pdb_write_back__"].items(): - locals[var] = value - locals.pop("__pdb_write_back__", None) - - if (ret := locals.get("__pdb_eval_result__")) is not None: - locals.pop("__pdb_eval_result__", None) - print(repr(ret)) - finally: - locals.pop("__pdb_eval_result__", None) - locals.pop("__pdb_write_back__", None) + exec(code, globals, locals_copy, closure=cells) + except Exception: + return False + + # get the data we need from the statement + pdb_eval = locals_copy.pop("__pdb_eval__") + + # __pdb_eval__ should not be updated back to locals + pdb_eval["write_back"].pop("__pdb_eval__") + + # Write all local variables back to locals + locals.update(pdb_eval["write_back"]) + eval_result = pdb_eval["result"] + if eval_result is not None: + print(repr(eval_result)) + + return True def default(self, line): if line[:1] == '!': line = line[1:].strip() @@ -718,9 +747,7 @@ def default(self, line): sys.stdin = self.stdin sys.stdout = self.stdout sys.displayhook = self.displayhook - try: - self._exec_in_closure(buffer, globals, locals) - except Exception: + if not self._exec_in_closure(buffer, globals, locals): exec(code, globals, locals) finally: sys.stdout = save_stdout diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 8030ecb6853674..1cb00e318d9163 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -171,7 +171,6 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) } else if (value != oldvalue) { Py_XSETREF(fast[i], Py_NewRef(value)); } - Py_XDECREF(value); return 0; } } From cc21873482db49b36afb223857dbaf54fc72b574 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 4 May 2024 17:58:39 -0700 Subject: [PATCH 05/10] Remove blank line --- Lib/pdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index f49d7d0472c360..e7a1fea38297a5 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -650,7 +650,7 @@ def _exec_in_closure(self, source, globals, locals): "result": None, "write_back": {} } - + # If the source is an expression, we need to print its value try: compile(source, "", "eval") From cd46a05ecbe136657b5cdfb41384b4d3739e72c0 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 4 May 2024 20:56:06 -0700 Subject: [PATCH 06/10] Update 2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst --- .../Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst b/Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst index 73577574ae0bf1..aaefbb9e2a6d3d 100644 --- a/Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst +++ b/Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst @@ -1 +1,3 @@ -Make closure work on :mod:`pdb` +Enabled arbitrary statements and evaluations in :mod:`pdb` shell to access the +local variables of the current frame, which made it possible for multi-scope +code like generators or nested function to work. From aa9123869cebe5b564c3cb599089713a5cd123cb Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 5 May 2024 10:12:00 -0700 Subject: [PATCH 07/10] Fix test for breakpoint change --- Lib/test/test_pdb.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index c3345ddcce97ba..db04bece4105ac 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2233,7 +2233,6 @@ def test_pdb_closure(): ... x = 2 ... g = 3 ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'k', @@ -2254,8 +2253,8 @@ def test_pdb_closure(): ... 'c' ... ]): ... test_function() - > (5)test_function() - -> pass + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) k 0 (Pdb) g From 7650aa95828b12dc285c7dfd7768824f43d81bfc Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 5 May 2024 21:12:13 -0700 Subject: [PATCH 08/10] Apply suggestions from code review Co-authored-by: Brandt Bucher --- Lib/pdb.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 415b110d164e82..b8afa7d27560c9 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -638,10 +638,7 @@ def _exec_in_closure(self, source, globals, locals): # Otherwise, we can just raise an exception and normal exec will be used. code = compile(source, "", "exec") - for const in code.co_consts: - if isinstance(const, CodeType): - break - else: + if any(isinstance(const, CodeType) for const in code.co_consts): return False # locals could be a proxy which does not support pop @@ -665,7 +662,7 @@ def _exec_in_closure(self, source, globals, locals): source = ("try:\n" + textwrap.indent(source, " ") + "\n" + "finally:\n" + - " __pdb_eval__['write_back'].update(locals())") + " __pdb_eval__['write_back'] = locals()") local_vars = list(locals_copy.keys()) @@ -687,7 +684,7 @@ def _exec_in_closure(self, source, globals, locals): # Get the code object of __pdb_scope() # The exec fills locals_copy with the __pdb_outer() function and we can call # that to get the code object of __pdb_scope() - code = exec(source_with_closure, {}, locals_copy) + exec(source_with_closure, {}, locals_copy) code = locals_copy.pop("__pdb_outer")() cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars) @@ -698,7 +695,7 @@ def _exec_in_closure(self, source, globals, locals): return False # get the data we need from the statement - pdb_eval = locals_copy.pop("__pdb_eval__") + pdb_eval = locals_copy["__pdb_eval__] # __pdb_eval__ should not be updated back to locals pdb_eval["write_back"].pop("__pdb_eval__") From 3c44f8814767f0713c0987d15664dba9a8d8db1d Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 5 May 2024 21:22:36 -0700 Subject: [PATCH 09/10] Fix some typos in review and add exception check --- Lib/pdb.py | 15 ++++++++------- Lib/test/test_pdb.py | 3 +++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index b8afa7d27560c9..8cb8fc1ab5f637 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -638,7 +638,7 @@ def _exec_in_closure(self, source, globals, locals): # Otherwise, we can just raise an exception and normal exec will be used. code = compile(source, "", "exec") - if any(isinstance(const, CodeType) for const in code.co_consts): + if not any(isinstance(const, CodeType) for const in code.co_consts): return False # locals could be a proxy which does not support pop @@ -664,8 +664,6 @@ def _exec_in_closure(self, source, globals, locals): "finally:\n" + " __pdb_eval__['write_back'] = locals()") - local_vars = list(locals_copy.keys()) - # Build a closure source code with freevars from locals like: # def outer(): # var = None @@ -674,9 +672,9 @@ def _exec_in_closure(self, source, globals, locals): # # return __pdb_scope.__code__ source_with_closure = ("def __pdb_outer():\n" + - "\n".join(f" {var} = None" for var in local_vars) + "\n" + + "\n".join(f" {var} = None" for var in locals_copy) + "\n" + " def __pdb_scope():\n" + - "\n".join(f" nonlocal {var}" for var in local_vars) + "\n" + + "\n".join(f" nonlocal {var}" for var in locals_copy) + "\n" + textwrap.indent(source, " ") + "\n" + " return __pdb_scope.__code__" ) @@ -684,7 +682,10 @@ def _exec_in_closure(self, source, globals, locals): # Get the code object of __pdb_scope() # The exec fills locals_copy with the __pdb_outer() function and we can call # that to get the code object of __pdb_scope() - exec(source_with_closure, {}, locals_copy) + try: + exec(source_with_closure, {}, locals_copy) + except Exception: + return False code = locals_copy.pop("__pdb_outer")() cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars) @@ -695,7 +696,7 @@ def _exec_in_closure(self, source, globals, locals): return False # get the data we need from the statement - pdb_eval = locals_copy["__pdb_eval__] + pdb_eval = locals_copy["__pdb_eval__"] # __pdb_eval__ should not be updated back to locals pdb_eval["write_back"].pop("__pdb_eval__") diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index db04bece4105ac..f47466410082ef 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2239,6 +2239,7 @@ def test_pdb_closure(): ... 'g', ... 'y = y', ... 'global g; g', + ... 'global g; (lambda: g)()', ... '(lambda: x)()', ... '(lambda: g)()', ... 'lst = [n for n in range(10) if (n % x) == 0]', @@ -2263,6 +2264,8 @@ def test_pdb_closure(): *** NameError: name 'y' is not defined (Pdb) global g; g 1 + (Pdb) global g; (lambda: g)() + 1 (Pdb) (lambda: x)() 2 (Pdb) (lambda: g)() From 18e7a16e759a7dff855fa55d4297fbebd5fdc4e4 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 6 May 2024 10:24:00 -0700 Subject: [PATCH 10/10] Apply suggestions from code review Co-authored-by: Brandt Bucher --- Lib/pdb.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 8cb8fc1ab5f637..e507a9bb896611 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -665,7 +665,7 @@ def _exec_in_closure(self, source, globals, locals): " __pdb_eval__['write_back'] = locals()") # Build a closure source code with freevars from locals like: - # def outer(): + # def __pdb_outer(): # var = None # def __pdb_scope(): # This is the code object we want to execute # nonlocal var @@ -682,11 +682,12 @@ def _exec_in_closure(self, source, globals, locals): # Get the code object of __pdb_scope() # The exec fills locals_copy with the __pdb_outer() function and we can call # that to get the code object of __pdb_scope() + ns = {} try: - exec(source_with_closure, {}, locals_copy) + exec(source_with_closure, {}, ns) except Exception: return False - code = locals_copy.pop("__pdb_outer")() + code = ns["__pdb_outer"]() cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars)