From 7b17af2e577ec43c1b4e830bcf7df005e18e8b2f Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Wed, 25 Sep 2024 17:02:53 -0700 Subject: [PATCH 1/4] gh-121377: Fix closure with intervening comprehension --- Lib/test/test_listcomps.py | 12 +++++++++++ Python/symtable.c | 43 +++++++++++++++++++++++++++++++------- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 45644d6c0927827..7036a04481d4f74 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -750,6 +750,18 @@ def iter_raises(): self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], expected) + def test_freevar_through_scope_containing_comprehension(self): + code = """ + x = 1 + def f(): + [x for x in [1]] + def g(): + return x + return g() + y = f() + """ + self._check_in_scopes(code, {"x": 1, "y": 1}, scopes=["module", "function"]) + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): diff --git a/Python/symtable.c b/Python/symtable.c index 8bc9db6d7d68116..ffeb0d5f18e4d2e 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -793,7 +793,7 @@ is_free_in_any_child(PySTEntryObject *entry, PyObject *key) static int inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, PyObject *scopes, PyObject *comp_free, - PyObject *inlined_cells) + PyObject *inlined_cells, PyObject *inlined_locals) { PyObject *k, *v; Py_ssize_t pos = 0; @@ -843,6 +843,11 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, return 0; } SET_SCOPE(scopes, k, scope); + if (scope == LOCAL) { + if (PySet_Add(inlined_locals, k) < 0) { + return 0; + } + } } else { long flags = PyLong_AsLong(existing); @@ -882,15 +887,20 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, */ static int -analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) +analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells, PyObject *inlined_locals) { - PyObject *name, *v, *v_cell; + PyObject *name, *v, *v_cell, *v_free; int success = 0; Py_ssize_t pos = 0; v_cell = PyLong_FromLong(CELL); if (!v_cell) return 0; + v_free = PyLong_FromLong(FREE); + if (!v_free) { + Py_DECREF(v_cell); + return 0; + } while (PyDict_Next(scopes, &pos, &name, &v)) { long scope = PyLong_AsLong(v); if (scope == -1 && PyErr_Occurred()) { @@ -911,6 +921,19 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) continue; } } + /* If the name is defined by an inlined comprehension, it must be + * preserved as free in the outer scope. */ + contains = PySet_Contains(inlined_locals, name); + if (contains < 0) { + goto error; + } + if (contains) { + if (PyDict_SetItem(scopes, name, v_free) < 0) { + goto error; + } + continue; + } + /* Replace LOCAL with CELL for this name, and remove from free. It is safe to replace the value of name in the dict, because it will not cause a resize. @@ -923,6 +946,7 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) success = 1; error: Py_DECREF(v_cell); + Py_DECREF(v_free); return success; } @@ -1099,7 +1123,8 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PySTEntryObject *class_entry) { PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL; - PyObject *newglobal = NULL, *newfree = NULL, *inlined_cells = NULL; + PyObject *newglobal = NULL, *newfree = NULL; + PyObject *inlined_cells = NULL, *inlined_locals = NULL; PyObject *temp; int success = 0; Py_ssize_t i, pos = 0; @@ -1134,6 +1159,9 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, inlined_cells = PySet_New(NULL); if (!inlined_cells) goto error; + inlined_locals = PySet_New(NULL); + if (!inlined_locals) + goto error; /* Class namespace has no effect on names visible in nested functions, so populate the global and bound @@ -1220,7 +1248,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, // we inline all non-generator-expression comprehensions, // except those in annotation scopes that are nested in classes - int inline_comp = + int inline_comp = true && entry->ste_comprehension && !entry->ste_generator && !ste->ste_can_see_class_scope; @@ -1231,7 +1259,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, goto error; } if (inline_comp) { - if (!inline_comprehension(ste, entry, scopes, child_free, inlined_cells)) { + if (!inline_comprehension(ste, entry, scopes, child_free, inlined_cells, inlined_locals)) { Py_DECREF(child_free); goto error; } @@ -1259,7 +1287,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } /* Check if any local variables must be converted to cell variables */ - if (_PyST_IsFunctionLike(ste) && !analyze_cells(scopes, newfree, inlined_cells)) + if (_PyST_IsFunctionLike(ste) && !analyze_cells(scopes, newfree, inlined_cells, inlined_locals)) goto error; else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) goto error; @@ -1280,6 +1308,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, Py_XDECREF(newglobal); Py_XDECREF(newfree); Py_XDECREF(inlined_cells); + Py_XDECREF(inlined_locals); if (!success) assert(PyErr_Occurred()); return success; From 5acc6baf124d0cc10a5090c0310db72bc566db8d Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 27 Sep 2024 13:23:46 -0700 Subject: [PATCH 2/4] also handle case where cell is inlined --- Lib/test/test_listcomps.py | 12 ++++++++++++ Python/symtable.c | 28 +++++++++++++++------------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 7036a04481d4f74..1a88e8cc1a5cc94 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -762,6 +762,18 @@ def g(): """ self._check_in_scopes(code, {"x": 1, "y": 1}, scopes=["module", "function"]) + def test_freevar_through_scope_containing_comprehension_with_cell(self): + code = """ + x = 1 + def f(): + [lambda: x for x in [1]] + def g(): + return x + return g() + y = f() + """ + self._check_in_scopes(code, {"x": 1, "y": 1}, scopes=["function"]) + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): diff --git a/Python/symtable.c b/Python/symtable.c index ffeb0d5f18e4d2e..428b267b3387162 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -843,7 +843,7 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, return 0; } SET_SCOPE(scopes, k, scope); - if (scope == LOCAL) { + if (scope == LOCAL || scope == CELL) { if (PySet_Add(inlined_locals, k) < 0) { return 0; } @@ -906,7 +906,8 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells, PyObjec if (scope == -1 && PyErr_Occurred()) { goto error; } - if (scope != LOCAL) + + if (scope != LOCAL && scope != CELL) continue; int contains = PySet_Contains(free, name); if (contains < 0) { @@ -920,18 +921,19 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells, PyObjec if (!contains) { continue; } - } - /* If the name is defined by an inlined comprehension, it must be - * preserved as free in the outer scope. */ - contains = PySet_Contains(inlined_locals, name); - if (contains < 0) { - goto error; - } - if (contains) { - if (PyDict_SetItem(scopes, name, v_free) < 0) { + } else { + /* If a free name is defined only by an inlined comprehension, it must be + * preserved as free in the outer scope. */ + contains = PySet_Contains(inlined_locals, name); + if (contains < 0) { goto error; } - continue; + if (contains) { + if (PyDict_SetItem(scopes, name, v_free) < 0) { + goto error; + } + continue; + } } /* Replace LOCAL with CELL for this name, and remove @@ -1248,7 +1250,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, // we inline all non-generator-expression comprehensions, // except those in annotation scopes that are nested in classes - int inline_comp = true && + int inline_comp = entry->ste_comprehension && !entry->ste_generator && !ste->ste_can_see_class_scope; From 34b85321aa2feab009ca773c964e92e1efdf6d06 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 20:28:31 +0000 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-09-27-20-28-27.gh-issue-121377.Svz8LR.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-20-28-27.gh-issue-121377.Svz8LR.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-20-28-27.gh-issue-121377.Svz8LR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-20-28-27.gh-issue-121377.Svz8LR.rst new file mode 100644 index 000000000000000..24e59bc48b2f19f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-20-28-27.gh-issue-121377.Svz8LR.rst @@ -0,0 +1 @@ +Fix multi-level non-local reference with intervening inlined comprehension binding the same name. From cbad048abfb3c9dafaa34af5f29bb2317e43cb0a Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 27 Sep 2024 16:26:43 -0700 Subject: [PATCH 4/4] update tests --- Lib/test/test_listcomps.py | 18 ++++++++++-------- Python/flowgraph.c | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 1a88e8cc1a5cc94..1e1e9e32dc2f761 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -754,25 +754,27 @@ def test_freevar_through_scope_containing_comprehension(self): code = """ x = 1 def f(): - [x for x in [1]] + for z in [x for x in [2]]: + pass def g(): return x - return g() - y = f() + return g(), z + y, z = f() """ - self._check_in_scopes(code, {"x": 1, "y": 1}, scopes=["module", "function"]) + self._check_in_scopes(code, {"x": 1, "y": 1, "z": 2}, scopes=["module", "function"]) def test_freevar_through_scope_containing_comprehension_with_cell(self): code = """ x = 1 def f(): - [lambda: x for x in [1]] + for l in [lambda: x for x in [2]]: + z = l() def g(): return x - return g() - y = f() + return g(), z + y, z = f() """ - self._check_in_scopes(code, {"x": 1, "y": 1}, scopes=["function"]) + self._check_in_scopes(code, {"x": 1, "y": 1, "z": 2}, scopes=["function"]) __test__ = {'doctests' : doctests} diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 69d7e0a872aa489..eb037b54aacc0d0 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2700,7 +2700,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl } cfg_instr make_cell = { .i_opcode = MAKE_CELL, - // This will get fixed in offset_derefs(). + // This will get fixed in fix_cell_offsets(). .i_oparg = oldindex, .i_loc = NO_LOCATION, .i_target = NULL,