python · carljm · May 16, 2023 · May 16, 2023 · May 17, 2023
diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
@@ -158,9 +158,10 @@ New Features
 PEP 709: Comprehension inlining
 -------------------------------
 
-Dictionary, list, and set comprehensions are now inlined, rather than creating a
-new single-use function object for each execution of the comprehension. This
-speeds up execution of a comprehension by up to 2x.
+Dictionary, list, and set comprehensions occurring inside functions are now
+inlined, rather than creating a new single-use function object for each
+execution of the comprehension. This speeds up execution of a comprehension by
+up to 2x.
 
 Comprehension iteration variables remain isolated; they don't overwrite a
 variable of the same name in the outer scope, nor are they visible after the

diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
@@ -128,7 +128,6 @@ struct callable_cache {
 // Note that these all fit within a byte, as do combinations.
 // Later, we will use the smaller numbers to differentiate the different
 // kinds of locals (e.g. pos-only arg, varkwargs, local-only).
-#define CO_FAST_HIDDEN  0x10
 #define CO_FAST_LOCAL   0x20
 #define CO_FAST_CELL    0x40
 #define CO_FAST_FREE    0x80

diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h
@@ -70,9 +70,6 @@ typedef struct {
     PyObject *u_varnames;  /* local variables */
     PyObject *u_cellvars;  /* cell variables */
     PyObject *u_freevars;  /* free variables */
-    PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only
-                               temporarily within an inlined comprehension. When
-                               value is True, treat as fast-local. */
 
     Py_ssize_t u_argcount;        /* number of arguments for block */
     Py_ssize_t u_posonlyargcount;        /* number of positional only arguments for block */

diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
@@ -1352,11 +1352,14 @@ def test_multiline_list_comprehension(self):
                                and x != 50)]
             """)
         compiled_code, _ = self.check_positions_against_ast(snippet)
+        compiled_code = compiled_code.co_consts[0]
         self.assertIsInstance(compiled_code, types.CodeType)
         self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
             line=1, end_line=2, column=1, end_column=8, occurrence=1)
         self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
             line=1, end_line=2, column=1, end_column=8, occurrence=1)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+            line=1, end_line=6, column=0, end_column=32, occurrence=1)
 
     def test_multiline_async_list_comprehension(self):
         snippet = textwrap.dedent("""\
@@ -1390,11 +1393,14 @@ def test_multiline_set_comprehension(self):
                                and x != 50)}
             """)
         compiled_code, _ = self.check_positions_against_ast(snippet)
+        compiled_code = compiled_code.co_consts[0]
         self.assertIsInstance(compiled_code, types.CodeType)
         self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD',
             line=1, end_line=2, column=1, end_column=8, occurrence=1)
         self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
             line=1, end_line=2, column=1, end_column=8, occurrence=1)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+            line=1, end_line=6, column=0, end_column=32, occurrence=1)
 
     def test_multiline_async_set_comprehension(self):
         snippet = textwrap.dedent("""\
@@ -1428,11 +1434,14 @@ def test_multiline_dict_comprehension(self):
                                and x != 50)}
             """)
         compiled_code, _ = self.check_positions_against_ast(snippet)
+        compiled_code = compiled_code.co_consts[0]
         self.assertIsInstance(compiled_code, types.CodeType)
         self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD',
             line=1, end_line=2, column=1, end_column=7, occurrence=1)
         self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
             line=1, end_line=2, column=1, end_column=7, occurrence=1)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+            line=1, end_line=6, column=0, end_column=32, occurrence=1)
 
     def test_multiline_async_dict_comprehension(self):
         snippet = textwrap.dedent("""\
@@ -1702,6 +1711,9 @@ def test_column_offset_deduplication(self):
         for source in [
             "lambda: a",
             "(a for b in c)",
+            "[a for b in c]",
+            "{a for b in c}",
+            "{a: b for c in d}",
         ]:
             with self.subTest(source):
                 code = compile(f"{source}, {source}", "<test>", "eval")

diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py
@@ -16,7 +16,7 @@ def complete_metadata(self, metadata, filename="myfile.py"):
             metadata.setdefault(key, key)
         for key in ['consts']:
             metadata.setdefault(key, [])
-        for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']:
+        for key in ['names', 'varnames', 'cellvars', 'freevars']:
             metadata.setdefault(key, {})
         for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']:
             metadata.setdefault(key, 0)
@@ -33,9 +33,6 @@ def assemble_test(self, insts, metadata, expected):
 
         expected_metadata = {}
         for key, value in metadata.items():
-            if key == "fasthidden":
-                # not exposed on code object
-                continue
             if isinstance(value, list):
                 expected_metadata[key] = tuple(value)
             elif isinstance(value, dict):

diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py
@@ -200,7 +200,7 @@ def f():
             y = [g for x in [1]]
         """
         outputs = {"y": [2]}
-        self._check_in_scopes(code, outputs)
+        self._check_in_scopes(code, outputs, scopes=["module", "function"])
 
     def test_inner_cell_shadows_outer_redefined(self):
         code = """
@@ -328,7 +328,7 @@ def test_nested_2(self):
             y = [x for [x ** x for x in range(x)][x - 1] in l]
         """
         outputs = {"y": [3, 3, 3]}
-        self._check_in_scopes(code, outputs)
+        self._check_in_scopes(code, outputs, scopes=["module", "function"])
 
     def test_nested_3(self):
         code = """
@@ -379,6 +379,42 @@ def f():
         with self.assertRaises(UnboundLocalError):
             f()
 
+    def test_name_error_in_class_scope(self):
+        code = """
+            y = 1
+            [x + y for x in range(2)]
+        """
+        self._check_in_scopes(code, raises=NameError, scopes=["class"])
+
+    def test_global_in_class_scope(self):
+        code = """
+            y = 2
+            vals = [(x, y) for x in range(2)]
+        """
+        outputs = {"vals": [(0, 1), (1, 1)]}
+        self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["class"])
+
+    def test_in_class_scope_inside_function_1(self):
+        code = """
+            class C:
+                y = 2
+                vals = [(x, y) for x in range(2)]
+            vals = C.vals
+        """
+        outputs = {"vals": [(0, 1), (1, 1)]}
+        self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["function"])
+
+    def test_in_class_scope_inside_function_2(self):
+        code = """
+            y = 1
+            class C:
+                y = 2
+                vals = [(x, y) for x in range(2)]
+            vals = C.vals
+        """
+        outputs = {"vals": [(0, 1), (1, 1)]}
+        self._check_in_scopes(code, outputs, scopes=["function"])
+
 
 __test__ = {'doctests' : doctests}
 

diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst
@@ -1,2 +1,2 @@
-:pep:`709`: inline list, dict and set comprehensions to improve performance
-and reduce bytecode size.
+:pep:`709`: inline list, dict and set comprehensions in functions to improve
+performance and reduce bytecode size.
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
@@ -669,14 +669,12 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module,
     umd.u_varnames = PyDict_GetItemString(metadata, "varnames");
     umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars");
     umd.u_freevars = PyDict_GetItemString(metadata, "freevars");
-    umd.u_fasthidden = PyDict_GetItemString(metadata, "fasthidden");
 
     assert(PyDict_Check(umd.u_consts));
     assert(PyDict_Check(umd.u_names));
     assert(PyDict_Check(umd.u_varnames));
     assert(PyDict_Check(umd.u_cellvars));
     assert(PyDict_Check(umd.u_freevars));
-    assert(PyDict_Check(umd.u_fasthidden));
 
     umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount");
     umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount");

@@ -1222,9 +1222,6 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame)
 
         PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
         _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i);
-        if (kind & CO_FAST_HIDDEN) {
-            continue;
-        }
         if (value == NULL) {
             if (PyObject_DelItem(locals, name) != 0) {
                 if (PyErr_ExceptionMatches(PyExc_KeyError)) {

@@ -457,9 +457,6 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus,
         assert(offset < nlocalsplus);
         // For now we do not distinguish arg kinds.
         _PyLocals_Kind kind = CO_FAST_LOCAL;
-        if (PyDict_Contains(umd->u_fasthidden, k)) {
-            kind |= CO_FAST_HIDDEN;
-        }
         if (PyDict_GetItem(umd->u_cellvars, k) != NULL) {
             kind |= CO_FAST_CELL;
         }

@@ -693,7 +693,6 @@ compiler_unit_free(struct compiler_unit *u)
     Py_CLEAR(u->u_metadata.u_varnames);
     Py_CLEAR(u->u_metadata.u_freevars);
     Py_CLEAR(u->u_metadata.u_cellvars);
-    Py_CLEAR(u->u_metadata.u_fasthidden);
     Py_CLEAR(u->u_private);
     PyObject_Free(u);
 }
@@ -1283,12 +1282,6 @@ compiler_enter_scope(struct compiler *c, identifier name,
         return ERROR;
     }
 
-    u->u_metadata.u_fasthidden = PyDict_New();
-    if (!u->u_metadata.u_fasthidden) {
-        compiler_unit_free(u);
-        return ERROR;
-    }
-
     u->u_nfblocks = 0;
     u->u_metadata.u_firstlineno = lineno;
     u->u_metadata.u_consts = PyDict_New();
@@ -4114,8 +4107,7 @@ compiler_nameop(struct compiler *c, location loc,
         optype = OP_DEREF;
         break;
     case LOCAL:
-        if (_PyST_IsFunctionLike(c->u->u_ste) ||
-                (PyDict_GetItem(c->u->u_metadata.u_fasthidden, mangled) == Py_True))
+        if (_PyST_IsFunctionLike(c->u->u_ste))
             optype = OP_FAST;
         break;
     case GLOBAL_IMPLICIT:
@@ -5407,7 +5399,6 @@ compiler_async_comprehension_generator(struct compiler *c, location loc,
 typedef struct {
     PyObject *pushed_locals;
     PyObject *temp_symbols;
-    PyObject *fast_hidden;
 } inlined_comprehension_state;
 
 static int
@@ -5427,24 +5418,6 @@ push_inlined_comprehension_state(struct compiler *c, location loc,
         // assignment expression to a nonlocal in the comprehension, these don't
         // need handling here since they shouldn't be isolated
         if (symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) {
-            if (!_PyST_IsFunctionLike(c->u->u_ste)) {
-                // non-function scope: override this name to use fast locals
-                PyObject *orig = PyDict_GetItem(c->u->u_metadata.u_fasthidden, k);
-                if (orig != Py_True) {
-                    if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) {
-                        return ERROR;
-                    }
-                    if (state->fast_hidden == NULL) {
-                        state->fast_hidden = PySet_New(NULL);
-                        if (state->fast_hidden == NULL) {
-                            return ERROR;
-                        }
-                    }
-                    if (PySet_Add(state->fast_hidden, k) < 0) {
-                        return ERROR;
-                    }
-                }
-            }
             long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK;
             PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k);
             if (outv == NULL) {
@@ -5550,22 +5523,6 @@ pop_inlined_comprehension_state(struct compiler *c, location loc,
         }
         Py_CLEAR(state.pushed_locals);
     }
-    if (state.fast_hidden) {
-        while (PySet_Size(state.fast_hidden) > 0) {
-            PyObject *k = PySet_Pop(state.fast_hidden);
-            if (k == NULL) {
-                return ERROR;
-            }
-            // we set to False instead of clearing, so we can track which names
-            // were temporarily fast-locals and should use CO_FAST_HIDDEN
-            if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_False)) {
-                Py_DECREF(k);
-                return ERROR;
-            }
-            Py_DECREF(k);
-        }
-        Py_CLEAR(state.fast_hidden);
-    }
     return SUCCESS;
 }
 
@@ -5712,7 +5669,6 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
     Py_XDECREF(entry);
     Py_XDECREF(inline_state.pushed_locals);
     Py_XDECREF(inline_state.temp_symbols);
-    Py_XDECREF(inline_state.fast_hidden);
     return ERROR;
 }
 

@@ -987,6 +987,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
 
         // we inline all non-generator-expression comprehensions
         int inline_comp =
+            ste->ste_type == FunctionBlock &&
             entry->ste_comprehension &&
             !entry->ste_generator;