From 9d452fb174388ed861a0939f6099eefa7c6ad29d Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Mon, 15 May 2023 17:33:43 -0700 Subject: [PATCH 1/2] gh-104374: don't inline comprehensions in non-function scopes --- Include/internal/pycore_code.h | 1 - Include/internal/pycore_compile.h | 3 -- Lib/test/test_compile.py | 12 ++++++++ Lib/test/test_compiler_assemble.py | 5 +--- Lib/test/test_listcomps.py | 40 ++++++++++++++++++++++++-- Modules/_testinternalcapi.c | 2 -- Objects/frameobject.c | 3 -- Python/assemble.c | 3 -- Python/compile.c | 46 +----------------------------- Python/symtable.c | 1 + 10 files changed, 53 insertions(+), 63 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 75a23f3f5af560..2d0c49dfa0c790 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -128,7 +128,6 @@ struct callable_cache { // Note that these all fit within a byte, as do combinations. // Later, we will use the smaller numbers to differentiate the different // kinds of locals (e.g. pos-only arg, varkwargs, local-only). -#define CO_FAST_HIDDEN 0x10 #define CO_FAST_LOCAL 0x20 #define CO_FAST_CELL 0x40 #define CO_FAST_FREE 0x80 diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 499f55f3e276be..d2b12c91fe7a00 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -70,9 +70,6 @@ typedef struct { PyObject *u_varnames; /* local variables */ PyObject *u_cellvars; /* cell variables */ PyObject *u_freevars; /* free variables */ - PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only - temporarily within an inlined comprehension. When - value is True, treat as fast-local. */ Py_ssize_t u_argcount; /* number of arguments for block */ Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */ diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index c68b9ce388466e..304edc9bb12b83 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1352,11 +1352,14 @@ def test_multiline_list_comprehension(self): and x != 50)] """) compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', line=1, end_line=2, column=1, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_list_comprehension(self): snippet = textwrap.dedent("""\ @@ -1390,11 +1393,14 @@ def test_multiline_set_comprehension(self): and x != 50)} """) compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', line=1, end_line=2, column=1, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_set_comprehension(self): snippet = textwrap.dedent("""\ @@ -1428,11 +1434,14 @@ def test_multiline_dict_comprehension(self): and x != 50)} """) compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', line=1, end_line=2, column=1, end_column=7, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=7, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_dict_comprehension(self): snippet = textwrap.dedent("""\ @@ -1702,6 +1711,9 @@ def test_column_offset_deduplication(self): for source in [ "lambda: a", "(a for b in c)", + "[a for b in c]", + "{a for b in c}", + "{a: b for c in d}", ]: with self.subTest(source): code = compile(f"{source}, {source}", "", "eval") diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 3e2a127de728cd..0bd7a09b001c1d 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -16,7 +16,7 @@ def complete_metadata(self, metadata, filename="myfile.py"): metadata.setdefault(key, key) for key in ['consts']: metadata.setdefault(key, []) - for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']: + for key in ['names', 'varnames', 'cellvars', 'freevars']: metadata.setdefault(key, {}) for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']: metadata.setdefault(key, 0) @@ -33,9 +33,6 @@ def assemble_test(self, insts, metadata, expected): expected_metadata = {} for key, value in metadata.items(): - if key == "fasthidden": - # not exposed on code object - continue if isinstance(value, list): expected_metadata[key] = tuple(value) elif isinstance(value, dict): diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 23e1b8c1ce3193..7e29c784f698a2 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -200,7 +200,7 @@ def f(): y = [g for x in [1]] """ outputs = {"y": [2]} - self._check_in_scopes(code, outputs) + self._check_in_scopes(code, outputs, scopes=["module", "function"]) def test_inner_cell_shadows_outer_redefined(self): code = """ @@ -328,7 +328,7 @@ def test_nested_2(self): y = [x for [x ** x for x in range(x)][x - 1] in l] """ outputs = {"y": [3, 3, 3]} - self._check_in_scopes(code, outputs) + self._check_in_scopes(code, outputs, scopes=["module", "function"]) def test_nested_3(self): code = """ @@ -379,6 +379,42 @@ def f(): with self.assertRaises(UnboundLocalError): f() + def test_name_error_in_class_scope(self): + code = """ + y = 1 + [x + y for x in range(2)] + """ + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_global_in_class_scope(self): + code = """ + y = 2 + vals = [(x, y) for x in range(2)] + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["class"]) + + def test_in_class_scope_inside_function_1(self): + code = """ + class C: + y = 2 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["function"]) + + def test_in_class_scope_inside_function_2(self): + code = """ + y = 1 + class C: + y = 2 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, scopes=["function"]) + __test__ = {'doctests' : doctests} diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ea9b6e72b3c924..8009dca3d0b746 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -669,14 +669,12 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, umd.u_varnames = PyDict_GetItemString(metadata, "varnames"); umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars"); umd.u_freevars = PyDict_GetItemString(metadata, "freevars"); - umd.u_fasthidden = PyDict_GetItemString(metadata, "fasthidden"); assert(PyDict_Check(umd.u_consts)); assert(PyDict_Check(umd.u_names)); assert(PyDict_Check(umd.u_varnames)); assert(PyDict_Check(umd.u_cellvars)); assert(PyDict_Check(umd.u_freevars)); - assert(PyDict_Check(umd.u_fasthidden)); umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount"); umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount"); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2c90a6b71311ca..df0910d58e116c 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1222,9 +1222,6 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - if (kind & CO_FAST_HIDDEN) { - continue; - } if (value == NULL) { if (PyObject_DelItem(locals, name) != 0) { if (PyErr_ExceptionMatches(PyExc_KeyError)) { diff --git a/Python/assemble.c b/Python/assemble.c index 8789d8ef978c22..aefff3cd9d8402 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -457,9 +457,6 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, assert(offset < nlocalsplus); // For now we do not distinguish arg kinds. _PyLocals_Kind kind = CO_FAST_LOCAL; - if (PyDict_Contains(umd->u_fasthidden, k)) { - kind |= CO_FAST_HIDDEN; - } if (PyDict_GetItem(umd->u_cellvars, k) != NULL) { kind |= CO_FAST_CELL; } diff --git a/Python/compile.c b/Python/compile.c index bf5e4a52482a4a..8991000ad7d415 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -691,7 +691,6 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_varnames); Py_CLEAR(u->u_metadata.u_freevars); Py_CLEAR(u->u_metadata.u_cellvars); - Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); PyObject_Free(u); } @@ -1260,12 +1259,6 @@ compiler_enter_scope(struct compiler *c, identifier name, return ERROR; } - u->u_metadata.u_fasthidden = PyDict_New(); - if (!u->u_metadata.u_fasthidden) { - compiler_unit_free(u); - return ERROR; - } - u->u_nfblocks = 0; u->u_metadata.u_firstlineno = lineno; u->u_metadata.u_consts = PyDict_New(); @@ -3725,8 +3718,7 @@ compiler_nameop(struct compiler *c, location loc, optype = OP_DEREF; break; case LOCAL: - if (c->u->u_ste->ste_type == FunctionBlock || - (PyDict_GetItem(c->u->u_metadata.u_fasthidden, mangled) == Py_True)) + if (c->u->u_ste->ste_type == FunctionBlock) optype = OP_FAST; break; case GLOBAL_IMPLICIT: @@ -4988,7 +4980,6 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, typedef struct { PyObject *pushed_locals; PyObject *temp_symbols; - PyObject *fast_hidden; } inlined_comprehension_state; static int @@ -5008,24 +4999,6 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // assignment expression to a nonlocal in the comprehension, these don't // need handling here since they shouldn't be isolated if (symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) { - if (c->u->u_ste->ste_type != FunctionBlock) { - // non-function scope: override this name to use fast locals - PyObject *orig = PyDict_GetItem(c->u->u_metadata.u_fasthidden, k); - if (orig != Py_True) { - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { - return ERROR; - } - if (state->fast_hidden == NULL) { - state->fast_hidden = PySet_New(NULL); - if (state->fast_hidden == NULL) { - return ERROR; - } - } - if (PySet_Add(state->fast_hidden, k) < 0) { - return ERROR; - } - } - } long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); if (outv == NULL) { @@ -5131,22 +5104,6 @@ pop_inlined_comprehension_state(struct compiler *c, location loc, } Py_CLEAR(state.pushed_locals); } - if (state.fast_hidden) { - while (PySet_Size(state.fast_hidden) > 0) { - PyObject *k = PySet_Pop(state.fast_hidden); - if (k == NULL) { - return ERROR; - } - // we set to False instead of clearing, so we can track which names - // were temporarily fast-locals and should use CO_FAST_HIDDEN - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_False)) { - Py_DECREF(k); - return ERROR; - } - Py_DECREF(k); - } - Py_CLEAR(state.fast_hidden); - } return SUCCESS; } @@ -5293,7 +5250,6 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, Py_XDECREF(entry); Py_XDECREF(inline_state.pushed_locals); Py_XDECREF(inline_state.temp_symbols); - Py_XDECREF(inline_state.fast_hidden); return ERROR; } diff --git a/Python/symtable.c b/Python/symtable.c index 2c29f608413501..c51c21988ec7f1 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -912,6 +912,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, // we inline all non-generator-expression comprehensions int inline_comp = + ste->ste_type == FunctionBlock && entry->ste_comprehension && !entry->ste_generator; From 2c47a7551dd67263cce189ef4017463c914f19e9 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Mon, 15 May 2023 17:51:30 -0700 Subject: [PATCH 2/2] update NEWS and What's New --- Doc/whatsnew/3.12.rst | 7 ++++--- .../2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3e55b3fa0f4734..abd6ed4f038d0d 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -158,9 +158,10 @@ New Features PEP 709: Comprehension inlining ------------------------------- -Dictionary, list, and set comprehensions are now inlined, rather than creating a -new single-use function object for each execution of the comprehension. This -speeds up execution of a comprehension by up to 2x. +Dictionary, list, and set comprehensions occurring inside functions are now +inlined, rather than creating a new single-use function object for each +execution of the comprehension. This speeds up execution of a comprehension by +up to 2x. Comprehension iteration variables remain isolated; they don't overwrite a variable of the same name in the outer scope, nor are they visible after the diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst index 2eec05cb3ace5c..8d55567a9a6d36 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst @@ -1,2 +1,2 @@ -:pep:`709`: inline list, dict and set comprehensions to improve performance -and reduce bytecode size. +:pep:`709`: inline list, dict and set comprehensions in functions to improve +performance and reduce bytecode size.