From 36a7d56e5ae06a4269052f6961dbe1154ccc57ef Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 14 Jul 2023 22:02:39 +0100 Subject: [PATCH 01/26] gh-105481: add haslocal to _opcode.py. Generate most oplists in opcode.py with the functions in _opcode.py --- Include/cpython/compile.h | 3 +- Include/internal/pycore_opcode_metadata.h | 34 +++++---- Lib/opcode.py | 93 ++++++++++------------- Lib/test/test__opcode.py | 18 ++--- Modules/_opcode.c | 20 +++++ Modules/clinic/_opcode.c.h | 65 +++++++++++++++- Python/bytecodes.c | 2 +- Python/ceval_macros.h | 1 + Python/compile.c | 5 ++ Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 2 +- Tools/cases_generator/generate_cases.py | 5 +- 12 files changed, 165 insertions(+), 85 deletions(-) diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index cd7fd7bd377663..dc663ffbd9342d 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -71,5 +71,6 @@ PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int j PyAPI_FUNC(int) PyUnstable_OpcodeIsValid(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasArg(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasConst(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 8373f56653b1c7..ceaba34a52f9b6 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -952,10 +952,12 @@ enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT #define HAS_CONST_FLAG (2) #define HAS_NAME_FLAG (4) #define HAS_JUMP_FLAG (8) +#define HAS_LOCAL_FLAG (16) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) #define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_JUMP_FLAG)) +#define OPCODE_HAS_LOCAL(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_LOCAL_FLAG)) struct opcode_metadata { bool valid_entry; @@ -992,16 +994,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [NOP] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [LOAD_CLOSURE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_MAYBE_NULL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, [END_FOR] = { true, INSTR_FMT_IB, 0 }, @@ -1025,7 +1027,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IBC, 0 }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IBC, 0 }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, 0 }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, HAS_LOCAL_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SLICE] = { true, INSTR_FMT_IX, 0 }, [STORE_SLICE] = { true, INSTR_FMT_IX, 0 }, @@ -1077,12 +1079,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, diff --git a/Lib/opcode.py b/Lib/opcode.py index bc885051c6454e..0d7db5be3207db 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -29,12 +29,6 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') -hasarg = [] -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] hascompare = [] hasfree = [] hasexc = [] @@ -45,8 +39,7 @@ def is_pseudo(op): return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE -oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs, - haslocal, hascompare, hasfree, hasexc] +oplists = [hascompare, hasfree, hasexc] opmap = {} @@ -57,18 +50,6 @@ def is_pseudo(op): def def_op(name, op): opmap[name] = op -def name_op(name, op): - def_op(name, op) - hasname.append(op) - -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) - -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) - def pseudo_op(name, op, real_ops): def_op(name, op) _pseudo_ops[name] = real_ops @@ -149,55 +130,49 @@ def pseudo_op(name, op, real_ops): HAVE_ARGUMENT = 90 # real opcodes from here have an argument: -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" +def_op('STORE_NAME', 90) # Index in name list +def_op('DELETE_NAME', 91) # "" def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) +def_op('FOR_ITER', 93) def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" +def_op('STORE_ATTR', 95) # Index in name list +def_op('DELETE_ATTR', 96) # "" +def_op('STORE_GLOBAL', 97) # "" +def_op('DELETE_GLOBAL', 98) # "" def_op('SWAP', 99) def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list +def_op('LOAD_NAME', 101) # Index in name list def_op('BUILD_TUPLE', 102) # Number of tuple items def_op('BUILD_LIST', 103) # Number of list items def_op('BUILD_SET', 104) # Number of set items def_op('BUILD_MAP', 105) # Number of dict entries -name_op('LOAD_ATTR', 106) # Index in name list +def_op('LOAD_ATTR', 106) # Index in name list def_op('COMPARE_OP', 107) # Comparison operator hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list -jrel_op('JUMP_FORWARD', 110) # Number of words to skip +def_op('IMPORT_NAME', 108) # Index in name list +def_op('IMPORT_FROM', 109) # Index in name list +def_op('JUMP_FORWARD', 110) # Number of words to skip -jrel_op('POP_JUMP_IF_FALSE', 114) -jrel_op('POP_JUMP_IF_TRUE', 115) -name_op('LOAD_GLOBAL', 116) # Index in name list +def_op('POP_JUMP_IF_FALSE', 114) +def_op('POP_JUMP_IF_TRUE', 115) +def_op('LOAD_GLOBAL', 116) # Index in name list def_op('IS_OP', 117) def_op('CONTAINS_OP', 118) def_op('RERAISE', 119) def_op('COPY', 120) def_op('RETURN_CONST', 121) -hasconst.append(121) def_op('BINARY_OP', 122) -jrel_op('SEND', 123) # Number of words to skip +def_op('SEND', 123) # Number of words to skip def_op('LOAD_FAST', 124) # Local variable number, no null check -haslocal.append(124) def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) def_op('LOAD_FAST_CHECK', 127) # Local variable number -haslocal.append(127) -jrel_op('POP_JUMP_IF_NOT_NONE', 128) -jrel_op('POP_JUMP_IF_NONE', 129) +def_op('POP_JUMP_IF_NOT_NONE', 128) +def_op('POP_JUMP_IF_NONE', 129) def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) def_op('GET_AWAITABLE', 131) def_op('BUILD_SLICE', 133) # Number of items -jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) +def_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) def_op('MAKE_CELL', 135) hasfree.append(135) def_op('LOAD_DEREF', 137) @@ -206,11 +181,10 @@ def pseudo_op(name, op, real_ops): hasfree.append(138) def_op('DELETE_DEREF', 139) hasfree.append(139) -jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) -name_op('LOAD_SUPER_ATTR', 141) +def_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) +def_op('LOAD_SUPER_ATTR', 141) def_op('CALL_FUNCTION_EX', 142) # Flags def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number -haslocal.append(143) def_op('EXTENDED_ARG', 144) EXTENDED_ARG = 144 def_op('LIST_APPEND', 145) @@ -236,10 +210,9 @@ def pseudo_op(name, op, real_ops): def_op('STORE_FAST_STORE_FAST', 170) def_op('CALL', 171) def_op('KW_NAMES', 172) -hasconst.append(172) def_op('CALL_INTRINSIC_1', 173) def_op('CALL_INTRINSIC_2', 174) -name_op('LOAD_FROM_DICT_OR_GLOBALS', 175) +def_op('LOAD_FROM_DICT_OR_GLOBALS', 175) def_op('LOAD_FROM_DICT_OR_DEREF', 176) hasfree.append(176) def_op('SET_FUNCTION_ATTRIBUTE', 177) # Attribute @@ -270,7 +243,6 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_LINE', 254) # 255 is reserved -hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) MIN_PSEUDO_OPCODE = 256 @@ -295,12 +267,29 @@ def pseudo_op(name, op, real_ops): MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 -del def_op, name_op, jrel_op, jabs_op, pseudo_op +del def_op, pseudo_op opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] for op, i in opmap.items(): opname[i] = op +# _opcode may not be ready during early stages of the build +try: + import _opcode +except ImportError: + hasarg = [] + hasconst = [] + hasname = [] + hasjrel = [] + hasjabs = [] + haslocal = [] +else: + hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] + hasconst = [op for op in opmap.values() if _opcode.has_const(op)] + hasname = [op for op in opmap.values() if _opcode.has_name(op)] + hasjrel = [op for op in opmap.values() if _opcode.has_jump(op)] + hasjabs = [] + haslocal = [op for op in opmap.values() if _opcode.has_local(op)] _nb_ops = [ ("NB_ADD", "+"), diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 7d9553d9e383a3..74be90bc3cb6eb 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -24,6 +24,7 @@ def test_invalid_opcodes(self): self.check_bool_function_result(_opcode.has_const, invalid, False) self.check_bool_function_result(_opcode.has_name, invalid, False) self.check_bool_function_result(_opcode.has_jump, invalid, False) + self.check_bool_function_result(_opcode.has_local, invalid, False) def test_is_valid(self): names = [ @@ -61,17 +62,12 @@ def test_has_jump(self): self.check_bool_function_result(_opcode.has_jump, has_jump, True) self.check_bool_function_result(_opcode.has_jump, no_jump, False) - # the following test is part of the refactor, it will be removed soon - def test_against_legacy_bool_values(self): - # limiting to ops up to ENTER_EXECUTOR, because everything after that - # is not currently categorized correctly in opcode.py. - for op in range(0, opcode.opmap['ENTER_EXECUTOR']): - with self.subTest(op=op): - if opcode.opname[op] != f'<{op}>': - self.assertEqual(op in dis.hasarg, _opcode.has_arg(op)) - self.assertEqual(op in dis.hasconst, _opcode.has_const(op)) - self.assertEqual(op in dis.hasname, _opcode.has_name(op)) - self.assertEqual(op in dis.hasjrel, _opcode.has_jump(op)) + def test_has_local(self): + has_local = ['LOAD_FAST', 'LOAD_FAST_CHECK', 'LOAD_FAST_AND_CLEAR', + 'STORE_FAST_MAYBE_NULL', 'LOAD_CLOSURE'] + no_local = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] + self.check_bool_function_result(_opcode.has_local, has_local, True) + self.check_bool_function_result(_opcode.has_local, no_local, False) def test_stack_effect(self): self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) diff --git a/Modules/_opcode.c b/Modules/_opcode.c index b3b9873d21a5be..1bc310c7d564d8 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -145,6 +145,25 @@ _opcode_has_jump_impl(PyObject *module, int opcode) } +/*[clinic input] + +_opcode.has_local -> bool + + opcode: int + +Return True if the opcode accesses a local variable, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_has_local_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=da5a8616b7a5097b input=9a798ee24aaef49d]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasLocal(opcode); + +} + + /*[clinic input] _opcode.get_specialization_stats @@ -171,6 +190,7 @@ opcode_functions[] = { _OPCODE_HAS_CONST_METHODDEF _OPCODE_HAS_NAME_METHODDEF _OPCODE_HAS_JUMP_METHODDEF + _OPCODE_HAS_LOCAL_METHODDEF _OPCODE_GET_SPECIALIZATION_STATS_METHODDEF {NULL, NULL, 0, NULL} }; diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index 3eb050e470c343..f25da9e1d0968c 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -401,6 +401,69 @@ _opcode_has_jump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(_opcode_has_local__doc__, +"has_local($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode accesses a local variable, False otherwise."); + +#define _OPCODE_HAS_LOCAL_METHODDEF \ + {"has_local", _PyCFunction_CAST(_opcode_has_local), METH_FASTCALL|METH_KEYWORDS, _opcode_has_local__doc__}, + +static int +_opcode_has_local_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_local(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_local", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_local_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_opcode_get_specialization_stats__doc__, "get_specialization_stats($module, /)\n" "--\n" @@ -418,4 +481,4 @@ _opcode_get_specialization_stats(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_specialization_stats_impl(module); } -/*[clinic end generated code: output=ae2b2ef56d582180 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=541693b77c7d8292 input=a9049054013a1b77]*/ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 15b48ae9d82672..93e8edf2cdde34 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3611,7 +3611,7 @@ dummy_func( _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 72800aaaaa2ac4..d76182db3d0b05 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -234,6 +234,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* Local variable macros */ +#define LOCALS_ARRAY (frame->localsplus) #define GETLOCAL(i) (frame->localsplus[i]) /* The SETLOCAL() macro must not DECREF the local variable in-place and diff --git a/Python/compile.c b/Python/compile.c index 9e86e06777ffa4..6524665bfb12ea 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -896,6 +896,11 @@ PyUnstable_OpcodeHasJump(int opcode) return OPCODE_HAS_JUMP(opcode); } +int PyUnstable_OpcodeHasLocal(int opcode) +{ + return OPCODE_HAS_LOCAL(opcode); +} + static int codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 626baece814607..ed8b1812032ae7 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2035,7 +2035,7 @@ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 68531dc074769e..2bd11a8ac5db7e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4463,7 +4463,7 @@ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 6589289121863b..5a5d016f202e53 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -263,6 +263,7 @@ class InstructionFlags: HAS_CONST_FLAG: bool HAS_NAME_FLAG: bool HAS_JUMP_FLAG: bool + HAS_LOCAL_FLAG: bool def __post_init__(self): self.bitmask = { @@ -276,11 +277,13 @@ def fromInstruction(instr: "AnyInstruction"): HAS_CONST_FLAG=variable_used(instr, "FRAME_CO_CONSTS"), HAS_NAME_FLAG=variable_used(instr, "FRAME_CO_NAMES"), HAS_JUMP_FLAG=variable_used(instr, "JUMPBY"), + HAS_LOCAL_FLAG=variable_used(instr, "GETLOCAL") or + variable_used(instr, "SETLOCAL"), ) @staticmethod def newEmpty(): - return InstructionFlags(False, False, False, False) + return InstructionFlags(*(5 * (False,))) def add(self, other: "InstructionFlags") -> None: for name, value in dataclasses.asdict(other).items(): From 75019ca4caeecf88cb391b63733c79dfc5e6804f Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 14 Jul 2023 23:08:28 +0100 Subject: [PATCH 02/26] added HAS_FREE flag. Removed HAS_FREE from the HAS_LOCAL list --- Include/cpython/compile.h | 4 +- Include/internal/pycore_opcode_metadata.h | 14 +++-- Lib/opcode.py | 25 +++----- Lib/test/test__opcode.py | 8 +++ Modules/_opcode.c | 24 ++++++++ Modules/clinic/_opcode.c.h | 70 ++++++++++++++++++++++- Python/compile.c | 9 ++- Tools/cases_generator/generate_cases.py | 14 ++++- 8 files changed, 140 insertions(+), 28 deletions(-) diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index dc663ffbd9342d..36c970a081ad9f 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -71,6 +71,8 @@ PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int j PyAPI_FUNC(int) PyUnstable_OpcodeIsValid(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasArg(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasConst(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasFree(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode); -PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode); + diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index ceaba34a52f9b6..f2b284ee46b101 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -952,11 +952,13 @@ enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT #define HAS_CONST_FLAG (2) #define HAS_NAME_FLAG (4) #define HAS_JUMP_FLAG (8) -#define HAS_LOCAL_FLAG (16) +#define HAS_FREE_FLAG (16) +#define HAS_LOCAL_FLAG (32) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) #define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_JUMP_FLAG)) +#define OPCODE_HAS_FREE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_FREE_FLAG)) #define OPCODE_HAS_LOCAL(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_LOCAL_FLAG)) struct opcode_metadata { @@ -1080,11 +1082,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, + [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, diff --git a/Lib/opcode.py b/Lib/opcode.py index 0d7db5be3207db..a45a938e2e17e3 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -30,7 +30,6 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') hascompare = [] -hasfree = [] hasexc = [] @@ -39,7 +38,7 @@ def is_pseudo(op): return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE -oplists = [hascompare, hasfree, hasexc] +oplists = [hascompare, hasexc] opmap = {} @@ -174,13 +173,9 @@ def pseudo_op(name, op, real_ops): def_op('BUILD_SLICE', 133) # Number of items def_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) def_op('MAKE_CELL', 135) -hasfree.append(135) def_op('LOAD_DEREF', 137) -hasfree.append(137) def_op('STORE_DEREF', 138) -hasfree.append(138) def_op('DELETE_DEREF', 139) -hasfree.append(139) def_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) def_op('LOAD_SUPER_ATTR', 141) def_op('CALL_FUNCTION_EX', 142) # Flags @@ -190,7 +185,6 @@ def pseudo_op(name, op, real_ops): def_op('LIST_APPEND', 145) def_op('SET_ADD', 146) def_op('MAP_ADD', 147) -hasfree.append(148) def_op('COPY_FREE_VARS', 149) def_op('YIELD_VALUE', 150) def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py @@ -214,7 +208,6 @@ def pseudo_op(name, op, real_ops): def_op('CALL_INTRINSIC_2', 174) def_op('LOAD_FROM_DICT_OR_GLOBALS', 175) def_op('LOAD_FROM_DICT_OR_DEREF', 176) -hasfree.append(176) def_op('SET_FUNCTION_ATTRIBUTE', 177) # Attribute # Optimizer hook @@ -276,20 +269,20 @@ def pseudo_op(name, op, real_ops): # _opcode may not be ready during early stages of the build try: import _opcode -except ImportError: - hasarg = [] - hasconst = [] - hasname = [] - hasjrel = [] - hasjabs = [] - haslocal = [] -else: hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] hasconst = [op for op in opmap.values() if _opcode.has_const(op)] hasname = [op for op in opmap.values() if _opcode.has_name(op)] hasjrel = [op for op in opmap.values() if _opcode.has_jump(op)] hasjabs = [] + hasfree = [op for op in opmap.values() if _opcode.has_free(op)] haslocal = [op for op in opmap.values() if _opcode.has_local(op)] +except (ImportError, AttributeError): + hasarg = [] + hasconst = [] + hasname = [] + hasjrel = [] + hasjabs = [] + haslocal = [] _nb_ops = [ ("NB_ADD", "+"), diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 74be90bc3cb6eb..18a0d8c110c22e 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -24,6 +24,7 @@ def test_invalid_opcodes(self): self.check_bool_function_result(_opcode.has_const, invalid, False) self.check_bool_function_result(_opcode.has_name, invalid, False) self.check_bool_function_result(_opcode.has_jump, invalid, False) + self.check_bool_function_result(_opcode.has_free, invalid, False) self.check_bool_function_result(_opcode.has_local, invalid, False) def test_is_valid(self): @@ -62,6 +63,13 @@ def test_has_jump(self): self.check_bool_function_result(_opcode.has_jump, has_jump, True) self.check_bool_function_result(_opcode.has_jump, no_jump, False) + def test_has_free(self): + has_free = ['MAKE_CELL', 'LOAD_DEREF', 'STORE_DEREF', 'DELETE_DEREF', + 'LOAD_FROM_DICT_OR_DEREF'] + no_free = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] + self.check_bool_function_result(_opcode.has_free, has_free, True) + self.check_bool_function_result(_opcode.has_free, no_free, False) + def test_has_local(self): has_local = ['LOAD_FAST', 'LOAD_FAST_CHECK', 'LOAD_FAST_AND_CLEAR', 'STORE_FAST_MAYBE_NULL', 'LOAD_CLOSURE'] diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 1bc310c7d564d8..9d0b658c57d401 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -147,6 +147,29 @@ _opcode_has_jump_impl(PyObject *module, int opcode) /*[clinic input] +_opcode.has_free -> bool + + opcode: int + +Return True if the opcode accesses a free variable, False otherwise. + +Note that ‘free’ in this context refers to names in the current scope +that are referenced by inner scopes or names in outer scopes that are +referenced from this scope. It does not include references to global +or builtin scopes. +[clinic start generated code]*/ + +static int +_opcode_has_free_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=d81ae4d79af0ee26 input=6b2ce67a4a4017b1]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasFree(opcode); + +} + +/*[clinic input] + _opcode.has_local -> bool opcode: int @@ -190,6 +213,7 @@ opcode_functions[] = { _OPCODE_HAS_CONST_METHODDEF _OPCODE_HAS_NAME_METHODDEF _OPCODE_HAS_JUMP_METHODDEF + _OPCODE_HAS_FREE_METHODDEF _OPCODE_HAS_LOCAL_METHODDEF _OPCODE_GET_SPECIALIZATION_STATS_METHODDEF {NULL, NULL, 0, NULL} diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index f25da9e1d0968c..36a815a648ae74 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -401,6 +401,74 @@ _opcode_has_jump(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +PyDoc_STRVAR(_opcode_has_free__doc__, +"has_free($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode accesses a free variable, False otherwise.\n" +"\n" +"Note that ‘free’ in this context refers to names in the current scope\n" +"that are referenced by inner scopes or names in outer scopes that are\n" +"referenced from this scope. It does not include references to global\n" +"or builtin scopes."); + +#define _OPCODE_HAS_FREE_METHODDEF \ + {"has_free", _PyCFunction_CAST(_opcode_has_free), METH_FASTCALL|METH_KEYWORDS, _opcode_has_free__doc__}, + +static int +_opcode_has_free_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_free", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_free_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_opcode_has_local__doc__, "has_local($module, /, opcode)\n" "--\n" @@ -481,4 +549,4 @@ _opcode_get_specialization_stats(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_specialization_stats_impl(module); } -/*[clinic end generated code: output=541693b77c7d8292 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=683ab701c714b3f2 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 6524665bfb12ea..9ba389581666b7 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -896,7 +896,14 @@ PyUnstable_OpcodeHasJump(int opcode) return OPCODE_HAS_JUMP(opcode); } -int PyUnstable_OpcodeHasLocal(int opcode) +int +PyUnstable_OpcodeHasFree(int opcode) +{ + return OPCODE_HAS_FREE(opcode); +} + +int +PyUnstable_OpcodeHasLocal(int opcode) { return OPCODE_HAS_LOCAL(opcode); } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 5a5d016f202e53..fc88bd385baffd 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -263,6 +263,7 @@ class InstructionFlags: HAS_CONST_FLAG: bool HAS_NAME_FLAG: bool HAS_JUMP_FLAG: bool + HAS_FREE_FLAG: bool HAS_LOCAL_FLAG: bool def __post_init__(self): @@ -272,18 +273,25 @@ def __post_init__(self): @staticmethod def fromInstruction(instr: "AnyInstruction"): + + has_free = (variable_used(instr, "PyCell_New") or + variable_used(instr, "PyCell_GET") or + variable_used(instr, "PyCell_SET")) + return InstructionFlags( HAS_ARG_FLAG=variable_used(instr, "oparg"), HAS_CONST_FLAG=variable_used(instr, "FRAME_CO_CONSTS"), HAS_NAME_FLAG=variable_used(instr, "FRAME_CO_NAMES"), HAS_JUMP_FLAG=variable_used(instr, "JUMPBY"), - HAS_LOCAL_FLAG=variable_used(instr, "GETLOCAL") or - variable_used(instr, "SETLOCAL"), + HAS_FREE_FLAG=has_free, + HAS_LOCAL_FLAG=(variable_used(instr, "GETLOCAL") or + variable_used(instr, "SETLOCAL")) and + not has_free, ) @staticmethod def newEmpty(): - return InstructionFlags(*(5 * (False,))) + return InstructionFlags(False, False, False, False, False, False) def add(self, other: "InstructionFlags") -> None: for name, value in dataclasses.asdict(other).items(): From 494680a6186e90ab6bb6a75ad38c7c49914214bc Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 15 Jul 2023 17:14:46 +0100 Subject: [PATCH 03/26] add hasjump, soft deprecate hasjrel and hasjabs. Fix build bug --- Doc/library/dis.rst | 28 +++++++++++++++++++++------- Lib/opcode.py | 11 +++++++---- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 099b6410f165ed..6beaad3825aba8 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1803,15 +1803,12 @@ instructions: Sequence of bytecodes that access an attribute by name. -.. data:: hasjrel - - Sequence of bytecodes that have a relative jump target. +.. data:: hasjump + Sequence of bytecodes that have a jump target. All jumps + are relative. -.. data:: hasjabs - - Sequence of bytecodes that have an absolute jump target. - + .. versionadded:: 3.13 .. data:: haslocal @@ -1827,3 +1824,20 @@ instructions: Sequence of bytecodes that set an exception handler. .. versionadded:: 3.12 + + +.. data:: hasjrel + + Sequence of bytecodes that have a relative jump target. + + .. deprecated:: 3.13 + All jumps are now relative. Use :data:`hasjump`. + + +.. data:: hasjabs + + Sequence of bytecodes that have an absolute jump target. + + .. deprecated:: 3.13 + All jumps are now relative. This list is empty. + diff --git a/Lib/opcode.py b/Lib/opcode.py index a45a938e2e17e3..9efdbf6c026cf5 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,9 +4,9 @@ operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", - "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG"] +__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjump", "hasjrel", + "hasjabs", "haslocal", "hascompare", "hasfree", "hasexc", + "opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"] # It's a chicken-and-egg I'm afraid: # We're imported before _opcode's made. @@ -272,7 +272,8 @@ def pseudo_op(name, op, real_ops): hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] hasconst = [op for op in opmap.values() if _opcode.has_const(op)] hasname = [op for op in opmap.values() if _opcode.has_name(op)] - hasjrel = [op for op in opmap.values() if _opcode.has_jump(op)] + hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] + hasjrel = hasjump # for backward compatibility hasjabs = [] hasfree = [op for op in opmap.values() if _opcode.has_free(op)] haslocal = [op for op in opmap.values() if _opcode.has_local(op)] @@ -280,8 +281,10 @@ def pseudo_op(name, op, real_ops): hasarg = [] hasconst = [] hasname = [] + hasjump = [] hasjrel = [] hasjabs = [] + hasfree = [] haslocal = [] _nb_ops = [ From 24786fbc33c6118a78c29a224f6e35456b02f4dc Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 15 Jul 2023 17:43:17 +0100 Subject: [PATCH 04/26] generate hasexc from the C macros instead of defining it again --- Include/cpython/compile.h | 1 + Lib/opcode.py | 8 ++--- Lib/test/test__opcode.py | 7 ++++ Modules/_opcode.c | 20 ++++++++++-- Modules/clinic/_opcode.c.h | 65 +++++++++++++++++++++++++++++++++++++- Python/compile.c | 6 ++++ 6 files changed, 99 insertions(+), 8 deletions(-) diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index 36c970a081ad9f..fd52697840203a 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -75,4 +75,5 @@ PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasFree(int opcode); PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode); +PyAPI_FUNC(int) PyUnstable_OpcodeHasExc(int opcode); diff --git a/Lib/opcode.py b/Lib/opcode.py index 9efdbf6c026cf5..ccbd6f5d479450 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -30,7 +30,6 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') hascompare = [] -hasexc = [] ENABLE_SPECIALIZATION = True @@ -38,7 +37,7 @@ def is_pseudo(op): return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE -oplists = [hascompare, hasexc] +oplists = [hascompare] opmap = {} @@ -240,11 +239,8 @@ def pseudo_op(name, op, real_ops): MIN_PSEUDO_OPCODE = 256 pseudo_op('SETUP_FINALLY', 256, ['NOP']) -hasexc.append(256) pseudo_op('SETUP_CLEANUP', 257, ['NOP']) -hasexc.append(257) pseudo_op('SETUP_WITH', 258, ['NOP']) -hasexc.append(258) pseudo_op('POP_BLOCK', 259, ['NOP']) pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) @@ -277,6 +273,7 @@ def pseudo_op(name, op, real_ops): hasjabs = [] hasfree = [op for op in opmap.values() if _opcode.has_free(op)] haslocal = [op for op in opmap.values() if _opcode.has_local(op)] + hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] except (ImportError, AttributeError): hasarg = [] hasconst = [] @@ -286,6 +283,7 @@ def pseudo_op(name, op, real_ops): hasjabs = [] hasfree = [] haslocal = [] + hasexc = [] _nb_ops = [ ("NB_ADD", "+"), diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 18a0d8c110c22e..ae468be6eeca4f 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -26,6 +26,7 @@ def test_invalid_opcodes(self): self.check_bool_function_result(_opcode.has_jump, invalid, False) self.check_bool_function_result(_opcode.has_free, invalid, False) self.check_bool_function_result(_opcode.has_local, invalid, False) + self.check_bool_function_result(_opcode.has_exc, invalid, False) def test_is_valid(self): names = [ @@ -77,6 +78,12 @@ def test_has_local(self): self.check_bool_function_result(_opcode.has_local, has_local, True) self.check_bool_function_result(_opcode.has_local, no_local, False) + def test_has_exc(self): + has_exc = ['SETUP_FINALLY', 'SETUP_WITH', 'SETUP_CLEANUP'] + no_exc = ['DELETE_DEREF', 'POP_TOP', 'NOP', 'CACHE'] + self.check_bool_function_result(_opcode.has_exc, has_exc, True) + self.check_bool_function_result(_opcode.has_exc, no_exc, False) + def test_stack_effect(self): self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1) diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 9d0b658c57d401..eda169fd9b3242 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -161,7 +161,7 @@ or builtin scopes. static int _opcode_has_free_impl(PyObject *module, int opcode) -/*[clinic end generated code: output=d81ae4d79af0ee26 input=6b2ce67a4a4017b1]*/ +/*[clinic end generated code: output=d81ae4d79af0ee26 input=f94ce9c2475f3ff0]*/ { return PyUnstable_OpcodeIsValid(opcode) && PyUnstable_OpcodeHasFree(opcode); @@ -183,9 +183,24 @@ _opcode_has_local_impl(PyObject *module, int opcode) { return PyUnstable_OpcodeIsValid(opcode) && PyUnstable_OpcodeHasLocal(opcode); - } +/*[clinic input] + +_opcode.has_exc -> bool + + opcode: int + +Return True if the opcode sets and exception handler, False otherwise. +[clinic start generated code]*/ + +static int +_opcode_has_exc_impl(PyObject *module, int opcode) +/*[clinic end generated code: output=41b68dff0ec82a52 input=a3f202bb94f233b5]*/ +{ + return PyUnstable_OpcodeIsValid(opcode) && + PyUnstable_OpcodeHasExc(opcode); +} /*[clinic input] @@ -215,6 +230,7 @@ opcode_functions[] = { _OPCODE_HAS_JUMP_METHODDEF _OPCODE_HAS_FREE_METHODDEF _OPCODE_HAS_LOCAL_METHODDEF + _OPCODE_HAS_EXC_METHODDEF _OPCODE_GET_SPECIALIZATION_STATS_METHODDEF {NULL, NULL, 0, NULL} }; diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index 36a815a648ae74..66ed445b3e7f91 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -532,6 +532,69 @@ _opcode_has_local(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } +PyDoc_STRVAR(_opcode_has_exc__doc__, +"has_exc($module, /, opcode)\n" +"--\n" +"\n" +"Return True if the opcode sets and exception handler, False otherwise."); + +#define _OPCODE_HAS_EXC_METHODDEF \ + {"has_exc", _PyCFunction_CAST(_opcode_has_exc), METH_FASTCALL|METH_KEYWORDS, _opcode_has_exc__doc__}, + +static int +_opcode_has_exc_impl(PyObject *module, int opcode); + +static PyObject * +_opcode_has_exc(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "has_exc", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int opcode; + int _return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = _PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = _opcode_has_exc_impl(module, opcode); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_opcode_get_specialization_stats__doc__, "get_specialization_stats($module, /)\n" "--\n" @@ -549,4 +612,4 @@ _opcode_get_specialization_stats(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_specialization_stats_impl(module); } -/*[clinic end generated code: output=683ab701c714b3f2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=36ba0ad318c53d65 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 9ba389581666b7..07b35028ac61a3 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -908,6 +908,12 @@ PyUnstable_OpcodeHasLocal(int opcode) return OPCODE_HAS_LOCAL(opcode); } +int +PyUnstable_OpcodeHasExc(int opcode) +{ + return IS_BLOCK_PUSH_OPCODE(opcode); +} + static int codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { From 563b431041eee7622aa7394af30a87debfc65e43 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 15 Jul 2023 21:41:37 +0100 Subject: [PATCH 05/26] typo --- Modules/_opcode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_opcode.c b/Modules/_opcode.c index eda169fd9b3242..637ec4c929e294 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -191,7 +191,7 @@ _opcode.has_exc -> bool opcode: int -Return True if the opcode sets and exception handler, False otherwise. +Return True if the opcode sets an exception handler, False otherwise. [clinic start generated code]*/ static int From 1b13b9683aeb9ad1c1771abe5b1463b9e840b8a1 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 17 Jul 2023 14:34:07 +0100 Subject: [PATCH 06/26] ascii quotes --- Modules/_opcode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 637ec4c929e294..e0e826fb7eaf94 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -153,7 +153,7 @@ _opcode.has_free -> bool Return True if the opcode accesses a free variable, False otherwise. -Note that ‘free’ in this context refers to names in the current scope +Note that 'free' in this context refers to names in the current scope that are referenced by inner scopes or names in outer scopes that are referenced from this scope. It does not include references to global or builtin scopes. @@ -196,7 +196,7 @@ Return True if the opcode sets an exception handler, False otherwise. static int _opcode_has_exc_impl(PyObject *module, int opcode) -/*[clinic end generated code: output=41b68dff0ec82a52 input=a3f202bb94f233b5]*/ +/*[clinic end generated code: output=41b68dff0ec82a52 input=db0e4bdb9bf13fa5]*/ { return PyUnstable_OpcodeIsValid(opcode) && PyUnstable_OpcodeHasExc(opcode); From 0f5ae32d9268aff0635d37480fbed34bb5067d14 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 17 Jul 2023 14:34:51 +0100 Subject: [PATCH 07/26] make clinic --- Modules/_opcode.c | 2 +- Modules/clinic/_opcode.c.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_opcode.c b/Modules/_opcode.c index e0e826fb7eaf94..daabdce1655777 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -161,7 +161,7 @@ or builtin scopes. static int _opcode_has_free_impl(PyObject *module, int opcode) -/*[clinic end generated code: output=d81ae4d79af0ee26 input=f94ce9c2475f3ff0]*/ +/*[clinic end generated code: output=d81ae4d79af0ee26 input=117dcd5c19c1139b]*/ { return PyUnstable_OpcodeIsValid(opcode) && PyUnstable_OpcodeHasFree(opcode); diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index 66ed445b3e7f91..e6381fa73a5506 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -407,7 +407,7 @@ PyDoc_STRVAR(_opcode_has_free__doc__, "\n" "Return True if the opcode accesses a free variable, False otherwise.\n" "\n" -"Note that ‘free’ in this context refers to names in the current scope\n" +"Note that \'free\' in this context refers to names in the current scope\n" "that are referenced by inner scopes or names in outer scopes that are\n" "referenced from this scope. It does not include references to global\n" "or builtin scopes."); @@ -536,7 +536,7 @@ PyDoc_STRVAR(_opcode_has_exc__doc__, "has_exc($module, /, opcode)\n" "--\n" "\n" -"Return True if the opcode sets and exception handler, False otherwise."); +"Return True if the opcode sets an exception handler, False otherwise."); #define _OPCODE_HAS_EXC_METHODDEF \ {"has_exc", _PyCFunction_CAST(_opcode_has_exc), METH_FASTCALL|METH_KEYWORDS, _opcode_has_exc__doc__}, @@ -612,4 +612,4 @@ _opcode_get_specialization_stats(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_specialization_stats_impl(module); } -/*[clinic end generated code: output=36ba0ad318c53d65 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e507bf14fb2796f8 input=a9049054013a1b77]*/ From 289fad199cb8f6033965ebc4efc392fd9570993b Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 17 Jul 2023 15:06:44 +0100 Subject: [PATCH 08/26] remove oplists --- Lib/opcode.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index ea87f5a67f2a75..48097adeb1be8f 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -25,8 +25,6 @@ def is_pseudo(op): return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE -oplists = [hascompare] - opmap = {} ## pseudo opcodes (used in the compiler) mapped to the values @@ -39,12 +37,6 @@ def def_op(name, op): def pseudo_op(name, op, real_ops): def_op(name, op) _pseudo_ops[name] = real_ops - # add the pseudo opcode to the lists its targets are in - for oplist in oplists: - res = [opmap[rop] in oplist for rop in real_ops] - if any(res): - assert all(res) - oplist.append(op) # Instruction opcodes for compiled code From 598ba2a621731338d5dcdccea9bd4f0c21713fb7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 17 Jul 2023 16:40:13 +0100 Subject: [PATCH 09/26] import less from opcode in build script. _opcode.has_* don't exist pre-3.13 --- Lib/opcode.py | 23 +++++++---------------- Tools/build/generate_opcode_h.py | 6 ------ 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 48097adeb1be8f..d7d1a3fb56cfd8 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,10 +4,9 @@ operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjump", "hasjrel", - "hasjabs", "haslocal", "hascompare", "hasfree", "hasexc", - "opname", "opmap", "stack_effect", "HAVE_ARGUMENT", "EXTENDED_ARG"] +__all__ = ["cmp_op", "opname", "opmap", "stack_effect", "HAVE_ARGUMENT", "EXTENDED_ARG"] +import _opcode from _opcode import stack_effect import sys @@ -242,9 +241,8 @@ def pseudo_op(name, op, real_ops): for op, i in opmap.items(): opname[i] = op -# _opcode may not be ready during early stages of the build -try: - import _opcode +# The build uses older versions of Python which do not have _opcode.has_* functions +if sys.version_info[:2] >= (3, 13): hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] hasconst = [op for op in opmap.values() if _opcode.has_const(op)] hasname = [op for op in opmap.values() if _opcode.has_name(op)] @@ -254,16 +252,9 @@ def pseudo_op(name, op, real_ops): hasfree = [op for op in opmap.values() if _opcode.has_free(op)] haslocal = [op for op in opmap.values() if _opcode.has_local(op)] hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] -except (ImportError, AttributeError): - hasarg = [] - hasconst = [] - hasname = [] - hasjump = [] - hasjrel = [] - hasjabs = [] - hasfree = [] - haslocal = [] - hasexc = [] + + __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", + "hasjabs", "hasfree", "haslocal", "hasexc"]) _nb_ops = [ ("NB_ADD", "+"), diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 2e841e6097aa25..5b0560e6b21a99 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -84,13 +84,7 @@ def main(opcode_py, opcode = get_python_module_dict(opcode_py) opmap = opcode['opmap'] opname = opcode['opname'] - hasarg = opcode['hasarg'] - hasconst = opcode['hasconst'] - hasjrel = opcode['hasjrel'] - hasjabs = opcode['hasjabs'] is_pseudo = opcode['is_pseudo'] - _pseudo_ops = opcode['_pseudo_ops'] - ENABLE_SPECIALIZATION = opcode["ENABLE_SPECIALIZATION"] MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] From d2d355eae2ae535d0f8fc18ca5265a7f6e3792b2 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 17 Jul 2023 17:26:51 +0100 Subject: [PATCH 10/26] hascompare in old versions as well --- Lib/opcode.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index d7d1a3fb56cfd8..47062347041f85 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,7 +4,8 @@ operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "opname", "opmap", "stack_effect", "HAVE_ARGUMENT", "EXTENDED_ARG"] +__all__ = ["cmp_op", "opname", "opmap", "stack_effect", "hascompare", + "HAVE_ARGUMENT", "EXTENDED_ARG"] import _opcode from _opcode import stack_effect @@ -125,7 +126,6 @@ def pseudo_op(name, op, real_ops): def_op('BUILD_MAP', 105) # Number of dict entries def_op('LOAD_ATTR', 106) # Index in name list def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) def_op('IMPORT_NAME', 108) # Index in name list def_op('IMPORT_FROM', 109) # Index in name list def_op('JUMP_FORWARD', 110) # Number of words to skip @@ -256,6 +256,8 @@ def pseudo_op(name, op, real_ops): __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"]) +hascompare = [opmap["COMPARE_OP"]] # for backwards compatibility + _nb_ops = [ ("NB_ADD", "+"), ("NB_AND", "&"), From 27eb2ccc28ba489c3be067781d34b81c18df7f44 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 17 Jul 2023 17:27:29 +0100 Subject: [PATCH 11/26] remove redundant init --- Lib/opcode.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 47062347041f85..32722ed7309d7f 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -17,8 +17,6 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') -hascompare = [] - ENABLE_SPECIALIZATION = True From 998024a771e4fac6d2b196246432996f701042b6 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 17 Jul 2023 16:46:04 +0000 Subject: [PATCH 12/26] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst diff --git a/Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst b/Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst new file mode 100644 index 00000000000000..d82eb987c83e96 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-17-16-46-00.gh-issue-105481.fek_Nn.rst @@ -0,0 +1 @@ +The various opcode lists in the :mod:`dis` module are now generated from bytecodes.c instead of explicitly constructed in opcode.py. From ce76a60a60b8d6be39dba4983c8e57d6ad5b2e90 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 17 Jul 2023 22:08:17 +0100 Subject: [PATCH 13/26] remove unnecessary comment --- Lib/opcode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 32722ed7309d7f..47f479941ca9b9 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -254,7 +254,7 @@ def pseudo_op(name, op, real_ops): __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"]) -hascompare = [opmap["COMPARE_OP"]] # for backwards compatibility +hascompare = [opmap["COMPARE_OP"]] _nb_ops = [ ("NB_ADD", "+"), From 4194a127c7ab2d469b1aa2280fde1ef0940d8f71 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 17 Jul 2023 23:23:55 +0100 Subject: [PATCH 14/26] make the tests more precise --- Lib/test/test__opcode.py | 135 +++++++++++++++++++++++---------------- 1 file changed, 81 insertions(+), 54 deletions(-) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index ae468be6eeca4f..ebe9d47d03bcff 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -7,16 +7,70 @@ from _opcode import stack_effect -class OpcodeTests(unittest.TestCase): - - def check_bool_function_result(self, func, ops, expected): - for op in ops: - if isinstance(op, str): - op = dis.opmap[op] - with self.subTest(opcode=op, func=func): - self.assertIsInstance(func(op), bool) - self.assertEqual(func(op), expected) - +EXPECTED_OPLISTS = { + 'HAS_ARG': [ + 'BINARY_OP', 'BUILD_CONST_KEY_MAP', 'BUILD_LIST', 'BUILD_MAP', + 'BUILD_SET', 'BUILD_SLICE', 'BUILD_STRING', 'BUILD_TUPLE', 'CALL', + 'CALL_FUNCTION_EX', 'CALL_INTRINSIC_1', 'CALL_INTRINSIC_2', 'COMPARE_OP', + 'CONTAINS_OP', 'CONVERT_VALUE', 'COPY', 'COPY_FREE_VARS', 'DELETE_ATTR', + 'DELETE_DEREF', 'DELETE_FAST', 'DELETE_GLOBAL', 'DELETE_NAME', + 'DICT_MERGE', 'DICT_UPDATE', 'ENTER_EXECUTOR', 'EXTENDED_ARG', + 'FOR_ITER', 'GET_AWAITABLE', 'IMPORT_FROM', 'IMPORT_NAME', + 'INSTRUMENTED_CALL', 'INSTRUMENTED_FOR_ITER', + 'INSTRUMENTED_JUMP_BACKWARD', 'INSTRUMENTED_JUMP_FORWARD', + 'INSTRUMENTED_LOAD_SUPER_ATTR', 'INSTRUMENTED_POP_JUMP_IF_FALSE', + 'INSTRUMENTED_POP_JUMP_IF_NONE', 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE', + 'INSTRUMENTED_POP_JUMP_IF_TRUE', 'INSTRUMENTED_RESUME', + 'INSTRUMENTED_RETURN_CONST', 'INSTRUMENTED_YIELD_VALUE', 'IS_OP', 'JUMP', + 'JUMP_BACKWARD', 'JUMP_BACKWARD_NO_INTERRUPT', 'JUMP_FORWARD', + 'JUMP_NO_INTERRUPT', 'KW_NAMES', 'LIST_APPEND', 'LIST_EXTEND', + 'LOAD_ATTR', 'LOAD_CLOSURE', 'LOAD_CONST', 'LOAD_DEREF', 'LOAD_FAST', + 'LOAD_FAST_AND_CLEAR', 'LOAD_FAST_CHECK', 'LOAD_FAST_LOAD_FAST', + 'LOAD_FROM_DICT_OR_DEREF', 'LOAD_FROM_DICT_OR_GLOBALS', 'LOAD_GLOBAL', + 'LOAD_METHOD', 'LOAD_NAME', 'LOAD_SUPER_ATTR', 'LOAD_SUPER_METHOD', + 'LOAD_ZERO_SUPER_ATTR', 'LOAD_ZERO_SUPER_METHOD', 'MAKE_CELL', 'MAP_ADD', + 'MATCH_CLASS', 'POP_JUMP_IF_FALSE', 'POP_JUMP_IF_NONE', + 'POP_JUMP_IF_NOT_NONE', 'POP_JUMP_IF_TRUE', 'RAISE_VARARGS', 'RERAISE', + 'RESUME', 'RETURN_CONST', 'SEND', 'SET_ADD', 'SET_FUNCTION_ATTRIBUTE', + 'SET_UPDATE', 'STORE_ATTR', 'STORE_DEREF', 'STORE_FAST', + 'STORE_FAST_LOAD_FAST', 'STORE_FAST_MAYBE_NULL', 'STORE_FAST_STORE_FAST', + 'STORE_GLOBAL', 'STORE_NAME', 'SWAP', 'UNPACK_EX', 'UNPACK_SEQUENCE', + 'YIELD_VALUE'], + + 'HAS_CONST': [ + 'LOAD_CONST', 'RETURN_CONST', 'KW_NAMES', 'INSTRUMENTED_RETURN_CONST'], + + 'HAS_NAME': [ + 'STORE_NAME', 'DELETE_NAME', 'STORE_ATTR', 'DELETE_ATTR', 'STORE_GLOBAL', + 'DELETE_GLOBAL', 'LOAD_NAME', 'LOAD_ATTR', 'IMPORT_NAME', 'IMPORT_FROM', + 'LOAD_GLOBAL', 'LOAD_SUPER_ATTR', 'LOAD_FROM_DICT_OR_GLOBALS', + 'LOAD_METHOD', 'LOAD_SUPER_METHOD', 'LOAD_ZERO_SUPER_METHOD', + 'LOAD_ZERO_SUPER_ATTR'], + + 'HAS_CONST': [ + 'LOAD_CONST', 'RETURN_CONST', 'KW_NAMES', 'INSTRUMENTED_RETURN_CONST'], + + 'HAS_JUMP': [ + 'FOR_ITER', 'JUMP_FORWARD', 'POP_JUMP_IF_FALSE', 'POP_JUMP_IF_TRUE', + 'SEND', 'POP_JUMP_IF_NOT_NONE', 'POP_JUMP_IF_NONE', + 'JUMP_BACKWARD_NO_INTERRUPT', 'JUMP_BACKWARD', 'ENTER_EXECUTOR', 'JUMP', + 'JUMP_NO_INTERRUPT'], + + 'HAS_FREE': [ + 'MAKE_CELL', 'LOAD_DEREF', 'STORE_DEREF', 'DELETE_DEREF', + 'LOAD_FROM_DICT_OR_DEREF'], + + 'HAS_LOCAL': [ + 'LOAD_FAST', 'STORE_FAST', 'DELETE_FAST', 'LOAD_FAST_CHECK', + 'LOAD_FAST_AND_CLEAR', 'LOAD_FAST_LOAD_FAST', 'STORE_FAST_LOAD_FAST', + 'STORE_FAST_STORE_FAST', 'STORE_FAST_MAYBE_NULL', 'LOAD_CLOSURE'], + + 'HAS_EXC': ['SETUP_FINALLY', 'SETUP_CLEANUP', 'SETUP_WITH'], + + 'HAS_COMPARE': ['COMPARE_OP'], +} + +class OpListTests(unittest.TestCase): def test_invalid_opcodes(self): invalid = [-100, -1, 255, 512, 513, 1000] self.check_bool_function_result(_opcode.is_valid, invalid, False) @@ -39,51 +93,24 @@ def test_is_valid(self): opcodes = [dis.opmap[opname] for opname in names] self.check_bool_function_result(_opcode.is_valid, opcodes, True) - def test_has_arg(self): - has_arg = ['SWAP', 'LOAD_FAST', 'INSTRUMENTED_POP_JUMP_IF_TRUE', 'JUMP'] - no_arg = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_arg, has_arg, True) - self.check_bool_function_result(_opcode.has_arg, no_arg, False) - - def test_has_const(self): - has_const = ['LOAD_CONST', 'RETURN_CONST', 'KW_NAMES'] - no_const = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_const, has_const, True) - self.check_bool_function_result(_opcode.has_const, no_const, False) - - def test_has_name(self): - has_name = ['STORE_NAME', 'DELETE_ATTR', 'STORE_GLOBAL', 'IMPORT_FROM', - 'LOAD_FROM_DICT_OR_GLOBALS'] - no_name = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_name, has_name, True) - self.check_bool_function_result(_opcode.has_name, no_name, False) - - def test_has_jump(self): - has_jump = ['FOR_ITER', 'JUMP_FORWARD', 'JUMP', 'POP_JUMP_IF_TRUE', 'SEND'] - no_jump = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_jump, has_jump, True) - self.check_bool_function_result(_opcode.has_jump, no_jump, False) - - def test_has_free(self): - has_free = ['MAKE_CELL', 'LOAD_DEREF', 'STORE_DEREF', 'DELETE_DEREF', - 'LOAD_FROM_DICT_OR_DEREF'] - no_free = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_free, has_free, True) - self.check_bool_function_result(_opcode.has_free, no_free, False) - - def test_has_local(self): - has_local = ['LOAD_FAST', 'LOAD_FAST_CHECK', 'LOAD_FAST_AND_CLEAR', - 'STORE_FAST_MAYBE_NULL', 'LOAD_CLOSURE'] - no_local = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_local, has_local, True) - self.check_bool_function_result(_opcode.has_local, no_local, False) - - def test_has_exc(self): - has_exc = ['SETUP_FINALLY', 'SETUP_WITH', 'SETUP_CLEANUP'] - no_exc = ['DELETE_DEREF', 'POP_TOP', 'NOP', 'CACHE'] - self.check_bool_function_result(_opcode.has_exc, has_exc, True) - self.check_bool_function_result(_opcode.has_exc, no_exc, False) + def test_oplists(self): + def check_function(self, func, expected): + for op in [-10, 520]: + with self.subTest(opcode=op, func=func): + res = func(op) + self.assertIsInstance(res, bool) + self.assertEqual(res, op in expected) + + check_function(self, _opcode.has_arg, EXPECTED_OPLISTS['HAS_ARG']) + check_function(self, _opcode.has_const, EXPECTED_OPLISTS['HAS_CONST']) + check_function(self, _opcode.has_name, EXPECTED_OPLISTS['HAS_NAME']) + check_function(self, _opcode.has_jump, EXPECTED_OPLISTS['HAS_JUMP']) + check_function(self, _opcode.has_free, EXPECTED_OPLISTS['HAS_FREE']) + check_function(self, _opcode.has_local, EXPECTED_OPLISTS['HAS_LOCAL']) + check_function(self, _opcode.has_exc, EXPECTED_OPLISTS['HAS_EXC']) + +class OpListTests(unittest.TestCase): def test_stack_effect(self): self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1) From 6b8c46bea1007f5cca10d32a2a78e58d3ded000c Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 17 Jul 2023 23:40:54 +0100 Subject: [PATCH 15/26] sort the lists --- Lib/test/test__opcode.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index ebe9d47d03bcff..53c3cbdd5e641b 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -38,34 +38,35 @@ 'YIELD_VALUE'], 'HAS_CONST': [ - 'LOAD_CONST', 'RETURN_CONST', 'KW_NAMES', 'INSTRUMENTED_RETURN_CONST'], + 'INSTRUMENTED_RETURN_CONST', 'KW_NAMES', 'LOAD_CONST', 'RETURN_CONST'], 'HAS_NAME': [ - 'STORE_NAME', 'DELETE_NAME', 'STORE_ATTR', 'DELETE_ATTR', 'STORE_GLOBAL', - 'DELETE_GLOBAL', 'LOAD_NAME', 'LOAD_ATTR', 'IMPORT_NAME', 'IMPORT_FROM', - 'LOAD_GLOBAL', 'LOAD_SUPER_ATTR', 'LOAD_FROM_DICT_OR_GLOBALS', - 'LOAD_METHOD', 'LOAD_SUPER_METHOD', 'LOAD_ZERO_SUPER_METHOD', - 'LOAD_ZERO_SUPER_ATTR'], + 'DELETE_ATTR', 'DELETE_GLOBAL', 'DELETE_NAME', 'IMPORT_FROM', + 'IMPORT_NAME', 'LOAD_ATTR', 'LOAD_FROM_DICT_OR_GLOBALS', 'LOAD_GLOBAL', + 'LOAD_METHOD', 'LOAD_NAME', 'LOAD_SUPER_ATTR', 'LOAD_SUPER_METHOD', + 'LOAD_ZERO_SUPER_ATTR', 'LOAD_ZERO_SUPER_METHOD', 'STORE_ATTR', + 'STORE_GLOBAL', 'STORE_NAME'], 'HAS_CONST': [ - 'LOAD_CONST', 'RETURN_CONST', 'KW_NAMES', 'INSTRUMENTED_RETURN_CONST'], + 'INSTRUMENTED_RETURN_CONST', 'KW_NAMES', 'LOAD_CONST', 'RETURN_CONST'], 'HAS_JUMP': [ - 'FOR_ITER', 'JUMP_FORWARD', 'POP_JUMP_IF_FALSE', 'POP_JUMP_IF_TRUE', - 'SEND', 'POP_JUMP_IF_NOT_NONE', 'POP_JUMP_IF_NONE', - 'JUMP_BACKWARD_NO_INTERRUPT', 'JUMP_BACKWARD', 'ENTER_EXECUTOR', 'JUMP', - 'JUMP_NO_INTERRUPT'], + 'ENTER_EXECUTOR', 'FOR_ITER', 'JUMP', 'JUMP_BACKWARD', + 'JUMP_BACKWARD_NO_INTERRUPT', 'JUMP_FORWARD', 'JUMP_NO_INTERRUPT', + 'POP_JUMP_IF_FALSE', 'POP_JUMP_IF_NONE', 'POP_JUMP_IF_NOT_NONE', + 'POP_JUMP_IF_TRUE', 'SEND'], 'HAS_FREE': [ - 'MAKE_CELL', 'LOAD_DEREF', 'STORE_DEREF', 'DELETE_DEREF', - 'LOAD_FROM_DICT_OR_DEREF'], + 'DELETE_DEREF', 'LOAD_DEREF', 'LOAD_FROM_DICT_OR_DEREF', 'MAKE_CELL', + 'STORE_DEREF'], 'HAS_LOCAL': [ - 'LOAD_FAST', 'STORE_FAST', 'DELETE_FAST', 'LOAD_FAST_CHECK', - 'LOAD_FAST_AND_CLEAR', 'LOAD_FAST_LOAD_FAST', 'STORE_FAST_LOAD_FAST', - 'STORE_FAST_STORE_FAST', 'STORE_FAST_MAYBE_NULL', 'LOAD_CLOSURE'], + 'DELETE_FAST', 'LOAD_CLOSURE', 'LOAD_FAST', 'LOAD_FAST_AND_CLEAR', + 'LOAD_FAST_CHECK', 'LOAD_FAST_LOAD_FAST', 'STORE_FAST', + 'STORE_FAST_LOAD_FAST', 'STORE_FAST_MAYBE_NULL', + 'STORE_FAST_STORE_FAST'], - 'HAS_EXC': ['SETUP_FINALLY', 'SETUP_CLEANUP', 'SETUP_WITH'], + 'HAS_EXC': ['SETUP_CLEANUP', 'SETUP_FINALLY', 'SETUP_WITH'], 'HAS_COMPARE': ['COMPARE_OP'], } From e15f7d5163daa84a5dfc07663dd12d692d67cb9d Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 10:56:06 +0100 Subject: [PATCH 16/26] address some of the code review comments --- Lib/opcode.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 47f479941ca9b9..797ee7d650d385 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,6 +4,8 @@ operate on bytecodes (e.g. peephole optimizers). """ + +# Note that __all__ is further extended below __all__ = ["cmp_op", "opname", "opmap", "stack_effect", "hascompare", "HAVE_ARGUMENT", "EXTENDED_ARG"] @@ -25,8 +27,8 @@ def is_pseudo(op): opmap = {} -## pseudo opcodes (used in the compiler) mapped to the values -## they can become in the actual code. +# pseudo opcodes (used in the compiler) mapped to the values +# they can become in the actual code. _pseudo_ops = {} def def_op(name, op): @@ -241,6 +243,7 @@ def pseudo_op(name, op, real_ops): # The build uses older versions of Python which do not have _opcode.has_* functions if sys.version_info[:2] >= (3, 13): + # These lists are documented as part of the dis module's API hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] hasconst = [op for op in opmap.values() if _opcode.has_const(op)] hasname = [op for op in opmap.values() if _opcode.has_name(op)] From 21fe0a4432ce14247af4ca6e5a2a5a858dc15fd8 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 11:11:17 +0100 Subject: [PATCH 17/26] remove hard-coded lists from the tests --- Lib/opcode.py | 2 +- Lib/test/test__opcode.py | 78 ++++------------------------------------ 2 files changed, 8 insertions(+), 72 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 797ee7d650d385..08dfd2674dca78 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -159,7 +159,7 @@ def pseudo_op(name, op, real_ops): def_op('CALL_FUNCTION_EX', 142) # Flags def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 +EXTENDED_ARG = opmap['EXTENDED_ARG'] def_op('LIST_APPEND', 145) def_op('SET_ADD', 146) def_op('MAP_ADD', 147) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 53c3cbdd5e641b..b3a9bcbe160453 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -7,70 +7,6 @@ from _opcode import stack_effect -EXPECTED_OPLISTS = { - 'HAS_ARG': [ - 'BINARY_OP', 'BUILD_CONST_KEY_MAP', 'BUILD_LIST', 'BUILD_MAP', - 'BUILD_SET', 'BUILD_SLICE', 'BUILD_STRING', 'BUILD_TUPLE', 'CALL', - 'CALL_FUNCTION_EX', 'CALL_INTRINSIC_1', 'CALL_INTRINSIC_2', 'COMPARE_OP', - 'CONTAINS_OP', 'CONVERT_VALUE', 'COPY', 'COPY_FREE_VARS', 'DELETE_ATTR', - 'DELETE_DEREF', 'DELETE_FAST', 'DELETE_GLOBAL', 'DELETE_NAME', - 'DICT_MERGE', 'DICT_UPDATE', 'ENTER_EXECUTOR', 'EXTENDED_ARG', - 'FOR_ITER', 'GET_AWAITABLE', 'IMPORT_FROM', 'IMPORT_NAME', - 'INSTRUMENTED_CALL', 'INSTRUMENTED_FOR_ITER', - 'INSTRUMENTED_JUMP_BACKWARD', 'INSTRUMENTED_JUMP_FORWARD', - 'INSTRUMENTED_LOAD_SUPER_ATTR', 'INSTRUMENTED_POP_JUMP_IF_FALSE', - 'INSTRUMENTED_POP_JUMP_IF_NONE', 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE', - 'INSTRUMENTED_POP_JUMP_IF_TRUE', 'INSTRUMENTED_RESUME', - 'INSTRUMENTED_RETURN_CONST', 'INSTRUMENTED_YIELD_VALUE', 'IS_OP', 'JUMP', - 'JUMP_BACKWARD', 'JUMP_BACKWARD_NO_INTERRUPT', 'JUMP_FORWARD', - 'JUMP_NO_INTERRUPT', 'KW_NAMES', 'LIST_APPEND', 'LIST_EXTEND', - 'LOAD_ATTR', 'LOAD_CLOSURE', 'LOAD_CONST', 'LOAD_DEREF', 'LOAD_FAST', - 'LOAD_FAST_AND_CLEAR', 'LOAD_FAST_CHECK', 'LOAD_FAST_LOAD_FAST', - 'LOAD_FROM_DICT_OR_DEREF', 'LOAD_FROM_DICT_OR_GLOBALS', 'LOAD_GLOBAL', - 'LOAD_METHOD', 'LOAD_NAME', 'LOAD_SUPER_ATTR', 'LOAD_SUPER_METHOD', - 'LOAD_ZERO_SUPER_ATTR', 'LOAD_ZERO_SUPER_METHOD', 'MAKE_CELL', 'MAP_ADD', - 'MATCH_CLASS', 'POP_JUMP_IF_FALSE', 'POP_JUMP_IF_NONE', - 'POP_JUMP_IF_NOT_NONE', 'POP_JUMP_IF_TRUE', 'RAISE_VARARGS', 'RERAISE', - 'RESUME', 'RETURN_CONST', 'SEND', 'SET_ADD', 'SET_FUNCTION_ATTRIBUTE', - 'SET_UPDATE', 'STORE_ATTR', 'STORE_DEREF', 'STORE_FAST', - 'STORE_FAST_LOAD_FAST', 'STORE_FAST_MAYBE_NULL', 'STORE_FAST_STORE_FAST', - 'STORE_GLOBAL', 'STORE_NAME', 'SWAP', 'UNPACK_EX', 'UNPACK_SEQUENCE', - 'YIELD_VALUE'], - - 'HAS_CONST': [ - 'INSTRUMENTED_RETURN_CONST', 'KW_NAMES', 'LOAD_CONST', 'RETURN_CONST'], - - 'HAS_NAME': [ - 'DELETE_ATTR', 'DELETE_GLOBAL', 'DELETE_NAME', 'IMPORT_FROM', - 'IMPORT_NAME', 'LOAD_ATTR', 'LOAD_FROM_DICT_OR_GLOBALS', 'LOAD_GLOBAL', - 'LOAD_METHOD', 'LOAD_NAME', 'LOAD_SUPER_ATTR', 'LOAD_SUPER_METHOD', - 'LOAD_ZERO_SUPER_ATTR', 'LOAD_ZERO_SUPER_METHOD', 'STORE_ATTR', - 'STORE_GLOBAL', 'STORE_NAME'], - - 'HAS_CONST': [ - 'INSTRUMENTED_RETURN_CONST', 'KW_NAMES', 'LOAD_CONST', 'RETURN_CONST'], - - 'HAS_JUMP': [ - 'ENTER_EXECUTOR', 'FOR_ITER', 'JUMP', 'JUMP_BACKWARD', - 'JUMP_BACKWARD_NO_INTERRUPT', 'JUMP_FORWARD', 'JUMP_NO_INTERRUPT', - 'POP_JUMP_IF_FALSE', 'POP_JUMP_IF_NONE', 'POP_JUMP_IF_NOT_NONE', - 'POP_JUMP_IF_TRUE', 'SEND'], - - 'HAS_FREE': [ - 'DELETE_DEREF', 'LOAD_DEREF', 'LOAD_FROM_DICT_OR_DEREF', 'MAKE_CELL', - 'STORE_DEREF'], - - 'HAS_LOCAL': [ - 'DELETE_FAST', 'LOAD_CLOSURE', 'LOAD_FAST', 'LOAD_FAST_AND_CLEAR', - 'LOAD_FAST_CHECK', 'LOAD_FAST_LOAD_FAST', 'STORE_FAST', - 'STORE_FAST_LOAD_FAST', 'STORE_FAST_MAYBE_NULL', - 'STORE_FAST_STORE_FAST'], - - 'HAS_EXC': ['SETUP_CLEANUP', 'SETUP_FINALLY', 'SETUP_WITH'], - - 'HAS_COMPARE': ['COMPARE_OP'], -} - class OpListTests(unittest.TestCase): def test_invalid_opcodes(self): invalid = [-100, -1, 255, 512, 513, 1000] @@ -102,13 +38,13 @@ def check_function(self, func, expected): self.assertIsInstance(res, bool) self.assertEqual(res, op in expected) - check_function(self, _opcode.has_arg, EXPECTED_OPLISTS['HAS_ARG']) - check_function(self, _opcode.has_const, EXPECTED_OPLISTS['HAS_CONST']) - check_function(self, _opcode.has_name, EXPECTED_OPLISTS['HAS_NAME']) - check_function(self, _opcode.has_jump, EXPECTED_OPLISTS['HAS_JUMP']) - check_function(self, _opcode.has_free, EXPECTED_OPLISTS['HAS_FREE']) - check_function(self, _opcode.has_local, EXPECTED_OPLISTS['HAS_LOCAL']) - check_function(self, _opcode.has_exc, EXPECTED_OPLISTS['HAS_EXC']) + check_function(self, _opcode.has_arg, dis.hasarg) + check_function(self, _opcode.has_const, dis.hasconst) + check_function(self, _opcode.has_name, dis.hasname) + check_function(self, _opcode.has_jump, dis.hasjump) + check_function(self, _opcode.has_free, dis.hasfree) + check_function(self, _opcode.has_local, dis.haslocal) + check_function(self, _opcode.has_exc, dis.hasexc) class OpListTests(unittest.TestCase): From 75795db0f4f59e0b529a3da3100d76d80cfdb623 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 11:12:12 +0100 Subject: [PATCH 18/26] remove opcode metadata file from generated files list so diff are visible --- .gitattributes | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 2616da74b48c0f..5d5558da711b17 100644 --- a/.gitattributes +++ b/.gitattributes @@ -87,7 +87,6 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Include/internal/pycore_opcode_metadata.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated From 1a0a2b257402689413bf287600cef4ab00c63a0a Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 11:54:14 +0100 Subject: [PATCH 19/26] import _opcode directly into dis, rather than via opcode --- Lib/dis.py | 18 +++++++++++++++++- Lib/opcode.py | 25 +------------------------ 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index f7a31f2f96b99b..db3730122f6b29 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -4,6 +4,8 @@ import types import collections import io +import _opcode +from _opcode import stack_effect from opcode import * from opcode import ( @@ -17,7 +19,9 @@ _specialized_instructions, ) -__all__ = ["code_info", "dis", "disassemble", "distb", "disco", +__all__ = ["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", + "hasjabs", "hasfree", "haslocal", "hasexc", "hascompare", + "code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all del _opcodes_all @@ -25,6 +29,18 @@ _have_code = (types.MethodType, types.FunctionType, types.CodeType, classmethod, staticmethod, type) +# These lists are documented as part of the dis module's API +hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] +hasconst = [op for op in opmap.values() if _opcode.has_const(op)] +hasname = [op for op in opmap.values() if _opcode.has_name(op)] +hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] +hasjrel = hasjump # for backward compatibility +hasjabs = [] +hasfree = [op for op in opmap.values() if _opcode.has_free(op)] +haslocal = [op for op in opmap.values() if _opcode.has_local(op)] +hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] +hascompare = [opmap["COMPARE_OP"]] + CONVERT_VALUE = opmap['CONVERT_VALUE'] SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] diff --git a/Lib/opcode.py b/Lib/opcode.py index 08dfd2674dca78..c2f8d8c4904054 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -5,12 +5,7 @@ """ -# Note that __all__ is further extended below -__all__ = ["cmp_op", "opname", "opmap", "stack_effect", "hascompare", - "HAVE_ARGUMENT", "EXTENDED_ARG"] - -import _opcode -from _opcode import stack_effect +__all__ = ["cmp_op", "opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"] import sys # The build uses older versions of Python which do not have _opcode_metadata @@ -241,24 +236,6 @@ def pseudo_op(name, op, real_ops): for op, i in opmap.items(): opname[i] = op -# The build uses older versions of Python which do not have _opcode.has_* functions -if sys.version_info[:2] >= (3, 13): - # These lists are documented as part of the dis module's API - hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] - hasconst = [op for op in opmap.values() if _opcode.has_const(op)] - hasname = [op for op in opmap.values() if _opcode.has_name(op)] - hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] - hasjrel = hasjump # for backward compatibility - hasjabs = [] - hasfree = [op for op in opmap.values() if _opcode.has_free(op)] - haslocal = [op for op in opmap.values() if _opcode.has_local(op)] - hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] - - __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", - "hasjabs", "hasfree", "haslocal", "hasexc"]) - -hascompare = [opmap["COMPARE_OP"]] - _nb_ops = [ ("NB_ADD", "+"), ("NB_AND", "&"), From f5304696e7f7eee6019452dd51a3554e3c293387 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 12:03:03 +0100 Subject: [PATCH 20/26] add stack effect to dis.__all__ --- Lib/dis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index db3730122f6b29..7c519a297bf1cf 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -21,8 +21,8 @@ __all__ = ["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc", "hascompare", - "code_info", "dis", "disassemble", "distb", "disco", - "findlinestarts", "findlabels", "show_code", + "stack_effect", "code_info", "dis", "disassemble", "distb", + "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all del _opcodes_all From d525c53b2792cab47cda4a7d834f75557c23fead Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 12:29:56 +0100 Subject: [PATCH 21/26] import _specializations, _specialized_instructions directly from _opcode_metadata rather than via opcode --- Lib/dis.py | 4 ++-- Lib/opcode.py | 5 ----- Lib/test/test__opcode.py | 3 ++- Lib/test/test_dis.py | 5 +++-- Lib/test/test_embed.py | 4 ++-- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index 7c519a297bf1cf..a46e42ca856048 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -15,10 +15,10 @@ _nb_ops, _intrinsic_1_descs, _intrinsic_2_descs, - _specializations, - _specialized_instructions, ) +from _opcode_metadata import _specializations, _specialized_instructions + __all__ = ["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc", "hascompare", "stack_effect", "code_info", "dis", "disassemble", "distb", diff --git a/Lib/opcode.py b/Lib/opcode.py index c2f8d8c4904054..179b6e414839b6 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -7,11 +7,6 @@ __all__ = ["cmp_op", "opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"] -import sys -# The build uses older versions of Python which do not have _opcode_metadata -if sys.version_info[:2] >= (3, 13): - from _opcode_metadata import _specializations, _specialized_instructions - cmp_op = ('<', '<=', '==', '!=', '>', '>=') diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index b3a9bcbe160453..d77c8128abe1c2 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -2,6 +2,7 @@ from test.support.import_helper import import_module import unittest import opcode +import _opcode_metadata _opcode = import_module("_opcode") from _opcode import stack_effect @@ -105,7 +106,7 @@ def test_specialization_stats(self): stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] specialized_opcodes = [ op.lower() - for op in opcode._specializations + for op in _opcode_metadata._specializations if opcode._inline_cache_entries[opcode.opmap[op]] ] self.assertIn('load_attr', specialized_opcodes) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 8597b8f14ac058..54f8d11d559846 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -12,6 +12,7 @@ from test.support.bytecode_helper import BytecodeTestCase import opcode +import _opcode_metadata def get_tb(): @@ -1939,12 +1940,12 @@ def test_baseopname_and_baseopcode(self): self.assertEqual(code, baseopcode) # Specialized instructions - for name in opcode._specialized_instructions: + for name in _opcode_metadata._specialized_instructions: instruction = Instruction(opname=name, opcode=dis._all_opmap[name], arg=None, argval=None, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None) baseopname = instruction.baseopname baseopcode = instruction.baseopcode - self.assertIn(name, opcode._specializations[baseopname]) + self.assertIn(name, _opcode_metadata._specializations[baseopname]) self.assertEqual(opcode.opmap[baseopname], baseopcode) def test_jump_target(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 582392ecddcb91..d2adda1ee9f272 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -354,14 +354,14 @@ def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): code = textwrap.dedent("""\ import dis import importlib._bootstrap - import opcode + import _opcode_metadata import test.test_dis def is_specialized(f): for instruction in dis.get_instructions(f, adaptive=True): opname = instruction.opname if ( - opname in opcode._specialized_instructions + opname in _opcode_metadata._specialized_instructions # Exclude superinstructions: and "__" not in opname ): From 5e1c4a49551db62d59792eb6e27feed2a122a38a Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 13:03:50 +0100 Subject: [PATCH 22/26] update Tools/scripts/summarize_stats.py to use _opcode_metadata instead of opcode for _specialized_instructions --- Tools/scripts/summarize_stats.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 9c881897c2de1d..833e5423e04304 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -7,6 +7,7 @@ import json import os.path import opcode +import _opcode_metadata from datetime import date import itertools import sys @@ -17,7 +18,7 @@ DEFAULT_DIR = "/tmp/py_stats/" #Create list of all instruction names -specialized = iter(opcode._specialized_instructions) +specialized = iter(_opcode_metadata._specialized_instructions) opname = ["<0>"] for name in opcode.opname[1:]: if name.startswith("<"): @@ -244,7 +245,7 @@ def categorized_counts(opcode_stats): specialized = 0 not_specialized = 0 specialized_instructions = { - op for op in opcode._specialized_instructions + op for op in _opcode_metadata._specialized_instructions if "__" not in op} for i, opcode_stat in enumerate(opcode_stats): if "execution_count" not in opcode_stat: From 607726316c73eed923938a1ec869e1536b17552d Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 18:28:31 +0100 Subject: [PATCH 23/26] Revert "update Tools/scripts/summarize_stats.py to use _opcode_metadata instead of opcode for _specialized_instructions" This reverts commit 5e1c4a49551db62d59792eb6e27feed2a122a38a. --- Tools/scripts/summarize_stats.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 833e5423e04304..9c881897c2de1d 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -7,7 +7,6 @@ import json import os.path import opcode -import _opcode_metadata from datetime import date import itertools import sys @@ -18,7 +17,7 @@ DEFAULT_DIR = "/tmp/py_stats/" #Create list of all instruction names -specialized = iter(_opcode_metadata._specialized_instructions) +specialized = iter(opcode._specialized_instructions) opname = ["<0>"] for name in opcode.opname[1:]: if name.startswith("<"): @@ -245,7 +244,7 @@ def categorized_counts(opcode_stats): specialized = 0 not_specialized = 0 specialized_instructions = { - op for op in _opcode_metadata._specialized_instructions + op for op in opcode._specialized_instructions if "__" not in op} for i, opcode_stat in enumerate(opcode_stats): if "execution_count" not in opcode_stat: From 98ba11f545483d3ec3984531ef37849d64877e31 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 18:28:53 +0100 Subject: [PATCH 24/26] Revert "import _specializations, _specialized_instructions directly from _opcode_metadata rather than via opcode" This reverts commit d525c53b2792cab47cda4a7d834f75557c23fead. --- Lib/dis.py | 4 ++-- Lib/opcode.py | 5 +++++ Lib/test/test__opcode.py | 3 +-- Lib/test/test_dis.py | 5 ++--- Lib/test/test_embed.py | 4 ++-- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index a46e42ca856048..7c519a297bf1cf 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -15,10 +15,10 @@ _nb_ops, _intrinsic_1_descs, _intrinsic_2_descs, + _specializations, + _specialized_instructions, ) -from _opcode_metadata import _specializations, _specialized_instructions - __all__ = ["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc", "hascompare", "stack_effect", "code_info", "dis", "disassemble", "distb", diff --git a/Lib/opcode.py b/Lib/opcode.py index 179b6e414839b6..c2f8d8c4904054 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -7,6 +7,11 @@ __all__ = ["cmp_op", "opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"] +import sys +# The build uses older versions of Python which do not have _opcode_metadata +if sys.version_info[:2] >= (3, 13): + from _opcode_metadata import _specializations, _specialized_instructions + cmp_op = ('<', '<=', '==', '!=', '>', '>=') diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index d77c8128abe1c2..b3a9bcbe160453 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -2,7 +2,6 @@ from test.support.import_helper import import_module import unittest import opcode -import _opcode_metadata _opcode = import_module("_opcode") from _opcode import stack_effect @@ -106,7 +105,7 @@ def test_specialization_stats(self): stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] specialized_opcodes = [ op.lower() - for op in _opcode_metadata._specializations + for op in opcode._specializations if opcode._inline_cache_entries[opcode.opmap[op]] ] self.assertIn('load_attr', specialized_opcodes) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 54f8d11d559846..8597b8f14ac058 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -12,7 +12,6 @@ from test.support.bytecode_helper import BytecodeTestCase import opcode -import _opcode_metadata def get_tb(): @@ -1940,12 +1939,12 @@ def test_baseopname_and_baseopcode(self): self.assertEqual(code, baseopcode) # Specialized instructions - for name in _opcode_metadata._specialized_instructions: + for name in opcode._specialized_instructions: instruction = Instruction(opname=name, opcode=dis._all_opmap[name], arg=None, argval=None, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None) baseopname = instruction.baseopname baseopcode = instruction.baseopcode - self.assertIn(name, _opcode_metadata._specializations[baseopname]) + self.assertIn(name, opcode._specializations[baseopname]) self.assertEqual(opcode.opmap[baseopname], baseopcode) def test_jump_target(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d2adda1ee9f272..582392ecddcb91 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -354,14 +354,14 @@ def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): code = textwrap.dedent("""\ import dis import importlib._bootstrap - import _opcode_metadata + import opcode import test.test_dis def is_specialized(f): for instruction in dis.get_instructions(f, adaptive=True): opname = instruction.opname if ( - opname in _opcode_metadata._specialized_instructions + opname in opcode._specialized_instructions # Exclude superinstructions: and "__" not in opname ): From cc8e5e194fcdd9b31aa335eb4a3fdc556919a2f2 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 18:29:31 +0100 Subject: [PATCH 25/26] Revert "add stack effect to dis.__all__" This reverts commit f5304696e7f7eee6019452dd51a3554e3c293387. --- Lib/dis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index 7c519a297bf1cf..db3730122f6b29 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -21,8 +21,8 @@ __all__ = ["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc", "hascompare", - "stack_effect", "code_info", "dis", "disassemble", "distb", - "disco", "findlinestarts", "findlabels", "show_code", + "code_info", "dis", "disassemble", "distb", "disco", + "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all del _opcodes_all From 31dbfeced967ca8c171d170eab0e211cbf7df68d Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Jul 2023 18:30:01 +0100 Subject: [PATCH 26/26] Revert "import _opcode directly into dis, rather than via opcode" This reverts commit 1a0a2b257402689413bf287600cef4ab00c63a0a. --- Lib/dis.py | 18 +----------------- Lib/opcode.py | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index db3730122f6b29..f7a31f2f96b99b 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -4,8 +4,6 @@ import types import collections import io -import _opcode -from _opcode import stack_effect from opcode import * from opcode import ( @@ -19,9 +17,7 @@ _specialized_instructions, ) -__all__ = ["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", - "hasjabs", "hasfree", "haslocal", "hasexc", "hascompare", - "code_info", "dis", "disassemble", "distb", "disco", +__all__ = ["code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all del _opcodes_all @@ -29,18 +25,6 @@ _have_code = (types.MethodType, types.FunctionType, types.CodeType, classmethod, staticmethod, type) -# These lists are documented as part of the dis module's API -hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] -hasconst = [op for op in opmap.values() if _opcode.has_const(op)] -hasname = [op for op in opmap.values() if _opcode.has_name(op)] -hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] -hasjrel = hasjump # for backward compatibility -hasjabs = [] -hasfree = [op for op in opmap.values() if _opcode.has_free(op)] -haslocal = [op for op in opmap.values() if _opcode.has_local(op)] -hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] -hascompare = [opmap["COMPARE_OP"]] - CONVERT_VALUE = opmap['CONVERT_VALUE'] SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] diff --git a/Lib/opcode.py b/Lib/opcode.py index c2f8d8c4904054..08dfd2674dca78 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -5,7 +5,12 @@ """ -__all__ = ["cmp_op", "opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"] +# Note that __all__ is further extended below +__all__ = ["cmp_op", "opname", "opmap", "stack_effect", "hascompare", + "HAVE_ARGUMENT", "EXTENDED_ARG"] + +import _opcode +from _opcode import stack_effect import sys # The build uses older versions of Python which do not have _opcode_metadata @@ -236,6 +241,24 @@ def pseudo_op(name, op, real_ops): for op, i in opmap.items(): opname[i] = op +# The build uses older versions of Python which do not have _opcode.has_* functions +if sys.version_info[:2] >= (3, 13): + # These lists are documented as part of the dis module's API + hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] + hasconst = [op for op in opmap.values() if _opcode.has_const(op)] + hasname = [op for op in opmap.values() if _opcode.has_name(op)] + hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] + hasjrel = hasjump # for backward compatibility + hasjabs = [] + hasfree = [op for op in opmap.values() if _opcode.has_free(op)] + haslocal = [op for op in opmap.values() if _opcode.has_local(op)] + hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] + + __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", + "hasjabs", "hasfree", "haslocal", "hasexc"]) + +hascompare = [opmap["COMPARE_OP"]] + _nb_ops = [ ("NB_ADD", "+"), ("NB_AND", "&"),