From 1975b4cafc32f43f1c49fc35cc75b01df7b11e3d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 10 Jan 2024 10:44:17 +0000 Subject: [PATCH 01/40] Cold exits: Work in progress. --- Include/internal/pycore_optimizer.h | 6 +++++ Include/internal/pycore_uop_ids.h | 3 ++- Include/internal/pycore_uop_metadata.h | 2 ++ Python/bytecodes.c | 22 +++++++++++++++++ Python/executor_cases.c.h | 11 +++++++++ Python/optimizer.c | 34 ++++++++++++++++++++++++++ 6 files changed, 77 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index b052460b44b791..1f88318954da71 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -19,6 +19,12 @@ extern PyTypeObject _PyDefaultOptimizer_Type; extern PyTypeObject _PyUOpExecutor_Type; extern PyTypeObject _PyUOpOptimizer_Type; +typedef struct _exit_data { + uint32_t target; + int16_t hotness; + _PyExecutorObject *exit; +} _PyExitData; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 4a9a00ba352d33..89019dda59bd32 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -231,7 +231,8 @@ extern "C" { #define _SAVE_RETURN_OFFSET 378 #define _INSERT 379 #define _CHECK_VALIDITY 380 -#define MAX_UOP_ID 380 +#define _COLD_EXIT 381 +#define MAX_UOP_ID 381 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 300bd3baa7b377..5a44bfb683f6a2 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -203,6 +203,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_EXIT_TRACE] = HAS_DEOPT_FLAG, [_INSERT] = HAS_ARG_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, + [_COLD_EXIT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { @@ -256,6 +257,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e1a6a256fbdf96..c97da815f91ee9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4066,6 +4066,28 @@ dummy_func( DEOPT_IF(!current_executor->base.vm_data.valid); } + op(_COLD_EXIT, (--)) { + TIER_TWO_ONLY + _PyExitData *exit = PyOptimizer_GetExit(current_executor, oparg); + Py_DECREF(current_executor); + exit->hotness++; + DEOPT_IF(exit->hotness < 0); + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; + _PyOptimizerObject *opt = interp->optimizer; + _PyExecutorObject *executor = NULL; + int optimized = opt->optimize(opt, code, target, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + ERROR_IF(optimized < 0, error); + if (optimized) { + exit->exit = executor; + Py_INCREF(executor); + GOTO_TIER_TWO(); + } + else { + exit->hotness = -10000; /* Choose a better number */ + } + } + // END BYTECODES // diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 14fb3a05a9f674..308c3ad28bcb0d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3397,4 +3397,15 @@ break; } + case _COLD_EXIT: { + oparg = CURRENT_OPARG(); + TIER_TWO_ONLY + _PyExitData *exit = PyOptimizer_GetExit(current_executor, oparg); + Py_DECREF(current_executor); + current_executor = exit->exit; + Py_INCREF(current_executor); + if (1) goto deoptimize; + break; + } + #undef TIER_TWO diff --git a/Python/optimizer.c b/Python/optimizer.c index f27af14d967cd3..0b296baf0b38d4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1139,3 +1139,37 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) } interp->executor_list_head = NULL; } + +/* Executor side exits */ + + +typedef struct _cold_exit { + PyObject_VAR_HEAD + _PyVMData vm_data; + union { + void *jit_code; // Will be a function pointer + _PyUOpInstruction *trace; + }; + _PyUOpInstruction cold; +} _PyColdExitObject; + +static int cold_exits_initialized = 0; +_PyColdExitObject COLD_EXITS[_Py_UOP_MAX_TRACE_LENGTH]; + +static void +initialize_cold_exits(void) { + if (cold_exits_initialized) { + return; + } + cold_exits_initialized = 1; + for (int i = 0; i < _Py_UOP_MAX_TRACE_LENGTH; i++) { + COLD_EXITS[i].ob_base.ob_base.ob_refcnt = _Py_IMMORTAL_REFCNT; + COLD_EXITS[i].ob_base.ob_base.ob_type = &_PyUOpExecutor_Type; + COLD_EXITS[i].ob_base.ob_size = 0; + COLD_EXITS[i].vm_data = (_PyVMData){ 0 }; + COLD_EXITS[i].cold.opcode = _COLD_EXIT; + COLD_EXITS[i].cold.oparg = i; + COLD_EXITS[i].trace = &COLD_EXITS[i].cold; + } +} + From f9aa23512bbd79086394887f0019517aa25b374f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 15:05:28 +0000 Subject: [PATCH 02/40] Optimize on side exits --- Include/cpython/optimizer.h | 9 +- Include/internal/pycore_opcode_metadata.h | 2 + Include/internal/pycore_optimizer.h | 6 - Include/internal/pycore_uop_metadata.h | 8 +- Python/bytecodes.c | 33 +++--- Python/ceval.c | 15 +++ Python/executor_cases.c.h | 33 +++--- Python/optimizer.c | 106 ++++++++++++------ Tools/cases_generator/analyzer.py | 8 +- Tools/cases_generator/generators_common.py | 3 + .../opcode_metadata_generator.py | 1 + Tools/cases_generator/tier2_generator.py | 16 +++ 12 files changed, 164 insertions(+), 76 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 96e829f8fbe97d..8437b35a473d38 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -36,10 +36,17 @@ typedef struct { uint64_t operand; // A cache entry } _PyUOpInstruction; +typedef struct _exit_data { + uint32_t target; + int16_t hotness; + struct _PyExecutorObject *executor; +} _PyExitData; + typedef struct _PyExecutorObject { PyObject_VAR_HEAD _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ - _PyUOpInstruction trace[1]; + _PyUOpInstruction *trace; + _PyExitData exits[1]; } _PyExecutorObject; typedef struct _PyOptimizerObject _PyOptimizerObject; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index a9d698da25a1db..c51edca7748311 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -909,6 +909,7 @@ enum InstructionFormat { #define HAS_DEOPT_FLAG (128) #define HAS_ERROR_FLAG (256) #define HAS_ESCAPES_FLAG (512) +#define HAS_EXIT_FLAG (1024) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -919,6 +920,7 @@ enum InstructionFormat { #define OPCODE_HAS_DEOPT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_DEOPT_FLAG)) #define OPCODE_HAS_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_FLAG)) #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) +#define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d06701917c4de0..31f30c673f207a 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -17,12 +17,6 @@ extern PyTypeObject _PyDefaultOptimizer_Type; extern PyTypeObject _PyUOpExecutor_Type; extern PyTypeObject _PyUOpOptimizer_Type; -typedef struct _exit_data { - uint32_t target; - int16_t hotness; - _PyExecutorObject *exit; -} _PyExitData; - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index eed75f4d1009ae..fcd5798999fb9b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -193,10 +193,10 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_COPY] = HAS_ARG_FLAG, [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SWAP] = HAS_ARG_FLAG, - [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG, - [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG, + [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, [_SET_IP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c1abf186c0cf6d..d730c2b62b4cf1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4013,21 +4013,21 @@ dummy_func( ///////// Tier-2 only opcodes ///////// op (_GUARD_IS_TRUE_POP, (flag -- )) { - DEOPT_IF(Py_IsFalse(flag)); + EXIT_IF(Py_IsFalse(flag)); assert(Py_IsTrue(flag)); } op (_GUARD_IS_FALSE_POP, (flag -- )) { - DEOPT_IF(Py_IsTrue(flag)); + EXIT_IF(Py_IsTrue(flag)); assert(Py_IsFalse(flag)); } op (_GUARD_IS_NONE_POP, (val -- )) { - DEOPT_IF(!Py_IsNone(val)); + EXIT_IF(!Py_IsNone(val)); } op (_GUARD_IS_NOT_NONE_POP, (val -- )) { - DEOPT_IF(Py_IsNone(val)); + EXIT_IF(Py_IsNone(val)); Py_DECREF(val); } @@ -4078,24 +4078,29 @@ dummy_func( op(_COLD_EXIT, (--)) { TIER_TWO_ONLY - _PyExitData *exit = PyOptimizer_GetExit(current_executor, oparg); + _PyExitData *exit = ¤t_executor->exits[oparg]; Py_DECREF(current_executor); exit->hotness++; DEOPT_IF(exit->hotness < 0); PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - _PyOptimizerObject *opt = interp->optimizer; _PyExecutorObject *executor = NULL; - int optimized = opt->optimize(opt, code, target, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); - ERROR_IF(optimized < 0, error); - if (optimized) { - exit->exit = executor; + if (target->op.code == ENTER_EXECUTOR) { + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; Py_INCREF(executor); - GOTO_TIER_TWO(); - } - else { - exit->hotness = -10000; /* Choose a better number */ + } else { + _PyOptimizerObject *opt = tstate->interp->optimizer; + int optimized = opt->optimize(opt, code, target, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + if (optimized <= 0) { + next_instr = target; + exit->hotness = -10000; /* Choose a better number */ + ERROR_IF(optimized < 0, error); + DISPATCH(); + } } + exit->executor = current_executor = executor; + Py_INCREF(current_executor); + GOTO_TIER_TWO(); } diff --git a/Python/ceval.c b/Python/ceval.c index 49388cd20377c0..2cccf64c40257c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1073,7 +1073,22 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int Py_DECREF(current_executor); DISPATCH(); +// Jump here from EXIT_IF() +side_exit: + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + UOP_STAT_INC(uopcode, miss); + _PyExitData *exit = ¤t_executor->exits[next_uop[-1].target]; + DPRINTF(2, "SIDE EXIT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit->target, + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); + Py_INCREF(exit->executor); + Py_DECREF(current_executor); + current_executor = exit->executor; + GOTO_TIER_TWO(); + } + #if defined(__GNUC__) # pragma GCC diagnostic pop #elif defined(_MSC_VER) /* MS_WINDOWS */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 81c12241e4991f..65de2b8bf44568 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3318,7 +3318,7 @@ case _GUARD_IS_TRUE_POP: { PyObject *flag; flag = stack_pointer[-1]; - if (Py_IsFalse(flag)) goto deoptimize; + if (Py_IsFalse(flag)) goto side_exit; assert(Py_IsTrue(flag)); stack_pointer += -1; break; @@ -3327,7 +3327,7 @@ case _GUARD_IS_FALSE_POP: { PyObject *flag; flag = stack_pointer[-1]; - if (Py_IsTrue(flag)) goto deoptimize; + if (Py_IsTrue(flag)) goto side_exit; assert(Py_IsFalse(flag)); stack_pointer += -1; break; @@ -3336,7 +3336,7 @@ case _GUARD_IS_NONE_POP: { PyObject *val; val = stack_pointer[-1]; - if (!Py_IsNone(val)) goto deoptimize; + if (!Py_IsNone(val)) goto side_exit; stack_pointer += -1; break; } @@ -3344,7 +3344,7 @@ case _GUARD_IS_NOT_NONE_POP: { PyObject *val; val = stack_pointer[-1]; - if (Py_IsNone(val)) goto deoptimize; + if (Py_IsNone(val)) goto side_exit; Py_DECREF(val); stack_pointer += -1; break; @@ -3418,24 +3418,29 @@ case _COLD_EXIT: { oparg = CURRENT_OPARG(); TIER_TWO_ONLY - _PyExitData *exit = PyOptimizer_GetExit(current_executor, oparg); + _PyExitData *exit = ¤t_executor->exits[oparg]; Py_DECREF(current_executor); exit->hotness++; if (exit->hotness < 0) goto deoptimize; PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - _PyOptimizerObject *opt = interp->optimizer; _PyExecutorObject *executor = NULL; - int optimized = opt->optimize(opt, code, target, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); - if (optimized < 0) goto error_tier_two; - if (optimized) { - exit->exit = executor; + if (target->op.code == ENTER_EXECUTOR) { + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; Py_INCREF(executor); - GOTO_TIER_TWO(); - } - else { - exit->hotness = -10000; /* Choose a better number */ + } else { + _PyOptimizerObject *opt = tstate->interp->optimizer; + int optimized = opt->optimize(opt, code, target, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + if (optimized <= 0) { + next_instr = target; + exit->hotness = -10000; /* Choose a better number */ + if (optimized < 0) goto error_tier_two; + DISPATCH(); + } } + exit->executor = current_executor = executor; + Py_INCREF(current_executor); + GOTO_TIER_TWO(); break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 96b7c243c1379c..1f23625348fc94 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -281,8 +281,8 @@ PySequenceMethods uop_as_sequence = { PyTypeObject _PyUOpExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "uop_executor", - .tp_basicsize = offsetof(_PyExecutorObject, trace), - .tp_itemsize = sizeof(_PyUOpInstruction), + .tp_basicsize = offsetof(_PyExecutorObject, exits), + .tp_itemsize = 1, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, @@ -700,9 +700,10 @@ translate_bytecode_to_trace( * NOPs are excluded from the count. */ static int -compute_used(_PyUOpInstruction *buffer, uint32_t *used) +compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) { int count = 0; + int exit_count = 0; SET_BIT(used, 0); for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { if (!BIT_IS_SET(used, i)) { @@ -720,14 +721,62 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used) /* Mark target as reachable */ SET_BIT(used, buffer[i].oparg); } + if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + exit_count++; + } if (opcode == NOP) { count--; UNSET_BIT(used, i); } } + *exit_count_ptr = exit_count; return count; } +/* Executor side exits */ + +typedef struct _cold_exit { + PyObject_VAR_HEAD + _PyVMData vm_data; + union { + void *jit_code; // Will be a function pointer + _PyUOpInstruction *trace; + }; + _PyUOpInstruction cold; +} _PyColdExitObject; + +static _PyColdExitObject COLD_EXITS[UOP_MAX_TRACE_LENGTH]; + +static _PyExecutorObject * +allocate_executor(int exit_count, int length) +{ + int size = exit_count*sizeof(_PyExitData) + length*sizeof(_PyUOpInstruction); + _PyExecutorObject *res = PyObject_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, size); + if (res != NULL) { + res->trace = (_PyUOpInstruction *)(res->exits + exit_count); + } + return res; +} + +static int cold_exits_initialized = 0; + +static void +initialize_cold_exits(void) { + if (cold_exits_initialized) { + return; + } + cold_exits_initialized = 1; + for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { + COLD_EXITS[i].ob_base.ob_base.ob_refcnt = _Py_IMMORTAL_REFCNT; + COLD_EXITS[i].ob_base.ob_base.ob_type = &_PyUOpExecutor_Type; + COLD_EXITS[i].ob_base.ob_size = 0; + COLD_EXITS[i].vm_data = (_PyVMData){ 0 }; + COLD_EXITS[i].cold.opcode = _COLD_EXIT; + COLD_EXITS[i].cold.oparg = i; + COLD_EXITS[i].trace = &COLD_EXITS[i].cold; + } +} + /* Makes an executor from a buffer of uops. * Account for the buffer having gaps and NOPs by computing a "used" * bit vector and only copying the used uops. Here "used" means reachable @@ -736,12 +785,22 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used) static _PyExecutorObject * make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) { + if (!cold_exits_initialized) { + initialize_cold_exits(); + } uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; - int length = compute_used(buffer, used); - _PyExecutorObject *executor = PyObject_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, length); + int exit_count; + int length = compute_used(buffer, used, &exit_count); + _PyExecutorObject *executor = allocate_executor(exit_count, length); if (executor == NULL) { return NULL; } + /* Initialize exits */ + for (int i = 0; i < exit_count; i++) { + executor->exits[i].executor = (_PyExecutorObject *)&COLD_EXITS[i]; + executor->exits[i].hotness = -273; /* Physics reference */ + } + int next_exit = 0; int dest = length - 1; /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */ for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) { @@ -757,11 +816,17 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) int oparg = executor->trace[dest].oparg; executor->trace[dest].oparg = buffer[oparg].oparg; } + if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + executor->exits[next_exit].target = buffer[i].target; + executor->trace[dest].target = next_exit; + next_exit++; + } /* Set the oparg to be the destination offset, * so that we can set the oparg of earlier jumps correctly. */ buffer[i].oparg = dest; dest--; } + assert(next_exit == exit_count); assert(dest == -1); _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG @@ -1138,36 +1203,5 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) interp->executor_list_head = NULL; } -/* Executor side exits */ - - -typedef struct _cold_exit { - PyObject_VAR_HEAD - _PyVMData vm_data; - union { - void *jit_code; // Will be a function pointer - _PyUOpInstruction *trace; - }; - _PyUOpInstruction cold; -} _PyColdExitObject; - -static int cold_exits_initialized = 0; -_PyColdExitObject COLD_EXITS[_Py_UOP_MAX_TRACE_LENGTH]; -static void -initialize_cold_exits(void) { - if (cold_exits_initialized) { - return; - } - cold_exits_initialized = 1; - for (int i = 0; i < _Py_UOP_MAX_TRACE_LENGTH; i++) { - COLD_EXITS[i].ob_base.ob_base.ob_refcnt = _Py_IMMORTAL_REFCNT; - COLD_EXITS[i].ob_base.ob_base.ob_type = &_PyUOpExecutor_Type; - COLD_EXITS[i].ob_base.ob_size = 0; - COLD_EXITS[i].vm_data = (_PyVMData){ 0 }; - COLD_EXITS[i].cold.opcode = _COLD_EXIT; - COLD_EXITS[i].cold.oparg = i; - COLD_EXITS[i].trace = &COLD_EXITS[i].cold; - } -} diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 82ef8888bfcee5..7f6c2811aa4e53 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -21,6 +21,7 @@ class Properties: uses_co_names: bool uses_locals: bool has_free: bool + side_exit: bool def dump(self, indent: str) -> None: print(indent, end="") @@ -45,6 +46,7 @@ def from_list(properties: list["Properties"]) -> "Properties": uses_co_names=any(p.uses_co_names for p in properties), uses_locals=any(p.uses_locals for p in properties), has_free=any(p.has_free for p in properties), + side_exit=any(p.side_exit for p in properties), ) @@ -64,6 +66,7 @@ def from_list(properties: list["Properties"]) -> "Properties": uses_co_names=False, uses_locals=False, has_free=False, + side_exit=False, ) @@ -423,10 +426,13 @@ def compute_properties(op: parser.InstDef) -> Properties: or variable_used(op, "PyCell_GET") or variable_used(op, "PyCell_SET") ) + deopts_if = variable_used(op, "DEOPT_IF") + exits_if = variable_used(op, "EXIT_IF") return Properties( escapes=makes_escaping_api_call(op), infallible=is_infallible(op), - deopts=variable_used(op, "DEOPT_IF"), + deopts=deopts_if or exits_if, + side_exit=exits_if, oparg=variable_used(op, "oparg"), jumps=variable_used(op, "JUMPBY"), eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"), diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 5a42a05c5c2ef2..31673deb3a18fe 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -154,6 +154,7 @@ def replace_check_eval_breaker( REPLACEMENT_FUNCTIONS = { + "EXIT_IF": replace_deopt, "DEOPT_IF": replace_deopt, "ERROR_IF": replace_error, "DECREF_INPUTS": replace_decrefs, @@ -205,6 +206,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_EVAL_BREAK_FLAG") if p.deopts: flags.append("HAS_DEOPT_FLAG") + if p.side_exit: + flags.append("HAS_EXIT_FLAG") if not p.infallible: flags.append("HAS_ERROR_FLAG") if p.escapes: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 9b7df9a54c7b3b..bdf00ccdd7931b 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -50,6 +50,7 @@ "DEOPT", "ERROR", "ESCAPES", + "EXIT", ] diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 7897b89b2752a7..8b4d1646dd5a87 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -98,9 +98,25 @@ def tier2_replace_deopt( out.emit(") goto deoptimize;\n") +def tier2_replace_exit_if( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("if ", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(") goto side_exit;\n") + + TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt +TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: From 7c6267ae7442732aa924f2169ce940603a7b4d92 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 18:46:53 +0000 Subject: [PATCH 03/40] Modify internal interfaces --- Include/cpython/optimizer.h | 4 +++- Python/bytecodes.c | 17 +++++++-------- Python/executor_cases.c.h | 8 +++---- Python/generated_cases.c.h | 9 ++++---- Python/optimizer.c | 42 +++++++++++++++++++++++++++++-------- 5 files changed, 52 insertions(+), 28 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 8437b35a473d38..b1d6ebff73c00a 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -45,6 +45,8 @@ typedef struct _exit_data { typedef struct _PyExecutorObject { PyObject_VAR_HEAD _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ + uint32_t exit_count; + uint32_t code_size; _PyUOpInstruction *trace; _PyExitData exits[1]; } _PyExecutorObject; @@ -79,7 +81,7 @@ PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); int -_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer); +_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr); extern _PyOptimizerObject _PyOptimizer_Default; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ac86fd6bc6d10d..6bd80faf6eb7cf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2331,13 +2331,12 @@ dummy_func( oparg >>= 8; start--; } - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer); + _PyExecutorObject *executor; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); ERROR_IF(optimized < 0, error); if (optimized) { - // Rewind and enter the executor: - assert(start->op.code == ENTER_EXECUTOR); - next_instr = start; - this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + current_executor = executor; + GOTO_TIER_TWO(); } else { int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); @@ -4079,13 +4078,13 @@ dummy_func( DEOPT_IF(exit->hotness < 0); PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - _PyExecutorObject *executor = NULL; + _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + executor = code->co_executors->executors[oparg & 255]; Py_INCREF(executor); } else { - _PyOptimizerObject *opt = tstate->interp->optimizer; - int optimized = opt->optimize(opt, code, target, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + printf("Hot side exit.\n"); + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { next_instr = target; exit->hotness = -10000; /* Choose a better number */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3553870e59ebce..252e8e09494799 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3414,13 +3414,13 @@ if (exit->hotness < 0) goto deoptimize; PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - _PyExecutorObject *executor = NULL; + _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + executor = code->co_executors->executors[oparg & 255]; Py_INCREF(executor); } else { - _PyOptimizerObject *opt = tstate->interp->optimizer; - int optimized = opt->optimize(opt, code, target, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + printf("Hot side exit.\n"); + int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { next_instr = target; exit->hotness = -10000; /* Choose a better number */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 68468728d44bf8..609bb7c78a5ad2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3297,13 +3297,12 @@ oparg >>= 8; start--; } - int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer); + _PyExecutorObject *executor; + int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); if (optimized < 0) goto error; if (optimized) { - // Rewind and enter the executor: - assert(start->op.code == ENTER_EXECUTOR); - next_instr = start; - this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + current_executor = executor; + GOTO_TIER_TWO(); } else { int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); diff --git a/Python/optimizer.c b/Python/optimizer.c index 1f23625348fc94..7b069ffa242940 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -162,8 +162,28 @@ PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) Py_DECREF(old); } +static void +dump_uop(_PyUOpInstruction *uop) +{ + printf(" %s\n", _PyOpcode_uop_name[uop->opcode]); +} + +static void +dump_executor(_PyExecutorObject *executor) +{ + printf("Executor:\n"); + for (uint32_t i = 0; i < executor->code_size; i++) { + dump_uop(&executor->trace[i]); + } +} + +/* Returns 1 if optimized, 0 if not optimized, and -1 for an error. + * If optimized, *executor_ptr contains a new reference to the executor + */ int -_PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer) +_PyOptimizer_Optimize( + _PyInterpreterFrame *frame, _Py_CODEUNIT *start, + PyObject **stack_pointer, _PyExecutorObject **executor_ptr) { PyCodeObject *code = (PyCodeObject *)frame->f_executable; assert(PyCode_Check(code)); @@ -172,12 +192,13 @@ _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject return 0; } _PyOptimizerObject *opt = interp->optimizer; - _PyExecutorObject *executor = NULL; - int err = opt->optimize(opt, code, start, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); + int err = opt->optimize(opt, code, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { - assert(executor == NULL); + assert(*executor_ptr == NULL); return err; } + assert(*executor_ptr != NULL); + dump_executor(*executor_ptr); int index = get_index_for_executor(code, start); if (index < 0) { /* Out of memory. Don't raise and assume that the @@ -186,11 +207,11 @@ _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject * If an optimizer has already produced an executor, * it might get confused by the executor disappearing, * but there is not much we can do about that here. */ - Py_DECREF(executor); + Py_DECREF(*executor_ptr); return 0; } - insert_executor(code, start, index, executor); - Py_DECREF(executor); + insert_executor(code, start, index, *executor_ptr); + assert((*executor_ptr)->vm_data.valid); return 1; } @@ -752,9 +773,12 @@ allocate_executor(int exit_count, int length) { int size = exit_count*sizeof(_PyExitData) + length*sizeof(_PyUOpInstruction); _PyExecutorObject *res = PyObject_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, size); - if (res != NULL) { - res->trace = (_PyUOpInstruction *)(res->exits + exit_count); + if (res == NULL) { + return NULL; } + res->trace = (_PyUOpInstruction *)(res->exits + exit_count); + res->code_size = length; + res->exit_count = exit_count; return res; } From 8b3c2e008193da2ab3f72bea45708b27b21829f8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 11 Jan 2024 21:08:08 +0000 Subject: [PATCH 04/40] Jump to next executor without updating current_executor. --- Include/cpython/optimizer.h | 2 +- Include/internal/pycore_uop_ids.h | 3 +- Include/internal/pycore_uop_metadata.h | 4 +- Python/bytecodes.c | 37 ++++++++++++----- Python/ceval.c | 23 +++++------ Python/executor_cases.c.h | 30 ++++++++++---- Python/generated_cases.c.h | 8 +++- Python/optimizer.c | 55 +++++++++----------------- 8 files changed, 93 insertions(+), 69 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index b1d6ebff73c00a..9708f3834cdfb1 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -44,10 +44,10 @@ typedef struct _exit_data { typedef struct _PyExecutorObject { PyObject_VAR_HEAD + _PyUOpInstruction *trace; _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ uint32_t exit_count; uint32_t code_size; - _PyUOpInstruction *trace; _PyExitData exits[1]; } _PyExecutorObject; diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 0c12686cfe29d9..8fee5f0a0faf64 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -233,7 +233,8 @@ extern "C" { #define _LOAD_CONST_INLINE_BORROW 380 #define _INTERNAL_INCREMENT_OPT_COUNTER 381 #define _COLD_EXIT 382 -#define MAX_UOP_ID 382 +#define _START_EXECUTOR 383 +#define MAX_UOP_ID 383 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f80740824b751c..e42d3aeefe44a1 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -204,7 +204,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE_BORROW] = 0, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, - [_COLD_EXIT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_START_EXECUTOR] = 0, }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { @@ -367,6 +368,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", [_SET_IP] = "_SET_IP", [_SET_UPDATE] = "_SET_UPDATE", + [_START_EXECUTOR] = "_START_EXECUTOR", [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0a5a7eb52c33ae..2b695493417dbd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2336,7 +2336,9 @@ dummy_func( int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); ERROR_IF(optimized < 0, error); if (optimized) { - current_executor = executor; + assert(current_executor == NULL); + current_executor = (_PyExecutorObject *)Py_None; + next_uop = executor->trace; GOTO_TIER_TWO(); } else { @@ -2367,12 +2369,13 @@ dummy_func( inst(ENTER_EXECUTOR, (--)) { TIER_ONE_ONLY CHECK_EVAL_BREAKER(); - PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; if (executor->vm_data.valid) { + assert(current_executor == NULL); + current_executor = (_PyExecutorObject *)Py_None; Py_INCREF(executor); - current_executor = executor; + next_uop = executor->trace; GOTO_TIER_TWO(); } else { @@ -4040,7 +4043,7 @@ dummy_func( } op(_JUMP_TO_TOP, (--)) { - next_uop = current_executor->trace; + next_uop = ¤t_executor->trace[1]; CHECK_EVAL_BREAKER(); } @@ -4081,31 +4084,45 @@ dummy_func( op(_COLD_EXIT, (--)) { TIER_TWO_ONLY + assert(current_executor->trace[0].opcode != _COLD_EXIT); _PyExitData *exit = ¤t_executor->exits[oparg]; - Py_DECREF(current_executor); exit->hotness++; - DEOPT_IF(exit->hotness < 0); + if (exit->hotness < 0) { + next_instr = exit->target + _PyCode_CODE(_PyFrame_GetCode(frame)); + Py_DECREF(current_executor); + current_executor = NULL; + DISPATCH(); + } PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { - executor = code->co_executors->executors[oparg & 255]; + executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); } else { - printf("Hot side exit.\n"); int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { + Py_DECREF(current_executor); + current_executor = NULL; next_instr = target; exit->hotness = -10000; /* Choose a better number */ ERROR_IF(optimized < 0, error); DISPATCH(); } } - exit->executor = current_executor = executor; - Py_INCREF(current_executor); + Py_INCREF(executor); + exit->executor = executor; + next_uop = executor->trace; GOTO_TIER_TWO(); } + op(_START_EXECUTOR, (executor/4 --)) { + TIER_TWO_ONLY + Py_DECREF(current_executor); + Py_INCREF(executor); + current_executor = (_PyExecutorObject*)executor; + } + // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index 2cccf64c40257c..0ff10c92c3e2ed 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -745,6 +745,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *next_instr; PyObject **stack_pointer; + _PyUOpInstruction *next_uop = NULL; start_frame: @@ -991,12 +992,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif OPT_STAT_INC(traces_executed); - _PyUOpInstruction *next_uop = current_executor->trace; uint16_t uopcode; #ifdef Py_STATS uint64_t trace_uop_execution_counter = 0; #endif + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); for (;;) { uopcode = next_uop->opcode; DPRINTF(3, @@ -1059,32 +1060,32 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(current_executor); + current_executor = NULL; goto resume_with_error; // Jump here from DEOPT_IF() deoptimize: next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); - DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", + DPRINTF(2, "DEOPT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d -> %s]\n", uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, next_uop[-1].target, - (int)(next_uop - current_executor->trace - 1), - _PyOpcode_OpName[frame->instr_ptr->op.code]); + _PyOpcode_OpName[next_instr->op.code]); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); Py_DECREF(current_executor); + current_executor = NULL; DISPATCH(); // Jump here from EXIT_IF() side_exit: OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); - _PyExitData *exit = ¤t_executor->exits[next_uop[-1].target]; - DPRINTF(2, "SIDE EXIT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", target %d @ %d -> %s]\n", - uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit->target, - (int)(next_uop - current_executor->trace - 1), - _PyOpcode_OpName[frame->instr_ptr->op.code]); + uint32_t exit_id = next_uop[-1].target; + _PyExitData *exit = ¤t_executor->exits[exit_id]; + DPRINTF(2, "SIDE EXIT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", exit %u, hotness %d, target %d -> %s]\n", + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit_id, exit->hotness, exit->target, + _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); Py_INCREF(exit->executor); - Py_DECREF(current_executor); - current_executor = exit->executor; + next_uop = exit->executor->trace; GOTO_TIER_TWO(); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0dba0c4d15679e..6c48c8269dbddd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3354,7 +3354,7 @@ } case _JUMP_TO_TOP: { - next_uop = current_executor->trace; + next_uop = ¤t_executor->trace[1]; CHECK_EVAL_BREAKER(); break; } @@ -3411,30 +3411,46 @@ case _COLD_EXIT: { oparg = CURRENT_OPARG(); TIER_TWO_ONLY + assert(current_executor->trace[0].opcode != _COLD_EXIT); _PyExitData *exit = ¤t_executor->exits[oparg]; - Py_DECREF(current_executor); exit->hotness++; - if (exit->hotness < 0) goto deoptimize; + if (exit->hotness < 0) { + next_instr = exit->target + _PyCode_CODE(_PyFrame_GetCode(frame)); + Py_DECREF(current_executor); + current_executor = NULL; + DISPATCH(); + } PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { - executor = code->co_executors->executors[oparg & 255]; + executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); } else { - printf("Hot side exit.\n"); int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { + Py_DECREF(current_executor); + current_executor = NULL; next_instr = target; exit->hotness = -10000; /* Choose a better number */ if (optimized < 0) goto error_tier_two; DISPATCH(); } } - exit->executor = current_executor = executor; - Py_INCREF(current_executor); + Py_INCREF(executor); + exit->executor = executor; + next_uop = executor->trace; GOTO_TIER_TWO(); break; } + case _START_EXECUTOR: { + PyObject *executor = (PyObject *)CURRENT_OPERAND(); + TIER_TWO_ONLY + Py_DECREF(current_executor); + Py_INCREF(executor); + current_executor = (_PyExecutorObject*)executor; + break; + } + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 609bb7c78a5ad2..a3dda2d9920ec9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2379,8 +2379,10 @@ PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; if (executor->vm_data.valid) { + assert(current_executor == NULL); + current_executor = (_PyExecutorObject *)Py_None; Py_INCREF(executor); - current_executor = executor; + next_uop = executor->trace; GOTO_TIER_TWO(); } else { @@ -3301,7 +3303,9 @@ int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); if (optimized < 0) goto error; if (optimized) { - current_executor = executor; + assert(current_executor == NULL); + current_executor = (_PyExecutorObject *)Py_None; + next_uop = executor->trace; GOTO_TIER_TWO(); } else { diff --git a/Python/optimizer.c b/Python/optimizer.c index 4bc174a795ad96..71212debf5f059 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -162,21 +162,6 @@ PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) Py_DECREF(old); } -static void -dump_uop(_PyUOpInstruction *uop) -{ - printf(" %s\n", _PyOpcode_uop_name[uop->opcode]); -} - -static void -dump_executor(_PyExecutorObject *executor) -{ - printf("Executor:\n"); - for (uint32_t i = 0; i < executor->code_size; i++) { - dump_uop(&executor->trace[i]); - } -} - /* Returns 1 if optimized, 0 if not optimized, and -1 for an error. * If optimized, *executor_ptr contains a new reference to the executor */ @@ -194,11 +179,9 @@ _PyOptimizer_Optimize( _PyOptimizerObject *opt = interp->optimizer; int err = opt->optimize(opt, code, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { - assert(*executor_ptr == NULL); return err; } assert(*executor_ptr != NULL); - dump_executor(*executor_ptr); int index = get_index_for_executor(code, start); if (index < 0) { /* Out of memory. Don't raise and assume that the @@ -260,7 +243,7 @@ _PyUOpName(int index) static Py_ssize_t uop_len(_PyExecutorObject *self) { - return Py_SIZE(self); + return self->code_size; } static PyObject * @@ -758,12 +741,7 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) /* Executor side exits */ typedef struct _cold_exit { - PyObject_VAR_HEAD - _PyVMData vm_data; - union { - void *jit_code; // Will be a function pointer - _PyUOpInstruction *trace; - }; + _PyExecutorObject base; _PyUOpInstruction cold; } _PyColdExitObject; @@ -792,13 +770,13 @@ initialize_cold_exits(void) { } cold_exits_initialized = 1; for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { - COLD_EXITS[i].ob_base.ob_base.ob_refcnt = _Py_IMMORTAL_REFCNT; - COLD_EXITS[i].ob_base.ob_base.ob_type = &_PyUOpExecutor_Type; - COLD_EXITS[i].ob_base.ob_size = 0; - COLD_EXITS[i].vm_data = (_PyVMData){ 0 }; + COLD_EXITS[i].base.ob_base.ob_base.ob_refcnt = _Py_IMMORTAL_REFCNT; + COLD_EXITS[i].base.ob_base.ob_base.ob_type = &_PyUOpExecutor_Type; + COLD_EXITS[i].base.ob_base.ob_size = 0; + COLD_EXITS[i].base.vm_data = (_PyVMData){ 0 }; COLD_EXITS[i].cold.opcode = _COLD_EXIT; COLD_EXITS[i].cold.oparg = i; - COLD_EXITS[i].trace = &COLD_EXITS[i].cold; + COLD_EXITS[i].base.trace = &COLD_EXITS[i].cold; } } @@ -816,17 +794,17 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; int exit_count; int length = compute_used(buffer, used, &exit_count); - _PyExecutorObject *executor = allocate_executor(exit_count, length); + _PyExecutorObject *executor = allocate_executor(exit_count, length+1); if (executor == NULL) { return NULL; } /* Initialize exits */ for (int i = 0; i < exit_count; i++) { executor->exits[i].executor = (_PyExecutorObject *)&COLD_EXITS[i]; - executor->exits[i].hotness = -273; /* Physics reference */ + executor->exits[i].hotness = -67; } - int next_exit = 0; - int dest = length - 1; + int next_exit = exit_count-1; + int dest = length; /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */ for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) { if (!BIT_IS_SET(used, i)) { @@ -844,15 +822,17 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { executor->exits[next_exit].target = buffer[i].target; executor->trace[dest].target = next_exit; - next_exit++; + next_exit--; } /* Set the oparg to be the destination offset, * so that we can set the oparg of earlier jumps correctly. */ buffer[i].oparg = dest; dest--; } - assert(next_exit == exit_count); - assert(dest == -1); + assert(next_exit == -1); + assert(dest == 0); + executor->trace[0].opcode = _START_EXECUTOR; + executor->trace[0].operand = (uintptr_t)executor; _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -1180,6 +1160,9 @@ void _Py_ExecutorClear(_PyExecutorObject *executor) { unlink_executor(executor); + for (uint32_t i = 0; i < executor->exit_count; i++) { + Py_DECREF(executor->exits[i].executor); + } } void From 92a3b614b0c57694c2c1fde2037537b8e8cca18f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 12 Jan 2024 03:41:52 +0000 Subject: [PATCH 05/40] Support cycle GC for executors. --- Include/internal/pycore_uop_ids.h | 1 - Python/bytecodes.c | 11 +-- Python/executor_cases.c.h | 12 ++-- Python/generated_cases.c.h | 1 + Python/optimizer.c | 108 +++++++++++++++++++----------- 5 files changed, 80 insertions(+), 53 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 8fee5f0a0faf64..e78e5d69567242 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -139,7 +139,6 @@ extern "C" { #define _CONTAINS_OP CONTAINS_OP #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _JUMP_BACKWARD JUMP_BACKWARD #define _POP_JUMP_IF_FALSE 339 #define _POP_JUMP_IF_TRUE 340 #define _IS_NONE 341 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2b695493417dbd..6d731c90111abd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2314,6 +2314,7 @@ dummy_func( } inst(JUMP_BACKWARD, (unused/1 --)) { + TIER_ONE_ONLY CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); @@ -4087,14 +4088,15 @@ dummy_func( assert(current_executor->trace[0].opcode != _COLD_EXIT); _PyExitData *exit = ¤t_executor->exits[oparg]; exit->hotness++; + assert(exit->executor->trace[0].opcode == _COLD_EXIT); + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; if (exit->hotness < 0) { - next_instr = exit->target + _PyCode_CODE(_PyFrame_GetCode(frame)); + next_instr = target; Py_DECREF(current_executor); current_executor = NULL; DISPATCH(); } - PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { executor = code->co_executors->executors[target->op.arg]; @@ -4102,10 +4104,10 @@ dummy_func( } else { int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { + exit->hotness = -10000; /* Choose a better number */ Py_DECREF(current_executor); current_executor = NULL; next_instr = target; - exit->hotness = -10000; /* Choose a better number */ ERROR_IF(optimized < 0, error); DISPATCH(); } @@ -4119,7 +4121,6 @@ dummy_func( op(_START_EXECUTOR, (executor/4 --)) { TIER_TWO_ONLY Py_DECREF(current_executor); - Py_INCREF(executor); current_executor = (_PyExecutorObject*)executor; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6c48c8269dbddd..7c53d6e8841ec1 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2013,8 +2013,6 @@ break; } - /* _JUMP_BACKWARD is not a viable micro-op for tier 2 */ - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ @@ -3414,14 +3412,15 @@ assert(current_executor->trace[0].opcode != _COLD_EXIT); _PyExitData *exit = ¤t_executor->exits[oparg]; exit->hotness++; + assert(exit->executor->trace[0].opcode == _COLD_EXIT); + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; if (exit->hotness < 0) { - next_instr = exit->target + _PyCode_CODE(_PyFrame_GetCode(frame)); + next_instr = target; Py_DECREF(current_executor); current_executor = NULL; DISPATCH(); } - PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { executor = code->co_executors->executors[target->op.arg]; @@ -3429,10 +3428,10 @@ } else { int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { + exit->hotness = -10000; /* Choose a better number */ Py_DECREF(current_executor); current_executor = NULL; next_instr = target; - exit->hotness = -10000; /* Choose a better number */ if (optimized < 0) goto error_tier_two; DISPATCH(); } @@ -3448,7 +3447,6 @@ PyObject *executor = (PyObject *)CURRENT_OPERAND(); TIER_TWO_ONLY Py_DECREF(current_executor); - Py_INCREF(executor); current_executor = (_PyExecutorObject*)executor; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a3dda2d9920ec9..23ac4bb3824dac 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3281,6 +3281,7 @@ next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); /* Skip 1 cache entry */ + TIER_ONE_ONLY CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); diff --git a/Python/optimizer.c b/Python/optimizer.c index 71212debf5f059..f78b92192ba758 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -2,6 +2,7 @@ #include "opcode.h" #include "pycore_interp.h" #include "pycore_bitutils.h" // _Py_popcount32() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_metadata.h" // _PyOpcode_OpName[] #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE #include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize() @@ -230,8 +231,9 @@ static PyMethodDef executor_methods[] = { static void uop_dealloc(_PyExecutorObject *self) { + _PyObject_GC_UNTRACK(self); _Py_ExecutorClear(self); - PyObject_Free(self); + PyObject_GC_Del(self); } const char * @@ -282,15 +284,69 @@ PySequenceMethods uop_as_sequence = { .sq_item = (ssizeargfunc)uop_item, }; +static void +unlink_executor(_PyExecutorObject *executor) +{ + if (!executor->vm_data.linked) { + return; + } + _PyExecutorLinkListNode *links = &executor->vm_data.links; + _PyExecutorObject *next = links->next; + _PyExecutorObject *prev = links->previous; + if (next != NULL) { + next->vm_data.links.previous = prev; + } + if (prev != NULL) { + prev->vm_data.links.next = next; + } + else { + // prev == NULL implies that executor is the list head + PyInterpreterState *interp = PyInterpreterState_Get(); + assert(interp->executor_list_head == executor); + interp->executor_list_head = next; + } + executor->vm_data.linked = false; +} + +/* This must be called by executors during dealloc */ +void +_Py_ExecutorClear(_PyExecutorObject *executor) +{ + executor->vm_data.valid = 0; + unlink_executor(executor); + for (uint32_t i = 0; i < executor->exit_count; i++) { + Py_CLEAR(executor->exits[i].executor); + } +} + +static int +executor_clear(PyObject *o) +{ + _Py_ExecutorClear((_PyExecutorObject *)o); + return 0; +} + +static int +executor_traverse(PyObject *o, visitproc visit, void *arg) +{ + _PyExecutorObject *executor = (_PyExecutorObject *)o; + for (uint32_t i = 0; i < executor->exit_count; i++) { + Py_VISIT(executor->exits[i].executor); + } + return 0; +} + PyTypeObject _PyUOpExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "uop_executor", .tp_basicsize = offsetof(_PyExecutorObject, exits), .tp_itemsize = 1, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, .tp_methods = executor_methods, + .tp_traverse = executor_traverse, + .tp_clear = executor_clear, }; /* TO DO -- Generate these tables */ @@ -316,6 +372,7 @@ BRANCH_TO_GUARD[4][2] = { #define TRACE_STACK_SIZE 5 + #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 @@ -751,7 +808,7 @@ static _PyExecutorObject * allocate_executor(int exit_count, int length) { int size = exit_count*sizeof(_PyExitData) + length*sizeof(_PyUOpInstruction); - _PyExecutorObject *res = PyObject_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, size); + _PyExecutorObject *res = PyObject_GC_NewVar(_PyExecutorObject, &_PyUOpExecutor_Type, size); if (res == NULL) { return NULL; } @@ -852,6 +909,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) } } #endif + _PyObject_GC_TRACK(executor); return executor; } @@ -919,7 +977,8 @@ PyUnstable_Optimizer_NewUOpOptimizer(void) static void counter_dealloc(_PyExecutorObject *self) { - PyObject *opt = (PyObject *)self->trace[0].operand; + /* The optimizer is the operand of the first uop. */ + PyObject *opt = (PyObject *)self->trace[1].operand; Py_DECREF(opt); uop_dealloc(self); } @@ -927,11 +986,13 @@ counter_dealloc(_PyExecutorObject *self) { PyTypeObject _PyCounterExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "counting_executor", - .tp_basicsize = offsetof(_PyExecutorObject, trace), - .tp_itemsize = sizeof(_PyUOpInstruction), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_basicsize = offsetof(_PyExecutorObject, exits), + .tp_itemsize = 1, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, .tp_dealloc = (destructor)counter_dealloc, .tp_methods = executor_methods, + .tp_traverse = executor_traverse, + .tp_clear = executor_clear, }; static int @@ -1120,29 +1181,6 @@ link_executor(_PyExecutorObject *executor) assert(interp->executor_list_head->vm_data.links.previous == NULL); } -static void -unlink_executor(_PyExecutorObject *executor) -{ - if (!executor->vm_data.linked) { - return; - } - _PyExecutorLinkListNode *links = &executor->vm_data.links; - _PyExecutorObject *next = links->next; - _PyExecutorObject *prev = links->previous; - if (next != NULL) { - next->vm_data.links.previous = prev; - } - if (prev != NULL) { - prev->vm_data.links.next = next; - } - else { - // prev == NULL implies that executor is the list head - PyInterpreterState *interp = PyInterpreterState_Get(); - assert(interp->executor_list_head == executor); - interp->executor_list_head = next; - } - executor->vm_data.linked = false; -} /* This must be called by optimizers before using the executor */ void @@ -1155,16 +1193,6 @@ _Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) link_executor(executor); } -/* This must be called by executors during dealloc */ -void -_Py_ExecutorClear(_PyExecutorObject *executor) -{ - unlink_executor(executor); - for (uint32_t i = 0; i < executor->exit_count; i++) { - Py_DECREF(executor->exits[i].executor); - } -} - void _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj) { From e3def486e9f2b57bba762742889ca96e4dfa7b10 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Jan 2024 19:54:12 +0000 Subject: [PATCH 06/40] Give cold exits their own class, to fix GC handling of exits --- Include/cpython/optimizer.h | 8 ++++++ Include/internal/pycore_uop_ids.h | 3 ++- Include/internal/pycore_uop_metadata.h | 2 ++ Python/bytecodes.c | 18 ++++++++----- Python/executor_cases.c.h | 7 +++++ Python/generated_cases.c.h | 14 +++++----- Python/optimizer.c | 37 ++++++++++++++++++++------ 7 files changed, 68 insertions(+), 21 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 9708f3834cdfb1..b7028f44c195a2 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -51,6 +51,14 @@ typedef struct _PyExecutorObject { _PyExitData exits[1]; } _PyExecutorObject; +typedef struct _cold_exit { + _PyExecutorObject base; + _PyUOpInstruction uop; +} _PyColdExitObject; + + +extern _PyColdExitObject Py_FatalErrorExecutor; + typedef struct _PyOptimizerObject _PyOptimizerObject; /* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index e78e5d69567242..30ed66bdf3a7ef 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -233,7 +233,8 @@ extern "C" { #define _INTERNAL_INCREMENT_OPT_COUNTER 381 #define _COLD_EXIT 382 #define _START_EXECUTOR 383 -#define MAX_UOP_ID 383 +#define _FATAL_ERROR 384 +#define MAX_UOP_ID 384 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index e42d3aeefe44a1..c39e1b2b80ee9c 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -206,6 +206,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_START_EXECUTOR] = 0, + [_FATAL_ERROR] = HAS_ESCAPES_FLAG, }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { @@ -279,6 +280,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_END_SEND] = "_END_SEND", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", + [_FATAL_ERROR] = "_FATAL_ERROR", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6d731c90111abd..3f50e7cf091416 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2338,7 +2338,7 @@ dummy_func( ERROR_IF(optimized < 0, error); if (optimized) { assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)Py_None; + current_executor = (_PyExecutorObject *)&Py_FatalErrorExecutor; next_uop = executor->trace; GOTO_TIER_TWO(); } @@ -2374,7 +2374,7 @@ dummy_func( _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; if (executor->vm_data.valid) { assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)Py_None; + current_executor = (_PyExecutorObject *)&Py_FatalErrorExecutor; Py_INCREF(executor); next_uop = executor->trace; GOTO_TIER_TWO(); @@ -4005,14 +4005,14 @@ dummy_func( inst(CACHE, (--)) { TIER_ONE_ONLY - assert(0 && "Executing a cache."); - Py_UNREACHABLE(); + assert(0); + Py_FatalError("Executing a cache."); } inst(RESERVED, (--)) { TIER_ONE_ONLY - assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); + assert(0); + Py_FatalError("Executing RESERVED instruction."); } ///////// Tier-2 only opcodes ///////// @@ -4124,6 +4124,12 @@ dummy_func( current_executor = (_PyExecutorObject*)executor; } + op(_FATAL_ERROR, (--)) { + TIER_TWO_ONLY + assert(0); + Py_FatalError("Fatal error uop executed."); + } + // END BYTECODES // diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7c53d6e8841ec1..52e3ba7cf31d0a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3451,4 +3451,11 @@ break; } + case _FATAL_ERROR: { + TIER_TWO_ONLY + assert(0); + Py_FatalError("Fatal error uop executed."); + break; + } + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 23ac4bb3824dac..5b4c5c23297c4b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -740,8 +740,9 @@ next_instr += 1; INSTRUCTION_STATS(CACHE); TIER_ONE_ONLY - assert(0 && "Executing a cache."); - Py_UNREACHABLE(); + assert(0); + Py_FatalError("Executing a cache."); + DISPATCH(); } TARGET(CALL) { @@ -2380,7 +2381,7 @@ _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; if (executor->vm_data.valid) { assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)Py_None; + current_executor = (_PyExecutorObject *)&Py_FatalErrorExecutor; Py_INCREF(executor); next_uop = executor->trace; GOTO_TIER_TWO(); @@ -3305,7 +3306,7 @@ if (optimized < 0) goto error; if (optimized) { assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)Py_None; + current_executor = (_PyExecutorObject *)&Py_FatalErrorExecutor; next_uop = executor->trace; GOTO_TIER_TWO(); } @@ -4795,8 +4796,9 @@ next_instr += 1; INSTRUCTION_STATS(RESERVED); TIER_ONE_ONLY - assert(0 && "Executing RESERVED instruction."); - Py_UNREACHABLE(); + assert(0); + Py_FatalError("Executing RESERVED instruction."); + DISPATCH(); } TARGET(RESUME) { diff --git a/Python/optimizer.c b/Python/optimizer.c index f78b92192ba758..7984ceae06cd77 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -797,10 +797,22 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) /* Executor side exits */ -typedef struct _cold_exit { - _PyExecutorObject base; - _PyUOpInstruction cold; -} _PyColdExitObject; +static void +cold_dealloc(_PyExecutorObject *self) { + _Py_ExecutorClear(self); + PyObject_Free(self); +} + +PyTypeObject _ColdExit_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "cold_exit", + .tp_basicsize = offsetof(_PyExecutorObject, exits), + .tp_itemsize = 1, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_dealloc = (destructor)cold_dealloc, + .tp_as_sequence = &uop_as_sequence, + .tp_methods = executor_methods, +}; static _PyColdExitObject COLD_EXITS[UOP_MAX_TRACE_LENGTH]; @@ -818,6 +830,15 @@ allocate_executor(int exit_count, int length) return res; } +_PyColdExitObject Py_FatalErrorExecutor = { + .base = { + PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) + .vm_data = { 0 }, + .trace = &Py_FatalErrorExecutor.uop + }, + .uop.opcode = _FATAL_ERROR, +}; + static int cold_exits_initialized = 0; static void @@ -828,12 +849,12 @@ initialize_cold_exits(void) { cold_exits_initialized = 1; for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { COLD_EXITS[i].base.ob_base.ob_base.ob_refcnt = _Py_IMMORTAL_REFCNT; - COLD_EXITS[i].base.ob_base.ob_base.ob_type = &_PyUOpExecutor_Type; + COLD_EXITS[i].base.ob_base.ob_base.ob_type = &_ColdExit_Type; COLD_EXITS[i].base.ob_base.ob_size = 0; COLD_EXITS[i].base.vm_data = (_PyVMData){ 0 }; - COLD_EXITS[i].cold.opcode = _COLD_EXIT; - COLD_EXITS[i].cold.oparg = i; - COLD_EXITS[i].base.trace = &COLD_EXITS[i].cold; + COLD_EXITS[i].uop.opcode = _COLD_EXIT; + COLD_EXITS[i].uop.oparg = i; + COLD_EXITS[i].base.trace = &COLD_EXITS[i].uop; } } From 87e544b9fe551ffcd54c02a7360f59597ddbc2e2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Jan 2024 20:18:26 +0000 Subject: [PATCH 07/40] Generate table of cold exits --- Python/optimizer.c | 545 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 523 insertions(+), 22 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 7984ceae06cd77..f0b737f54534c3 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -839,25 +839,6 @@ _PyColdExitObject Py_FatalErrorExecutor = { .uop.opcode = _FATAL_ERROR, }; -static int cold_exits_initialized = 0; - -static void -initialize_cold_exits(void) { - if (cold_exits_initialized) { - return; - } - cold_exits_initialized = 1; - for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { - COLD_EXITS[i].base.ob_base.ob_base.ob_refcnt = _Py_IMMORTAL_REFCNT; - COLD_EXITS[i].base.ob_base.ob_base.ob_type = &_ColdExit_Type; - COLD_EXITS[i].base.ob_base.ob_size = 0; - COLD_EXITS[i].base.vm_data = (_PyVMData){ 0 }; - COLD_EXITS[i].uop.opcode = _COLD_EXIT; - COLD_EXITS[i].uop.oparg = i; - COLD_EXITS[i].base.trace = &COLD_EXITS[i].uop; - } -} - /* Makes an executor from a buffer of uops. * Account for the buffer having gaps and NOPs by computing a "used" * bit vector and only copying the used uops. Here "used" means reachable @@ -866,9 +847,6 @@ initialize_cold_exits(void) { static _PyExecutorObject * make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) { - if (!cold_exits_initialized) { - initialize_cold_exits(); - } uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; int exit_count; int length = compute_used(buffer, used, &exit_count); @@ -1260,5 +1238,528 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) interp->executor_list_head = NULL; } +/* The following table is generated from this script: +print(f'static_assert(UOP_MAX_TRACE_LENGTH == {UOP_MAX_TRACE_LENGTH}, "COLD_EXITS must be regenerated");') +print("static _PyColdExitObject COLD_EXITS[{UOP_MAX_TRACE_LENGTH}] = {") +for i in range(UOP_MAX_TRACE_LENGTH): + print(f" [{i}] = {{ .base = {{ PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = {{ 0 }}," + f" .trace = &COLD_EXITS[{i}].uop }}, .uop.opcode = _COLD_EXIT, .uop.oparg = {i} }},") +print("};") +*/ +static_assert(UOP_MAX_TRACE_LENGTH == 512, "COLD_EXITS must be regenerated"); +static _PyColdExitObject COLD_EXITS[512] = { + [0] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[0].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 0 }, + [1] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[1].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 1 }, + [2] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[2].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 2 }, + [3] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[3].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 3 }, + [4] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[4].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 4 }, + [5] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[5].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 5 }, + [6] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[6].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 6 }, + [7] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[7].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 7 }, + [8] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[8].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 8 }, + [9] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[9].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 9 }, + [10] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[10].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 10 }, + [11] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[11].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 11 }, + [12] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[12].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 12 }, + [13] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[13].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 13 }, + [14] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[14].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 14 }, + [15] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[15].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 15 }, + [16] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[16].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 16 }, + [17] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[17].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 17 }, + [18] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[18].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 18 }, + [19] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[19].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 19 }, + [20] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[20].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 20 }, + [21] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[21].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 21 }, + [22] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[22].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 22 }, + [23] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[23].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 23 }, + [24] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[24].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 24 }, + [25] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[25].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 25 }, + [26] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[26].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 26 }, + [27] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[27].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 27 }, + [28] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[28].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 28 }, + [29] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[29].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 29 }, + [30] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[30].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 30 }, + [31] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[31].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 31 }, + [32] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[32].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 32 }, + [33] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[33].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 33 }, + [34] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[34].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 34 }, + [35] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[35].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 35 }, + [36] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[36].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 36 }, + [37] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[37].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 37 }, + [38] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[38].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 38 }, + [39] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[39].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 39 }, + [40] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[40].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 40 }, + [41] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[41].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 41 }, + [42] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[42].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 42 }, + [43] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[43].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 43 }, + [44] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[44].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 44 }, + [45] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[45].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 45 }, + [46] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[46].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 46 }, + [47] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[47].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 47 }, + [48] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[48].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 48 }, + [49] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[49].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 49 }, + [50] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[50].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 50 }, + [51] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[51].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 51 }, + [52] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[52].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 52 }, + [53] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[53].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 53 }, + [54] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[54].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 54 }, + [55] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[55].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 55 }, + [56] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[56].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 56 }, + [57] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[57].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 57 }, + [58] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[58].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 58 }, + [59] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[59].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 59 }, + [60] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[60].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 60 }, + [61] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[61].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 61 }, + [62] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[62].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 62 }, + [63] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[63].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 63 }, + [64] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[64].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 64 }, + [65] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[65].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 65 }, + [66] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[66].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 66 }, + [67] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[67].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 67 }, + [68] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[68].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 68 }, + [69] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[69].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 69 }, + [70] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[70].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 70 }, + [71] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[71].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 71 }, + [72] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[72].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 72 }, + [73] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[73].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 73 }, + [74] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[74].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 74 }, + [75] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[75].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 75 }, + [76] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[76].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 76 }, + [77] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[77].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 77 }, + [78] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[78].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 78 }, + [79] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[79].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 79 }, + [80] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[80].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 80 }, + [81] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[81].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 81 }, + [82] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[82].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 82 }, + [83] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[83].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 83 }, + [84] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[84].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 84 }, + [85] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[85].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 85 }, + [86] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[86].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 86 }, + [87] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[87].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 87 }, + [88] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[88].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 88 }, + [89] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[89].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 89 }, + [90] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[90].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 90 }, + [91] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[91].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 91 }, + [92] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[92].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 92 }, + [93] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[93].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 93 }, + [94] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[94].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 94 }, + [95] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[95].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 95 }, + [96] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[96].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 96 }, + [97] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[97].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 97 }, + [98] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[98].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 98 }, + [99] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[99].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 99 }, + [100] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[100].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 100 }, + [101] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[101].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 101 }, + [102] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[102].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 102 }, + [103] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[103].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 103 }, + [104] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[104].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 104 }, + [105] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[105].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 105 }, + [106] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[106].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 106 }, + [107] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[107].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 107 }, + [108] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[108].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 108 }, + [109] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[109].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 109 }, + [110] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[110].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 110 }, + [111] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[111].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 111 }, + [112] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[112].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 112 }, + [113] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[113].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 113 }, + [114] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[114].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 114 }, + [115] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[115].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 115 }, + [116] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[116].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 116 }, + [117] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[117].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 117 }, + [118] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[118].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 118 }, + [119] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[119].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 119 }, + [120] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[120].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 120 }, + [121] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[121].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 121 }, + [122] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[122].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 122 }, + [123] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[123].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 123 }, + [124] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[124].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 124 }, + [125] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[125].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 125 }, + [126] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[126].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 126 }, + [127] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[127].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 127 }, + [128] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[128].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 128 }, + [129] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[129].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 129 }, + [130] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[130].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 130 }, + [131] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[131].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 131 }, + [132] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[132].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 132 }, + [133] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[133].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 133 }, + [134] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[134].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 134 }, + [135] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[135].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 135 }, + [136] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[136].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 136 }, + [137] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[137].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 137 }, + [138] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[138].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 138 }, + [139] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[139].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 139 }, + [140] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[140].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 140 }, + [141] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[141].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 141 }, + [142] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[142].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 142 }, + [143] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[143].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 143 }, + [144] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[144].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 144 }, + [145] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[145].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 145 }, + [146] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[146].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 146 }, + [147] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[147].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 147 }, + [148] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[148].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 148 }, + [149] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[149].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 149 }, + [150] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[150].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 150 }, + [151] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[151].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 151 }, + [152] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[152].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 152 }, + [153] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[153].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 153 }, + [154] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[154].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 154 }, + [155] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[155].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 155 }, + [156] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[156].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 156 }, + [157] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[157].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 157 }, + [158] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[158].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 158 }, + [159] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[159].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 159 }, + [160] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[160].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 160 }, + [161] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[161].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 161 }, + [162] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[162].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 162 }, + [163] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[163].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 163 }, + [164] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[164].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 164 }, + [165] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[165].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 165 }, + [166] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[166].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 166 }, + [167] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[167].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 167 }, + [168] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[168].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 168 }, + [169] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[169].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 169 }, + [170] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[170].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 170 }, + [171] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[171].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 171 }, + [172] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[172].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 172 }, + [173] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[173].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 173 }, + [174] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[174].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 174 }, + [175] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[175].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 175 }, + [176] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[176].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 176 }, + [177] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[177].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 177 }, + [178] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[178].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 178 }, + [179] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[179].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 179 }, + [180] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[180].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 180 }, + [181] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[181].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 181 }, + [182] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[182].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 182 }, + [183] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[183].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 183 }, + [184] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[184].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 184 }, + [185] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[185].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 185 }, + [186] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[186].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 186 }, + [187] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[187].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 187 }, + [188] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[188].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 188 }, + [189] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[189].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 189 }, + [190] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[190].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 190 }, + [191] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[191].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 191 }, + [192] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[192].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 192 }, + [193] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[193].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 193 }, + [194] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[194].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 194 }, + [195] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[195].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 195 }, + [196] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[196].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 196 }, + [197] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[197].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 197 }, + [198] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[198].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 198 }, + [199] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[199].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 199 }, + [200] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[200].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 200 }, + [201] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[201].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 201 }, + [202] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[202].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 202 }, + [203] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[203].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 203 }, + [204] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[204].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 204 }, + [205] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[205].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 205 }, + [206] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[206].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 206 }, + [207] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[207].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 207 }, + [208] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[208].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 208 }, + [209] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[209].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 209 }, + [210] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[210].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 210 }, + [211] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[211].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 211 }, + [212] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[212].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 212 }, + [213] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[213].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 213 }, + [214] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[214].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 214 }, + [215] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[215].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 215 }, + [216] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[216].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 216 }, + [217] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[217].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 217 }, + [218] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[218].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 218 }, + [219] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[219].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 219 }, + [220] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[220].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 220 }, + [221] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[221].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 221 }, + [222] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[222].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 222 }, + [223] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[223].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 223 }, + [224] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[224].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 224 }, + [225] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[225].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 225 }, + [226] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[226].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 226 }, + [227] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[227].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 227 }, + [228] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[228].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 228 }, + [229] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[229].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 229 }, + [230] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[230].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 230 }, + [231] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[231].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 231 }, + [232] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[232].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 232 }, + [233] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[233].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 233 }, + [234] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[234].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 234 }, + [235] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[235].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 235 }, + [236] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[236].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 236 }, + [237] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[237].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 237 }, + [238] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[238].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 238 }, + [239] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[239].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 239 }, + [240] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[240].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 240 }, + [241] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[241].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 241 }, + [242] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[242].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 242 }, + [243] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[243].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 243 }, + [244] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[244].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 244 }, + [245] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[245].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 245 }, + [246] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[246].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 246 }, + [247] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[247].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 247 }, + [248] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[248].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 248 }, + [249] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[249].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 249 }, + [250] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[250].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 250 }, + [251] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[251].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 251 }, + [252] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[252].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 252 }, + [253] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[253].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 253 }, + [254] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[254].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 254 }, + [255] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[255].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 255 }, + [256] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[256].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 256 }, + [257] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[257].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 257 }, + [258] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[258].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 258 }, + [259] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[259].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 259 }, + [260] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[260].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 260 }, + [261] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[261].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 261 }, + [262] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[262].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 262 }, + [263] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[263].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 263 }, + [264] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[264].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 264 }, + [265] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[265].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 265 }, + [266] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[266].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 266 }, + [267] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[267].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 267 }, + [268] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[268].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 268 }, + [269] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[269].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 269 }, + [270] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[270].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 270 }, + [271] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[271].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 271 }, + [272] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[272].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 272 }, + [273] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[273].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 273 }, + [274] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[274].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 274 }, + [275] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[275].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 275 }, + [276] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[276].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 276 }, + [277] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[277].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 277 }, + [278] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[278].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 278 }, + [279] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[279].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 279 }, + [280] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[280].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 280 }, + [281] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[281].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 281 }, + [282] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[282].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 282 }, + [283] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[283].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 283 }, + [284] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[284].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 284 }, + [285] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[285].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 285 }, + [286] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[286].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 286 }, + [287] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[287].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 287 }, + [288] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[288].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 288 }, + [289] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[289].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 289 }, + [290] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[290].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 290 }, + [291] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[291].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 291 }, + [292] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[292].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 292 }, + [293] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[293].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 293 }, + [294] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[294].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 294 }, + [295] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[295].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 295 }, + [296] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[296].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 296 }, + [297] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[297].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 297 }, + [298] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[298].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 298 }, + [299] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[299].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 299 }, + [300] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[300].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 300 }, + [301] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[301].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 301 }, + [302] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[302].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 302 }, + [303] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[303].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 303 }, + [304] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[304].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 304 }, + [305] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[305].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 305 }, + [306] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[306].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 306 }, + [307] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[307].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 307 }, + [308] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[308].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 308 }, + [309] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[309].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 309 }, + [310] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[310].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 310 }, + [311] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[311].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 311 }, + [312] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[312].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 312 }, + [313] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[313].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 313 }, + [314] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[314].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 314 }, + [315] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[315].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 315 }, + [316] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[316].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 316 }, + [317] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[317].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 317 }, + [318] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[318].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 318 }, + [319] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[319].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 319 }, + [320] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[320].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 320 }, + [321] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[321].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 321 }, + [322] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[322].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 322 }, + [323] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[323].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 323 }, + [324] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[324].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 324 }, + [325] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[325].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 325 }, + [326] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[326].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 326 }, + [327] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[327].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 327 }, + [328] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[328].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 328 }, + [329] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[329].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 329 }, + [330] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[330].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 330 }, + [331] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[331].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 331 }, + [332] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[332].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 332 }, + [333] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[333].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 333 }, + [334] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[334].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 334 }, + [335] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[335].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 335 }, + [336] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[336].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 336 }, + [337] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[337].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 337 }, + [338] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[338].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 338 }, + [339] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[339].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 339 }, + [340] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[340].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 340 }, + [341] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[341].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 341 }, + [342] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[342].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 342 }, + [343] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[343].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 343 }, + [344] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[344].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 344 }, + [345] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[345].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 345 }, + [346] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[346].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 346 }, + [347] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[347].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 347 }, + [348] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[348].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 348 }, + [349] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[349].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 349 }, + [350] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[350].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 350 }, + [351] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[351].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 351 }, + [352] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[352].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 352 }, + [353] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[353].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 353 }, + [354] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[354].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 354 }, + [355] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[355].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 355 }, + [356] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[356].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 356 }, + [357] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[357].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 357 }, + [358] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[358].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 358 }, + [359] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[359].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 359 }, + [360] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[360].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 360 }, + [361] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[361].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 361 }, + [362] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[362].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 362 }, + [363] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[363].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 363 }, + [364] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[364].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 364 }, + [365] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[365].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 365 }, + [366] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[366].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 366 }, + [367] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[367].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 367 }, + [368] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[368].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 368 }, + [369] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[369].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 369 }, + [370] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[370].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 370 }, + [371] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[371].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 371 }, + [372] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[372].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 372 }, + [373] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[373].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 373 }, + [374] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[374].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 374 }, + [375] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[375].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 375 }, + [376] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[376].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 376 }, + [377] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[377].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 377 }, + [378] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[378].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 378 }, + [379] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[379].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 379 }, + [380] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[380].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 380 }, + [381] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[381].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 381 }, + [382] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[382].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 382 }, + [383] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[383].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 383 }, + [384] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[384].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 384 }, + [385] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[385].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 385 }, + [386] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[386].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 386 }, + [387] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[387].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 387 }, + [388] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[388].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 388 }, + [389] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[389].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 389 }, + [390] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[390].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 390 }, + [391] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[391].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 391 }, + [392] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[392].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 392 }, + [393] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[393].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 393 }, + [394] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[394].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 394 }, + [395] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[395].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 395 }, + [396] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[396].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 396 }, + [397] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[397].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 397 }, + [398] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[398].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 398 }, + [399] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[399].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 399 }, + [400] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[400].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 400 }, + [401] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[401].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 401 }, + [402] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[402].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 402 }, + [403] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[403].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 403 }, + [404] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[404].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 404 }, + [405] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[405].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 405 }, + [406] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[406].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 406 }, + [407] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[407].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 407 }, + [408] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[408].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 408 }, + [409] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[409].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 409 }, + [410] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[410].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 410 }, + [411] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[411].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 411 }, + [412] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[412].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 412 }, + [413] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[413].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 413 }, + [414] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[414].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 414 }, + [415] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[415].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 415 }, + [416] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[416].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 416 }, + [417] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[417].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 417 }, + [418] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[418].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 418 }, + [419] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[419].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 419 }, + [420] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[420].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 420 }, + [421] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[421].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 421 }, + [422] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[422].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 422 }, + [423] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[423].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 423 }, + [424] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[424].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 424 }, + [425] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[425].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 425 }, + [426] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[426].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 426 }, + [427] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[427].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 427 }, + [428] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[428].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 428 }, + [429] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[429].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 429 }, + [430] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[430].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 430 }, + [431] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[431].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 431 }, + [432] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[432].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 432 }, + [433] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[433].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 433 }, + [434] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[434].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 434 }, + [435] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[435].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 435 }, + [436] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[436].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 436 }, + [437] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[437].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 437 }, + [438] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[438].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 438 }, + [439] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[439].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 439 }, + [440] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[440].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 440 }, + [441] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[441].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 441 }, + [442] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[442].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 442 }, + [443] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[443].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 443 }, + [444] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[444].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 444 }, + [445] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[445].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 445 }, + [446] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[446].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 446 }, + [447] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[447].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 447 }, + [448] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[448].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 448 }, + [449] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[449].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 449 }, + [450] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[450].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 450 }, + [451] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[451].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 451 }, + [452] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[452].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 452 }, + [453] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[453].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 453 }, + [454] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[454].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 454 }, + [455] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[455].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 455 }, + [456] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[456].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 456 }, + [457] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[457].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 457 }, + [458] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[458].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 458 }, + [459] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[459].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 459 }, + [460] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[460].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 460 }, + [461] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[461].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 461 }, + [462] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[462].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 462 }, + [463] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[463].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 463 }, + [464] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[464].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 464 }, + [465] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[465].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 465 }, + [466] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[466].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 466 }, + [467] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[467].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 467 }, + [468] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[468].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 468 }, + [469] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[469].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 469 }, + [470] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[470].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 470 }, + [471] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[471].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 471 }, + [472] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[472].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 472 }, + [473] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[473].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 473 }, + [474] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[474].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 474 }, + [475] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[475].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 475 }, + [476] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[476].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 476 }, + [477] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[477].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 477 }, + [478] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[478].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 478 }, + [479] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[479].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 479 }, + [480] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[480].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 480 }, + [481] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[481].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 481 }, + [482] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[482].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 482 }, + [483] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[483].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 483 }, + [484] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[484].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 484 }, + [485] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[485].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 485 }, + [486] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[486].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 486 }, + [487] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[487].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 487 }, + [488] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[488].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 488 }, + [489] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[489].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 489 }, + [490] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[490].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 490 }, + [491] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[491].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 491 }, + [492] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[492].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 492 }, + [493] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[493].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 493 }, + [494] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[494].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 494 }, + [495] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[495].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 495 }, + [496] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[496].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 496 }, + [497] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[497].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 497 }, + [498] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[498].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 498 }, + [499] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[499].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 499 }, + [500] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[500].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 500 }, + [501] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[501].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 501 }, + [502] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[502].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 502 }, + [503] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[503].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 503 }, + [504] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[504].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 504 }, + [505] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[505].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 505 }, + [506] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[506].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 506 }, + [507] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[507].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 507 }, + [508] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[508].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 508 }, + [509] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[509].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 509 }, + [510] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[510].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 510 }, + [511] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[511].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 511 }, +}; From 2172d680218d913746d986a3b9186c6b12f416a1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Jan 2024 21:04:36 +0000 Subject: [PATCH 08/40] Treat EXIT_TRACE as a side exit --- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 2 +- Python/optimizer.c | 14 ++++++-------- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index c39e1b2b80ee9c..4d9c8cfeee33a0 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -200,7 +200,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, [_SET_IP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_DEOPT_FLAG, + [_EXIT_TRACE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE_BORROW] = 0, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3f50e7cf091416..131ae1bb709d67 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4065,7 +4065,7 @@ dummy_func( op(_EXIT_TRACE, (--)) { TIER_TWO_ONLY - DEOPT_IF(1); + EXIT_IF(1); } op(_CHECK_VALIDITY, (--)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 52e3ba7cf31d0a..b9a8ee882195c3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3378,7 +3378,7 @@ case _EXIT_TRACE: { TIER_TWO_ONLY - if (1) goto deoptimize; + if (1) goto side_exit; break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index f0b737f54534c3..15822a3d851d7c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -108,16 +108,14 @@ PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutor } static int -error_optimize( +never_optimize( _PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **exec, int Py_UNUSED(stack_entries)) { - assert(0); - PyErr_Format(PyExc_SystemError, "Should never call error_optimize"); - return -1; + return 0; } PyTypeObject _PyDefaultOptimizer_Type = { @@ -130,7 +128,7 @@ PyTypeObject _PyDefaultOptimizer_Type = { _PyOptimizerObject _PyOptimizer_Default = { PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type) - .optimize = error_optimize, + .optimize = never_optimize, .resume_threshold = INT16_MAX, .backedge_threshold = INT16_MAX, }; @@ -773,6 +771,9 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) } count++; int opcode = buffer[i].opcode; + if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + exit_count++; + } if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { continue; } @@ -783,9 +784,6 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) /* Mark target as reachable */ SET_BIT(used, buffer[i].oparg); } - if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { - exit_count++; - } if (opcode == NOP) { count--; UNSET_BIT(used, i); From 44487933ee8b6ee5d2b9f52352b6b67f0fd9fc2a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Jan 2024 21:49:38 +0000 Subject: [PATCH 09/40] Treat most common guard failures as side exits --- Include/internal/pycore_opcode_metadata.h | 48 +++++++++++------------ Include/internal/pycore_uop_metadata.h | 20 +++++----- Python/bytecodes.c | 26 ++++++------ Python/executor_cases.c.h | 26 ++++++------ 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 146625230e3a86..b420645cc73d52 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -947,14 +947,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1062,16 +1062,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1120,8 +1120,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1135,12 +1135,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 4d9c8cfeee33a0..9ed63a249e7791 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -32,22 +32,22 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, - [_TO_BOOL_INT] = HAS_DEOPT_FLAG, - [_TO_BOOL_LIST] = HAS_DEOPT_FLAG, - [_TO_BOOL_NONE] = HAS_DEOPT_FLAG, - [_TO_BOOL_STR] = HAS_DEOPT_FLAG, - [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG, + [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_TO_BOOL_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, - [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -112,7 +112,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 131ae1bb709d67..5f3b35c3393b77 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -340,12 +340,12 @@ dummy_func( macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL; inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { - DEOPT_IF(!PyBool_Check(value)); + EXIT_IF(!PyBool_Check(value)); STAT_INC(TO_BOOL, hit); } inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyLong_CheckExact(value)); + EXIT_IF(!PyLong_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -358,7 +358,7 @@ dummy_func( } inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyList_CheckExact(value)); + EXIT_IF(!PyList_CheckExact(value)); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; DECREF_INPUTS(); @@ -366,13 +366,13 @@ dummy_func( inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value)); + EXIT_IF(!Py_IsNone(value)); STAT_INC(TO_BOOL, hit); res = Py_False; } inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(value)); + EXIT_IF(!PyUnicode_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -388,7 +388,7 @@ dummy_func( inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version); + EXIT_IF(Py_TYPE(value)->tp_version_tag != version); STAT_INC(TO_BOOL, hit); DECREF_INPUTS(); res = Py_True; @@ -412,8 +412,8 @@ dummy_func( }; op(_GUARD_BOTH_INT, (left, right -- left: &PYLONG_TYPE, right: &PYLONG_TYPE)) { - DEOPT_IF(!PyLong_CheckExact(left)); - DEOPT_IF(!PyLong_CheckExact(right)); + EXIT_IF(!PyLong_CheckExact(left)); + EXIT_IF(!PyLong_CheckExact(right)); } pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res: &PYLONG_TYPE)) { @@ -448,8 +448,8 @@ dummy_func( _GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT; op(_GUARD_BOTH_FLOAT, (left, right -- left: &PYFLOAT_TYPE, right: &PYFLOAT_TYPE)) { - DEOPT_IF(!PyFloat_CheckExact(left)); - DEOPT_IF(!PyFloat_CheckExact(right)); + EXIT_IF(!PyFloat_CheckExact(left)); + EXIT_IF(!PyFloat_CheckExact(right)); } pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res: &PYFLOAT_TYPE)) { @@ -484,8 +484,8 @@ dummy_func( _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT; op(_GUARD_BOTH_UNICODE, (left, right -- left: &PYUNICODE_TYPE, right: &PYUNICODE_TYPE)) { - DEOPT_IF(!PyUnicode_CheckExact(left)); - DEOPT_IF(!PyUnicode_CheckExact(right)); + EXIT_IF(!PyUnicode_CheckExact(left)); + EXIT_IF(!PyUnicode_CheckExact(right)); } pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res: &PYUNICODE_TYPE)) { @@ -1904,7 +1904,7 @@ dummy_func( op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner: &(GUARD_TYPE_VERSION_TYPE + type_version))) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version); + EXIT_IF(tp->tp_version_tag != type_version); } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b9a8ee882195c3..c326bad2861018 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -141,7 +141,7 @@ case _TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - if (!PyBool_Check(value)) goto deoptimize; + if (!PyBool_Check(value)) goto side_exit; STAT_INC(TO_BOOL, hit); break; } @@ -150,7 +150,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) goto deoptimize; + if (!PyLong_CheckExact(value)) goto side_exit; STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -168,7 +168,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyList_CheckExact(value)) goto deoptimize; + if (!PyList_CheckExact(value)) goto side_exit; STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -181,7 +181,7 @@ PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - if (!Py_IsNone(value)) goto deoptimize; + if (!Py_IsNone(value)) goto side_exit; STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -192,7 +192,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyUnicode_CheckExact(value)) goto deoptimize; + if (!PyUnicode_CheckExact(value)) goto side_exit; STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -214,7 +214,7 @@ uint32_t version = (uint32_t)CURRENT_OPERAND(); // This one is a bit weird, because we expect *some* failures: assert(version); - if (Py_TYPE(value)->tp_version_tag != version) goto deoptimize; + if (Py_TYPE(value)->tp_version_tag != version) goto side_exit; STAT_INC(TO_BOOL, hit); Py_DECREF(value); res = Py_True; @@ -238,8 +238,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) goto deoptimize; - if (!PyLong_CheckExact(right)) goto deoptimize; + if (!PyLong_CheckExact(left)) goto side_exit; + if (!PyLong_CheckExact(right)) goto side_exit; break; } @@ -296,8 +296,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) goto deoptimize; - if (!PyFloat_CheckExact(right)) goto deoptimize; + if (!PyFloat_CheckExact(left)) goto side_exit; + if (!PyFloat_CheckExact(right)) goto side_exit; break; } @@ -354,8 +354,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) goto deoptimize; - if (!PyUnicode_CheckExact(right)) goto deoptimize; + if (!PyUnicode_CheckExact(left)) goto side_exit; + if (!PyUnicode_CheckExact(right)) goto side_exit; break; } @@ -1623,7 +1623,7 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - if (tp->tp_version_tag != type_version) goto deoptimize; + if (tp->tp_version_tag != type_version) goto side_exit; break; } From c70f12f7e54c0a7eeac3b078505e7d81b53a58f2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Jan 2024 22:14:46 +0000 Subject: [PATCH 10/40] Tweak generated tble to help C analyzer --- Python/optimizer.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 15822a3d851d7c..32ea69ebc01634 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1237,8 +1237,8 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) } /* The following table is generated from this script: -print(f'static_assert(UOP_MAX_TRACE_LENGTH == {UOP_MAX_TRACE_LENGTH}, "COLD_EXITS must be regenerated");') -print("static _PyColdExitObject COLD_EXITS[{UOP_MAX_TRACE_LENGTH}] = {") +print(f'static_assert(UOP_MAX_TRACE_LENGTH == {UOP_MAX_TRACE_LENGTH}, "COLD_EXITS must be regenerated");\n') +print("static _PyColdExitObject COLD_EXITS[] = {") for i in range(UOP_MAX_TRACE_LENGTH): print(f" [{i}] = {{ .base = {{ PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = {{ 0 }}," f" .trace = &COLD_EXITS[{i}].uop }}, .uop.opcode = _COLD_EXIT, .uop.oparg = {i} }},") @@ -1246,7 +1246,8 @@ print("};") */ static_assert(UOP_MAX_TRACE_LENGTH == 512, "COLD_EXITS must be regenerated"); -static _PyColdExitObject COLD_EXITS[512] = { + +static _PyColdExitObject COLD_EXITS[] = { [0] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[0].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 0 }, [1] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[1].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 1 }, [2] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[2].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 2 }, From d73fe0a0dc3b3e323e6bbd4614d0b4f76df90b65 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 16 Jan 2024 23:52:45 +0000 Subject: [PATCH 11/40] Add some documentation about the tier 2 engine --- Python/tier2_engine.md | 126 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 Python/tier2_engine.md diff --git a/Python/tier2_engine.md b/Python/tier2_engine.md new file mode 100644 index 00000000000000..d23f88918959fd --- /dev/null +++ b/Python/tier2_engine.md @@ -0,0 +1,126 @@ +# The tier 2 execution engine + +## General idea + +When execution in tier 1 becomes "hot", that is the counter for that point in +the code reaches some threshold, we create an executor and execute that +instead of the tier 1 bytecode. + +Since each executor must exit, we also track the "hotness" of those +exits and attach new executors to those exits. + +That way we form a graph of executors that covers the most frequently +executed parts of the program. + +## Superblocks and Executors + +Once a point in the code has become hot enough, we want to optimize it. +Starting from that point we project the likely path of execution, +using information gathered by tier 1 to guide that projection to +form a "superblock", a mostly linear sequence of micro-ops. +Although mostly linear, it may include a single loop. + +We then optimize this superblock to from an optimized superblock, +which is equivalent but more efficient. + +A superblock is a representation of the code we want to execute, +but it is not in executable form. +The executable form is know as an executor. + +Executors are semantically equivalent to the superblock they are +created from, but are in a form that can be efficiently executable. + +There are two execution engines for executors, adn two types of executors: +* The hardware which runs machine code executors created by the JIT compiler. +* The tier 2 interpreter runs bytecode executors. + +The choice of engine is a configuration option. +We will not support both the tier 2 interpreter and JIT in a +single executable. + +### Tier 2 Interpreter + +For platforms without a JIT and for testing, we need an interpreter +for executors. It is similar in design to the tier 1 interpreter, but has a +different instruction set, and does not adapt. + +### JIT compiler + +The JIT compiler converts superblocks into machine code executors. +These have identical behavior to interpreted executors, except that +they consume a bit more memory and are a lot faster. + +## Transfering control + +There are three types of control transfer that we need to consider: +* Tier 1 to tier 2 +* Tier 2 to tier 1 +* One executor to another within tier 2 + +Since we expect the graph of executors to span most of the hot +part of the program, transfers from one executor to another should +be the most common. +Therefore, we want to make those transfers fast. + +### Tier 2 to tier 2 + +#### Handling reference counts + +There must be an implicit reference to the currently executing +executor, otherwise it might be freed. +Consequently, we must increment the reference count of an +executor just before executing it, and decrement it just after +executing it. + +Note that since executors are objects, they can contain references to +themselves, which means we do not need to pass a reference to an +executor when we start to execute it. +However, we do need to pass a reference to the previous executor, +so that it can be decref'd. + +#### The interpreter + +The tier 2 interpreter has a variable `current_executor` which +points to the currently live executor. When transfering from executor +`A` to executor `B` we do the following: +(Initially `current_executor` points to `A`, and the refcount of +`A` is elevated by one) + +1. Set the instruction pointer to start at the beginning of `B` +2. Increment the reference count of `B` +3. Start executing `B` + +We also make the first instruction in `B` do the following: +1. Decrement the reference count of `A` (through `current_executor`) +2. Set `current_executor` to point to `B` + +The net effect of the above is to safely decrement the refcount of `A`, +increment the refcount of `B` and set `current_executor` to point to `B`. + +#### In the JIT + +Transfering control form one executor to another is done via tailcalls. + +The compiled executor should do the same, except that there is no local +variable `current_executor`, so that the old executor, `A`, must be passed +as an additional parameter when tailcalling. + +### Tier 1 to tier 2 + +We create a single, immortal executor that cannot be executed. +We can then perform a tier 1 to tier 2 transfer, +by setting `current_executor` to this singleton, and then performing +a tier 2 to tier 2 transfer as above. + +### Tier 2 to tier 1 + +Each micro-op that might exit to tier 1 contains a `target` value, +which is the offset of the tier 1 instruction to exit to in the +current code object. + +## Counters + +TO DO. +The implementation will change soon, so there is no point in +documenting it until then. + From 5c8f0bd0ed81e59425dcfad575b72bfa26632905 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Jan 2024 03:27:28 +0000 Subject: [PATCH 12/40] Fix constness and rename hotness --- Include/cpython/optimizer.h | 4 ++-- Python/bytecodes.c | 6 +++--- Python/ceval.c | 6 +++--- Python/executor_cases.c.h | 6 +++--- Python/optimizer.c | 24 ++++++++++++------------ Tools/c-analyzer/cpython/_parser.py | 1 + 6 files changed, 24 insertions(+), 23 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index b7028f44c195a2..4f772492e69af6 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -38,13 +38,13 @@ typedef struct { typedef struct _exit_data { uint32_t target; - int16_t hotness; + int16_t temperature; struct _PyExecutorObject *executor; } _PyExitData; typedef struct _PyExecutorObject { PyObject_VAR_HEAD - _PyUOpInstruction *trace; + const _PyUOpInstruction *trace; _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ uint32_t exit_count; uint32_t code_size; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5f3b35c3393b77..7a46212940a67a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4087,11 +4087,11 @@ dummy_func( TIER_TWO_ONLY assert(current_executor->trace[0].opcode != _COLD_EXIT); _PyExitData *exit = ¤t_executor->exits[oparg]; - exit->hotness++; + exit->temperature++; assert(exit->executor->trace[0].opcode == _COLD_EXIT); PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - if (exit->hotness < 0) { + if (exit->temperature < 0) { next_instr = target; Py_DECREF(current_executor); current_executor = NULL; @@ -4104,7 +4104,7 @@ dummy_func( } else { int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { - exit->hotness = -10000; /* Choose a better number */ + exit->temperature = -10000; /* Choose a better number */ Py_DECREF(current_executor); current_executor = NULL; next_instr = target; diff --git a/Python/ceval.c b/Python/ceval.c index 0ff10c92c3e2ed..0e42917987e130 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -745,7 +745,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *next_instr; PyObject **stack_pointer; - _PyUOpInstruction *next_uop = NULL; + const _PyUOpInstruction *next_uop = NULL; start_frame: @@ -1081,8 +1081,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int UOP_STAT_INC(uopcode, miss); uint32_t exit_id = next_uop[-1].target; _PyExitData *exit = ¤t_executor->exits[exit_id]; - DPRINTF(2, "SIDE EXIT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", exit %u, hotness %d, target %d -> %s]\n", - uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit_id, exit->hotness, exit->target, + DPRINTF(2, "SIDE EXIT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", exit %u, temp %d, target %d -> %s]\n", + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit_id, exit->temperature, exit->target, _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); Py_INCREF(exit->executor); next_uop = exit->executor->trace; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c326bad2861018..1f6329ef1b4656 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3411,11 +3411,11 @@ TIER_TWO_ONLY assert(current_executor->trace[0].opcode != _COLD_EXIT); _PyExitData *exit = ¤t_executor->exits[oparg]; - exit->hotness++; + exit->temperature++; assert(exit->executor->trace[0].opcode == _COLD_EXIT); PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - if (exit->hotness < 0) { + if (exit->temperature < 0) { next_instr = target; Py_DECREF(current_executor); current_executor = NULL; @@ -3428,7 +3428,7 @@ } else { int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { - exit->hotness = -10000; /* Choose a better number */ + exit->temperature = -10000; /* Choose a better number */ Py_DECREF(current_executor); current_executor = NULL; next_instr = target; diff --git a/Python/optimizer.c b/Python/optimizer.c index 32ea69ebc01634..c75dcc17b2f5d9 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -812,7 +812,7 @@ PyTypeObject _ColdExit_Type = { .tp_methods = executor_methods, }; -static _PyColdExitObject COLD_EXITS[UOP_MAX_TRACE_LENGTH]; +static const _PyColdExitObject COLD_EXITS[UOP_MAX_TRACE_LENGTH]; static _PyExecutorObject * allocate_executor(int exit_count, int length) @@ -855,38 +855,38 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) /* Initialize exits */ for (int i = 0; i < exit_count; i++) { executor->exits[i].executor = (_PyExecutorObject *)&COLD_EXITS[i]; - executor->exits[i].hotness = -67; + executor->exits[i].temperature = -67; } int next_exit = exit_count-1; - int dest = length; + _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */ for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) { if (!BIT_IS_SET(used, i)) { continue; } - executor->trace[dest] = buffer[i]; + *dest = buffer[i]; int opcode = buffer[i].opcode; if (opcode == _POP_JUMP_IF_FALSE || opcode == _POP_JUMP_IF_TRUE) { /* The oparg of the target will already have been set to its new offset */ - int oparg = executor->trace[dest].oparg; - executor->trace[dest].oparg = buffer[oparg].oparg; + int oparg = dest->oparg; + dest->oparg = buffer[oparg].oparg; } if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { executor->exits[next_exit].target = buffer[i].target; - executor->trace[dest].target = next_exit; + dest->target = next_exit; next_exit--; } /* Set the oparg to be the destination offset, * so that we can set the oparg of earlier jumps correctly. */ - buffer[i].oparg = dest; + buffer[i].oparg = dest - executor->trace; dest--; } assert(next_exit == -1); - assert(dest == 0); - executor->trace[0].opcode = _START_EXECUTOR; - executor->trace[0].operand = (uintptr_t)executor; + assert(dest == executor->trace); + dest->opcode = _START_EXECUTOR; + dest->operand = (uintptr_t)executor; _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -1247,7 +1247,7 @@ print("};") static_assert(UOP_MAX_TRACE_LENGTH == 512, "COLD_EXITS must be regenerated"); -static _PyColdExitObject COLD_EXITS[] = { +static const _PyColdExitObject COLD_EXITS[] = { [0] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[0].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 0 }, [1] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[1].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 1 }, [2] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[2].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 2 }, diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 444063d2148934..fad4b1e9e7a970 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -318,6 +318,7 @@ def clean_lines(text): _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), _abs('Objects/typeobject.c'): (35_000, 200), _abs('Python/compile.c'): (20_000, 500), + _abs('Python/optimizer.c'): (100_000, 5_000), _abs('Python/parking_lot.c'): (40_000, 1000), _abs('Python/pylifecycle.c'): (500_000, 5000), _abs('Python/pystate.c'): (500_000, 5000), From 140486b05f5d7cec8290c95a30b74006a5b05b2a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 17 Jan 2024 03:50:39 +0000 Subject: [PATCH 13/40] Add new static objects to ignored file. --- Python/optimizer.c | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index c75dcc17b2f5d9..1eb715aadb0676 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -801,7 +801,7 @@ cold_dealloc(_PyExecutorObject *self) { PyObject_Free(self); } -PyTypeObject _ColdExit_Type = { +static PyTypeObject _ColdExit_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "cold_exit", .tp_basicsize = offsetof(_PyExecutorObject, exits), diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 2f9e80d6ab6737..c4b9a1d371cc2c 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -381,6 +381,9 @@ Python/optimizer.c - _PyCounterOptimizer_Type - Python/optimizer.c - _PyUOpExecutor_Type - Python/optimizer.c - _PyUOpOptimizer_Type - Python/optimizer.c - _PyOptimizer_Default - +Python/optimizer.c - _ColdExit_Type - +Python/optimizer.c - COLD_EXITS - +Python/optimizer.c - Py_FatalErrorExecutor - ##----------------------- ## test code From 3362c9322a3baa723442c75a121d33f63165f116 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 18 Jan 2024 01:25:49 +0000 Subject: [PATCH 14/40] Address review comments --- Include/cpython/optimizer.h | 7 +++-- Python/bytecodes.c | 11 +++---- Python/ceval.c | 6 ++-- Python/executor_cases.c.h | 3 +- Python/generated_cases.c.h | 8 ++--- Python/optimizer.c | 6 ++-- Python/tier2_engine.md | 59 ++++++++++++++++++++++++++----------- 7 files changed, 64 insertions(+), 36 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 4f772492e69af6..92d9a649a04a75 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -32,7 +32,10 @@ typedef struct { typedef struct { uint16_t opcode; uint16_t oparg; - uint32_t target; + union { + uint32_t target; + uint16_t exit_index; + }; uint64_t operand; // A cache entry } _PyUOpInstruction; @@ -57,7 +60,7 @@ typedef struct _cold_exit { } _PyColdExitObject; -extern _PyColdExitObject Py_FatalErrorExecutor; +extern _PyColdExitObject Py_NeverExecutedExecutor; typedef struct _PyOptimizerObject _PyOptimizerObject; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7a46212940a67a..8c943f465e387d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2338,7 +2338,7 @@ dummy_func( ERROR_IF(optimized < 0, error); if (optimized) { assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)&Py_FatalErrorExecutor; + current_executor = (_PyExecutorObject *)&Py_NeverExecutedExecutor; next_uop = executor->trace; GOTO_TIER_TWO(); } @@ -2374,7 +2374,7 @@ dummy_func( _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; if (executor->vm_data.valid) { assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)&Py_FatalErrorExecutor; + current_executor = (_PyExecutorObject *)&Py_NeverExecutedExecutor; Py_INCREF(executor); next_uop = executor->trace; GOTO_TIER_TWO(); @@ -4005,13 +4005,13 @@ dummy_func( inst(CACHE, (--)) { TIER_ONE_ONLY - assert(0); + assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); } inst(RESERVED, (--)) { TIER_ONE_ONLY - assert(0); + assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); } @@ -4120,8 +4120,9 @@ dummy_func( op(_START_EXECUTOR, (executor/4 --)) { TIER_TWO_ONLY - Py_DECREF(current_executor); + _PyExecutorObject *old = current_executor; current_executor = (_PyExecutorObject*)executor; + Py_DECREF(old); } op(_FATAL_ERROR, (--)) { diff --git a/Python/ceval.c b/Python/ceval.c index 0e42917987e130..b9dde73a3f2e16 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1079,10 +1079,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int side_exit: OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); - uint32_t exit_id = next_uop[-1].target; - _PyExitData *exit = ¤t_executor->exits[exit_id]; + uint16_t exit_index = next_uop[-1].exit_index; + _PyExitData *exit = ¤t_executor->exits[exit_index]; DPRINTF(2, "SIDE EXIT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", exit %u, temp %d, target %d -> %s]\n", - uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit_id, exit->temperature, exit->target, + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit_index, exit->temperature, exit->target, _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); Py_INCREF(exit->executor); next_uop = exit->executor->trace; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1f6329ef1b4656..540755cf85c2cf 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3446,8 +3446,9 @@ case _START_EXECUTOR: { PyObject *executor = (PyObject *)CURRENT_OPERAND(); TIER_TWO_ONLY - Py_DECREF(current_executor); + _PyExecutorObject *old = current_executor; current_executor = (_PyExecutorObject*)executor; + Py_DECREF(old); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5b4c5c23297c4b..1425c2e0c3a0e7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -740,7 +740,7 @@ next_instr += 1; INSTRUCTION_STATS(CACHE); TIER_ONE_ONLY - assert(0); + assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); DISPATCH(); } @@ -2381,7 +2381,7 @@ _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; if (executor->vm_data.valid) { assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)&Py_FatalErrorExecutor; + current_executor = (_PyExecutorObject *)&Py_NeverExecutedExecutor; Py_INCREF(executor); next_uop = executor->trace; GOTO_TIER_TWO(); @@ -3306,7 +3306,7 @@ if (optimized < 0) goto error; if (optimized) { assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)&Py_FatalErrorExecutor; + current_executor = (_PyExecutorObject *)&Py_NeverExecutedExecutor; next_uop = executor->trace; GOTO_TIER_TWO(); } @@ -4796,7 +4796,7 @@ next_instr += 1; INSTRUCTION_STATS(RESERVED); TIER_ONE_ONLY - assert(0); + assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); DISPATCH(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index 1eb715aadb0676..58da12a3e26378 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -828,11 +828,11 @@ allocate_executor(int exit_count, int length) return res; } -_PyColdExitObject Py_FatalErrorExecutor = { +_PyColdExitObject Py_NeverExecutedExecutor = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, - .trace = &Py_FatalErrorExecutor.uop + .trace = &Py_NeverExecutedExecutor.uop }, .uop.opcode = _FATAL_ERROR, }; @@ -875,7 +875,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) } if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { executor->exits[next_exit].target = buffer[i].target; - dest->target = next_exit; + dest->exit_index = next_exit; next_exit--; } /* Set the oparg to be the destination offset, diff --git a/Python/tier2_engine.md b/Python/tier2_engine.md index d23f88918959fd..f9a06f77d1e495 100644 --- a/Python/tier2_engine.md +++ b/Python/tier2_engine.md @@ -9,8 +9,8 @@ instead of the tier 1 bytecode. Since each executor must exit, we also track the "hotness" of those exits and attach new executors to those exits. -That way we form a graph of executors that covers the most frequently -executed parts of the program. +As the program executes, and the hot parts of the program get optimized, +a graph of executors will form. ## Superblocks and Executors @@ -20,7 +20,7 @@ using information gathered by tier 1 to guide that projection to form a "superblock", a mostly linear sequence of micro-ops. Although mostly linear, it may include a single loop. -We then optimize this superblock to from an optimized superblock, +We then optimize this superblock to form an optimized superblock, which is equivalent but more efficient. A superblock is a representation of the code we want to execute, @@ -30,13 +30,15 @@ The executable form is know as an executor. Executors are semantically equivalent to the superblock they are created from, but are in a form that can be efficiently executable. -There are two execution engines for executors, adn two types of executors: +There are two execution engines for executors, and two types of executors: * The hardware which runs machine code executors created by the JIT compiler. * The tier 2 interpreter runs bytecode executors. -The choice of engine is a configuration option. -We will not support both the tier 2 interpreter and JIT in a -single executable. +It would be very wasteful to support both a tier 2 interpreter and +JIT compiler in the same process. +For now, we will make the choice of engine a configuration option, +but we could make it a command line option in the future if that would prove useful. + ### Tier 2 Interpreter @@ -64,6 +66,18 @@ Therefore, we want to make those transfers fast. ### Tier 2 to tier 2 +#### Cold exits + +All side exits start cold and most stay cold, but a few become +hot. We want to keep the memory consumption small for the many +cold exits, but those that become hot need to be fast. +However we cannot know in advance, which will be which. + +So that tier 2 to tier 2 transfers are fast for hot exits, +exits must be implemented as executors. In order to patch +executor exits when they get hot, a pointer to the current +executor must be passed to the exit executor. + #### Handling reference counts There must be an implicit reference to the currently executing @@ -72,11 +86,17 @@ Consequently, we must increment the reference count of an executor just before executing it, and decrement it just after executing it. -Note that since executors are objects, they can contain references to -themselves, which means we do not need to pass a reference to an -executor when we start to execute it. +We want to minimize the amount of data that is passed from +one executor to the next. In the JIT, this reduces the number +of arguments in the tailcall, freeing up registers for other uses. +It is less important in the interpreter, but follwing the same +design as the JIT simplifies debugging and is good for performance. + +Provided that we incref the new executor before executing it, we +can jump directly to the code of the executor, without needing +to pass a reference to that executor object. However, we do need to pass a reference to the previous executor, -so that it can be decref'd. +so that it can be decref'd and for handling of cold exits. #### The interpreter @@ -91,15 +111,15 @@ points to the currently live executor. When transfering from executor 3. Start executing `B` We also make the first instruction in `B` do the following: -1. Decrement the reference count of `A` (through `current_executor`) -2. Set `current_executor` to point to `B` +1. Set `current_executor` to point to `B` +2. Decrement the reference count of `A` The net effect of the above is to safely decrement the refcount of `A`, increment the refcount of `B` and set `current_executor` to point to `B`. #### In the JIT -Transfering control form one executor to another is done via tailcalls. +Transfering control from one executor to another is done via tailcalls. The compiled executor should do the same, except that there is no local variable `current_executor`, so that the old executor, `A`, must be passed @@ -107,10 +127,13 @@ as an additional parameter when tailcalling. ### Tier 1 to tier 2 -We create a single, immortal executor that cannot be executed. -We can then perform a tier 1 to tier 2 transfer, -by setting `current_executor` to this singleton, and then performing -a tier 2 to tier 2 transfer as above. +Since the executor doesn't know if the previous code was tier 1 or tier 2, +we need to make a transfer from tier 1 to tier 2 look like a tier 2 to tier 2 +transfer to the executor. + +To do that, we create a single, immortal executor that cannot be executed. +We can then perform a tier 1 to tier 2 transfer by setting `current_executor` +to this singleton, and then performing a tier 2 to tier 2 transfer as above. ### Tier 2 to tier 1 From 63fe653cec52aec71b42739c7ceebabbad6d66aa Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 Feb 2024 20:08:24 +0000 Subject: [PATCH 15/40] Transfer executor on thread-state and othe minor changes to be more jit-friendly --- Include/cpython/pystate.h | 2 ++ Python/bytecodes.c | 36 ++++++++++++++++++------------------ Python/ceval.c | 9 +++++---- Python/ceval_macros.h | 2 +- Python/executor_cases.c.h | 22 ++++++++++++---------- Python/generated_cases.c.h | 14 ++++++-------- Python/pystate.c | 1 + 7 files changed, 45 insertions(+), 41 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ed7dd829d4b6f0..818abc1d154346 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -212,6 +212,8 @@ struct _ts { /* The thread's exception stack entry. (Always the last entry.) */ _PyErr_StackItem exc_state; + PyObject *previous_executor; + }; #ifdef Py_DEBUG diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8c943f465e387d..e764fae788732a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2337,10 +2337,9 @@ dummy_func( int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); ERROR_IF(optimized < 0, error); if (optimized) { - assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)&Py_NeverExecutedExecutor; - next_uop = executor->trace; - GOTO_TIER_TWO(); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } else { int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); @@ -2373,11 +2372,10 @@ dummy_func( PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; if (executor->vm_data.valid) { - assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)&Py_NeverExecutedExecutor; + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; Py_INCREF(executor); - next_uop = executor->trace; - GOTO_TIER_TWO(); + GOTO_TIER_TWO(executor); } else { /* ENTER_EXECUTOR will be the first code unit of the instruction */ @@ -4085,16 +4083,17 @@ dummy_func( op(_COLD_EXIT, (--)) { TIER_TWO_ONLY - assert(current_executor->trace[0].opcode != _COLD_EXIT); - _PyExitData *exit = ¤t_executor->exits[oparg]; + _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; + assert(previous->trace[0].opcode != _COLD_EXIT); + _PyExitData *exit = &previous->exits[oparg]; exit->temperature++; assert(exit->executor->trace[0].opcode == _COLD_EXIT); PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; if (exit->temperature < 0) { next_instr = target; - Py_DECREF(current_executor); - current_executor = NULL; + Py_DECREF(previous); + tstate->previous_executor = NULL; DISPATCH(); } _PyExecutorObject *executor; @@ -4105,8 +4104,8 @@ dummy_func( int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { exit->temperature = -10000; /* Choose a better number */ - Py_DECREF(current_executor); - current_executor = NULL; + Py_DECREF(previous); + tstate->previous_executor = NULL; next_instr = target; ERROR_IF(optimized < 0, error); DISPATCH(); @@ -4114,15 +4113,16 @@ dummy_func( } Py_INCREF(executor); exit->executor = executor; - next_uop = executor->trace; - GOTO_TIER_TWO(); + GOTO_TIER_TWO(executor); } op(_START_EXECUTOR, (executor/4 --)) { TIER_TWO_ONLY - _PyExecutorObject *old = current_executor; + Py_DECREF(tstate->previous_executor); + tstate->previous_executor = NULL; +#ifndef JIT current_executor = (_PyExecutorObject*)executor; - Py_DECREF(old); +#endif } op(_FATAL_ERROR, (--)) { diff --git a/Python/ceval.c b/Python/ceval.c index b9dde73a3f2e16..13b032fd430e64 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -15,6 +15,7 @@ #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode_metadata.h" // EXTRA_CASES +#include "pycore_optimizer.h" // _PyUOpExecutor_Type #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -1060,7 +1061,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(current_executor); - current_executor = NULL; + tstate->previous_executor = NULL; goto resume_with_error; // Jump here from DEOPT_IF() @@ -1072,7 +1073,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); Py_DECREF(current_executor); - current_executor = NULL; + tstate->previous_executor = NULL; DISPATCH(); // Jump here from EXIT_IF() @@ -1085,8 +1086,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit_index, exit->temperature, exit->target, _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); Py_INCREF(exit->executor); - next_uop = exit->executor->trace; - GOTO_TIER_TWO(); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_TWO(exit->executor); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a3606b17b71c62..48797b15b3ce56 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -390,7 +390,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); /* Tier-switching macros. */ -#define GOTO_TIER_TWO() goto enter_tier_two; +#define GOTO_TIER_TWO(EXECUTOR) next_uop = (EXECUTOR)->trace; goto enter_tier_two; #define CURRENT_OPARG() (next_uop[-1].oparg) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 540755cf85c2cf..b75e9ba024f04c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3409,16 +3409,17 @@ case _COLD_EXIT: { oparg = CURRENT_OPARG(); TIER_TWO_ONLY - assert(current_executor->trace[0].opcode != _COLD_EXIT); - _PyExitData *exit = ¤t_executor->exits[oparg]; + _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; + assert(previous->trace[0].opcode != _COLD_EXIT); + _PyExitData *exit = &previous->exits[oparg]; exit->temperature++; assert(exit->executor->trace[0].opcode == _COLD_EXIT); PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; if (exit->temperature < 0) { next_instr = target; - Py_DECREF(current_executor); - current_executor = NULL; + Py_DECREF(previous); + tstate->previous_executor = NULL; DISPATCH(); } _PyExecutorObject *executor; @@ -3429,8 +3430,8 @@ int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { exit->temperature = -10000; /* Choose a better number */ - Py_DECREF(current_executor); - current_executor = NULL; + Py_DECREF(previous); + tstate->previous_executor = NULL; next_instr = target; if (optimized < 0) goto error_tier_two; DISPATCH(); @@ -3438,17 +3439,18 @@ } Py_INCREF(executor); exit->executor = executor; - next_uop = executor->trace; - GOTO_TIER_TWO(); + GOTO_TIER_TWO(executor); break; } case _START_EXECUTOR: { PyObject *executor = (PyObject *)CURRENT_OPERAND(); TIER_TWO_ONLY - _PyExecutorObject *old = current_executor; + Py_DECREF(tstate->previous_executor); + tstate->previous_executor = NULL; + #ifndef JIT current_executor = (_PyExecutorObject*)executor; - Py_DECREF(old); + #endif break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1425c2e0c3a0e7..222323cb9d6d47 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2380,11 +2380,10 @@ PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; if (executor->vm_data.valid) { - assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)&Py_NeverExecutedExecutor; + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; Py_INCREF(executor); - next_uop = executor->trace; - GOTO_TIER_TWO(); + GOTO_TIER_TWO(executor); } else { /* ENTER_EXECUTOR will be the first code unit of the instruction */ @@ -3305,10 +3304,9 @@ int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor); if (optimized < 0) goto error; if (optimized) { - assert(current_executor == NULL); - current_executor = (_PyExecutorObject *)&Py_NeverExecutedExecutor; - next_uop = executor->trace; - GOTO_TIER_TWO(); + assert(tstate->previous_executor == NULL); + tstate->previous_executor = Py_None; + GOTO_TIER_TWO(executor); } else { int backoff = this_instr[1].cache & ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); diff --git a/Python/pystate.c b/Python/pystate.c index eaf77b0da62a84..a8f7d28ecf8754 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1335,6 +1335,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_top = NULL; tstate->datastack_limit = NULL; tstate->what_event = -1; + tstate->previous_executor = NULL; tstate->_status.initialized = 1; } From 625bce26c627ac2370b76149fcd9a2beb7a8dc71 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 9 Feb 2024 00:24:19 +0000 Subject: [PATCH 16/40] Get side exits to build with jit enabled. --- Python/bytecodes.c | 10 +++++----- Python/ceval.c | 4 ++-- Python/ceval_macros.h | 4 ++++ Python/executor_cases.c.h | 10 +++++----- Tools/jit/template.c | 21 ++++++++++++++++++++- 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 950104bbeb678c..9f7bd0014af2e3 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4042,7 +4042,9 @@ dummy_func( } op(_JUMP_TO_TOP, (--)) { +#ifndef Py_JIT next_uop = ¤t_executor->trace[1]; +#endif CHECK_EVAL_BREAKER(); } @@ -4119,10 +4121,9 @@ dummy_func( PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; if (exit->temperature < 0) { - next_instr = target; Py_DECREF(previous); tstate->previous_executor = NULL; - DISPATCH(); + GOTO_TIER_ONE(target); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -4134,9 +4135,8 @@ dummy_func( exit->temperature = -10000; /* Choose a better number */ Py_DECREF(previous); tstate->previous_executor = NULL; - next_instr = target; ERROR_IF(optimized < 0, error); - DISPATCH(); + GOTO_TIER_ONE(target); } } Py_INCREF(executor); @@ -4148,7 +4148,7 @@ dummy_func( TIER_TWO_ONLY Py_DECREF(tstate->previous_executor); tstate->previous_executor = NULL; -#ifndef JIT +#ifndef Py_JIT current_executor = (_PyExecutorObject*)executor; #endif } diff --git a/Python/ceval.c b/Python/ceval.c index 1be21d960170e4..7e927b4c8b7dee 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1092,8 +1092,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int tstate->previous_executor = NULL; DISPATCH(); -#endif // _Py_JIT - // Jump here from EXIT_IF() side_exit: OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); @@ -1107,6 +1105,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_TWO(exit->executor); +#endif // _Py_JIT + } #if defined(__GNUC__) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 7e21dfbd0fe3a7..58a778dd728ed1 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -394,6 +394,10 @@ do { \ goto enter_tier_two; \ } while (0) +#define GOTO_TIER_ONE(TARGET) \ + next_instr = target; \ + DISPATCH(); + #define CURRENT_OPARG() (next_uop[-1].oparg) #define CURRENT_OPERAND() (next_uop[-1].operand) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bc51b35eeaf284..cd1e6ecb8e6d98 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3352,7 +3352,9 @@ } case _JUMP_TO_TOP: { + #ifndef Py_JIT next_uop = ¤t_executor->trace[1]; + #endif CHECK_EVAL_BREAKER(); break; } @@ -3468,10 +3470,9 @@ PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; if (exit->temperature < 0) { - next_instr = target; Py_DECREF(previous); tstate->previous_executor = NULL; - DISPATCH(); + GOTO_TIER_ONE(target); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -3483,9 +3484,8 @@ exit->temperature = -10000; /* Choose a better number */ Py_DECREF(previous); tstate->previous_executor = NULL; - next_instr = target; if (optimized < 0) goto error_tier_two; - DISPATCH(); + GOTO_TIER_ONE(target); } } Py_INCREF(executor); @@ -3499,7 +3499,7 @@ TIER_TWO_ONLY Py_DECREF(tstate->previous_executor); tstate->previous_executor = NULL; - #ifndef JIT + #ifndef Py_JIT current_executor = (_PyExecutorObject*)executor; #endif break; diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 12303a550d8879..9051604b3ca725 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -38,6 +38,16 @@ goto LABEL ## _tier_two; \ } while (0) +#undef GOTO_TIER_TWO +#define GOTO_TIER_TWO(EXECUTOR) \ + __attribute__((musttail)) \ + return ((jit_func)((EXECUTOR)->jit_code))(frame, stack_pointer, tstate); + +#undef GOTO_TIER_ONE +#define GOTO_TIER_ONE(TARGET) \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + return TARGET; + #undef LOAD_IP #define LOAD_IP(UNUSED) \ do { \ @@ -52,6 +62,8 @@ __attribute__((musttail)) \ return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); +#define Py_JIT 1 + _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) { @@ -59,7 +71,6 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) int oparg; int opcode = _JIT_OPCODE; - _PyUOpInstruction *next_uop; // Other stuff we need handy: PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) @@ -95,4 +106,12 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * deoptimize: _PyFrame_SetStackPointer(frame, stack_pointer); return _PyCode_CODE(_PyFrame_GetCode(frame)) + _target; +side_exit: + { + _PyExitData *exit = ¤t_executor->exits[_target]; + Py_INCREF(exit->executor); + tstate->previous_executor = (PyObject *)current_executor; + __attribute__((musttail)) + return ((jit_func)exit->executor->jit_code)(frame, stack_pointer, tstate); + } } From e191fd73226f9a8362c618e1b5ef039f7b6c62a6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 9 Feb 2024 01:52:30 +0000 Subject: [PATCH 17/40] Initialize cold exits dynamically on demand --- Include/cpython/optimizer.h | 8 +- Python/optimizer.c | 610 +++--------------------------------- 2 files changed, 51 insertions(+), 567 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 80059efb42d9db..f9506c0fded3f8 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -42,7 +42,7 @@ typedef struct { typedef struct _exit_data { uint32_t target; int16_t temperature; - struct _PyExecutorObject *executor; + const struct _PyExecutorObject *executor; } _PyExitData; typedef struct _PyExecutorObject { @@ -51,7 +51,7 @@ typedef struct _PyExecutorObject { _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ uint32_t exit_count; uint32_t code_size; - void *jit_code; + void *jit_code; _PyExitData exits[1]; } _PyExecutorObject; @@ -61,8 +61,6 @@ typedef struct _cold_exit { } _PyColdExitObject; -extern _PyColdExitObject Py_NeverExecutedExecutor; - typedef struct _PyOptimizerObject _PyOptimizerObject; /* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ @@ -100,7 +98,7 @@ _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, Py extern _PyOptimizerObject _PyOptimizer_Default; -void _Py_ExecutorInit(_PyExecutorObject *, _PyBloomFilter *); +void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); void _Py_ExecutorClear(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); diff --git a/Python/optimizer.c b/Python/optimizer.c index f2996812d5d0cf..a1fb86342ce9e6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -147,9 +147,26 @@ PyUnstable_GetOptimizer(void) return interp->optimizer; } +static _PyExecutorObject * +make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies); + +static _PyExecutorObject * +make_cold_exit_executor(int oparg); + +static int cold_exits_initialized = 0; +static const _PyExecutorObject *COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 }; + +static const _PyBloomFilter EMPTY = { 0 }; + void PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) { + if (cold_exits_initialized == 0) { + cold_exits_initialized = 1; + for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { + COLD_EXITS[i] = make_cold_exit_executor(i); + } + } PyInterpreterState *interp = _PyInterpreterState_GET(); if (optimizer == NULL) { optimizer = &_PyOptimizer_Default; @@ -812,25 +829,6 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) /* Executor side exits */ -static void -cold_dealloc(_PyExecutorObject *self) { - _Py_ExecutorClear(self); - PyObject_Free(self); -} - -static PyTypeObject _ColdExit_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "cold_exit", - .tp_basicsize = offsetof(_PyExecutorObject, exits), - .tp_itemsize = 1, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_dealloc = (destructor)cold_dealloc, - .tp_as_sequence = &uop_as_sequence, - .tp_methods = executor_methods, -}; - -static const _PyColdExitObject COLD_EXITS[UOP_MAX_TRACE_LENGTH]; - static _PyExecutorObject * allocate_executor(int exit_count, int length) { @@ -845,22 +843,13 @@ allocate_executor(int exit_count, int length) return res; } -_PyColdExitObject Py_NeverExecutedExecutor = { - .base = { - PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) - .vm_data = { 0 }, - .trace = &Py_NeverExecutedExecutor.uop - }, - .uop.opcode = _FATAL_ERROR, -}; - /* Makes an executor from a buffer of uops. * Account for the buffer having gaps and NOPs by computing a "used" * bit vector and only copying the used uops. Here "used" means reachable * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies) { uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; int exit_count; @@ -871,7 +860,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) } /* Initialize exits */ for (int i = 0; i < exit_count; i++) { - executor->exits[i].executor = (_PyExecutorObject *)&COLD_EXITS[i]; + executor->exits[i].executor = COLD_EXITS[i]; executor->exits[i].temperature = -67; } int next_exit = exit_count-1; @@ -935,6 +924,32 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies) return executor; } +static _PyExecutorObject * +make_cold_exit_executor(int oparg) +{ + _PyExecutorObject *executor = allocate_executor(0, 1); + if (executor == NULL) { + return NULL; + } + _PyUOpInstruction *inst = (_PyUOpInstruction *)&executor->trace[0]; + inst->opcode = _COLD_EXIT; + inst->oparg = oparg; + executor->vm_data.valid = true; + for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { + executor->vm_data.bloom.bits[i] = 0; + } +#ifdef _Py_JIT + executor->jit_code = NULL; + executor->code_size = 0; + if (_PyJIT_Compile(executor, executor->trace, Py_SIZE(executor))) { + Py_DECREF(executor); + return NULL; + } +#endif + _Py_SetImmortal(executor); + return executor; +} + static int uop_optimize( _PyOptimizerObject *self, @@ -1038,14 +1053,12 @@ counter_optimize( return 0; } _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; - _PyUOpInstruction buffer[3] = { + _PyUOpInstruction buffer[4] = { { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) } }; - _PyBloomFilter empty; - _Py_BloomFilter_Init(&empty); - _PyExecutorObject *executor = make_executor_from_uops(buffer, &empty); + _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY); if (executor == NULL) { return -1; } @@ -1208,7 +1221,7 @@ link_executor(_PyExecutorObject *executor) /* This must be called by optimizers before using the executor */ void -_Py_ExecutorInit(_PyExecutorObject *executor, _PyBloomFilter *dependency_set) +_Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_set) { executor->vm_data.valid = true; for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { @@ -1262,530 +1275,3 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp) } interp->executor_list_head = NULL; } - -/* The following table is generated from this script: -print(f'static_assert(UOP_MAX_TRACE_LENGTH == {UOP_MAX_TRACE_LENGTH}, "COLD_EXITS must be regenerated");\n') -print("static _PyColdExitObject COLD_EXITS[] = {") -for i in range(UOP_MAX_TRACE_LENGTH): - print(f" [{i}] = {{ .base = {{ PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = {{ 0 }}," - f" .trace = &COLD_EXITS[{i}].uop }}, .uop.opcode = _COLD_EXIT, .uop.oparg = {i} }},") -print("};") -*/ - -static_assert(UOP_MAX_TRACE_LENGTH == 512, "COLD_EXITS must be regenerated"); - -static const _PyColdExitObject COLD_EXITS[] = { - [0] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[0].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 0 }, - [1] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[1].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 1 }, - [2] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[2].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 2 }, - [3] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[3].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 3 }, - [4] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[4].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 4 }, - [5] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[5].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 5 }, - [6] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[6].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 6 }, - [7] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[7].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 7 }, - [8] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[8].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 8 }, - [9] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[9].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 9 }, - [10] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[10].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 10 }, - [11] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[11].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 11 }, - [12] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[12].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 12 }, - [13] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[13].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 13 }, - [14] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[14].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 14 }, - [15] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[15].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 15 }, - [16] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[16].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 16 }, - [17] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[17].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 17 }, - [18] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[18].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 18 }, - [19] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[19].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 19 }, - [20] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[20].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 20 }, - [21] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[21].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 21 }, - [22] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[22].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 22 }, - [23] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[23].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 23 }, - [24] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[24].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 24 }, - [25] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[25].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 25 }, - [26] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[26].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 26 }, - [27] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[27].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 27 }, - [28] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[28].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 28 }, - [29] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[29].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 29 }, - [30] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[30].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 30 }, - [31] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[31].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 31 }, - [32] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[32].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 32 }, - [33] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[33].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 33 }, - [34] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[34].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 34 }, - [35] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[35].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 35 }, - [36] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[36].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 36 }, - [37] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[37].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 37 }, - [38] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[38].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 38 }, - [39] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[39].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 39 }, - [40] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[40].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 40 }, - [41] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[41].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 41 }, - [42] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[42].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 42 }, - [43] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[43].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 43 }, - [44] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[44].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 44 }, - [45] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[45].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 45 }, - [46] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[46].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 46 }, - [47] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[47].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 47 }, - [48] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[48].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 48 }, - [49] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[49].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 49 }, - [50] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[50].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 50 }, - [51] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[51].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 51 }, - [52] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[52].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 52 }, - [53] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[53].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 53 }, - [54] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[54].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 54 }, - [55] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[55].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 55 }, - [56] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[56].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 56 }, - [57] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[57].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 57 }, - [58] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[58].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 58 }, - [59] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[59].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 59 }, - [60] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[60].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 60 }, - [61] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[61].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 61 }, - [62] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[62].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 62 }, - [63] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[63].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 63 }, - [64] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[64].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 64 }, - [65] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[65].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 65 }, - [66] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[66].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 66 }, - [67] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[67].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 67 }, - [68] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[68].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 68 }, - [69] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[69].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 69 }, - [70] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[70].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 70 }, - [71] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[71].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 71 }, - [72] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[72].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 72 }, - [73] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[73].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 73 }, - [74] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[74].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 74 }, - [75] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[75].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 75 }, - [76] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[76].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 76 }, - [77] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[77].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 77 }, - [78] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[78].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 78 }, - [79] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[79].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 79 }, - [80] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[80].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 80 }, - [81] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[81].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 81 }, - [82] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[82].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 82 }, - [83] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[83].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 83 }, - [84] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[84].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 84 }, - [85] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[85].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 85 }, - [86] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[86].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 86 }, - [87] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[87].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 87 }, - [88] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[88].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 88 }, - [89] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[89].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 89 }, - [90] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[90].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 90 }, - [91] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[91].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 91 }, - [92] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[92].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 92 }, - [93] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[93].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 93 }, - [94] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[94].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 94 }, - [95] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[95].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 95 }, - [96] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[96].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 96 }, - [97] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[97].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 97 }, - [98] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[98].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 98 }, - [99] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[99].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 99 }, - [100] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[100].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 100 }, - [101] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[101].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 101 }, - [102] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[102].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 102 }, - [103] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[103].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 103 }, - [104] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[104].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 104 }, - [105] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[105].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 105 }, - [106] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[106].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 106 }, - [107] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[107].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 107 }, - [108] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[108].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 108 }, - [109] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[109].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 109 }, - [110] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[110].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 110 }, - [111] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[111].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 111 }, - [112] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[112].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 112 }, - [113] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[113].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 113 }, - [114] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[114].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 114 }, - [115] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[115].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 115 }, - [116] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[116].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 116 }, - [117] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[117].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 117 }, - [118] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[118].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 118 }, - [119] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[119].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 119 }, - [120] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[120].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 120 }, - [121] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[121].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 121 }, - [122] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[122].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 122 }, - [123] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[123].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 123 }, - [124] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[124].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 124 }, - [125] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[125].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 125 }, - [126] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[126].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 126 }, - [127] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[127].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 127 }, - [128] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[128].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 128 }, - [129] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[129].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 129 }, - [130] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[130].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 130 }, - [131] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[131].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 131 }, - [132] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[132].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 132 }, - [133] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[133].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 133 }, - [134] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[134].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 134 }, - [135] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[135].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 135 }, - [136] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[136].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 136 }, - [137] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[137].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 137 }, - [138] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[138].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 138 }, - [139] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[139].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 139 }, - [140] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[140].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 140 }, - [141] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[141].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 141 }, - [142] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[142].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 142 }, - [143] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[143].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 143 }, - [144] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[144].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 144 }, - [145] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[145].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 145 }, - [146] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[146].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 146 }, - [147] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[147].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 147 }, - [148] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[148].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 148 }, - [149] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[149].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 149 }, - [150] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[150].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 150 }, - [151] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[151].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 151 }, - [152] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[152].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 152 }, - [153] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[153].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 153 }, - [154] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[154].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 154 }, - [155] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[155].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 155 }, - [156] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[156].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 156 }, - [157] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[157].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 157 }, - [158] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[158].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 158 }, - [159] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[159].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 159 }, - [160] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[160].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 160 }, - [161] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[161].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 161 }, - [162] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[162].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 162 }, - [163] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[163].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 163 }, - [164] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[164].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 164 }, - [165] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[165].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 165 }, - [166] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[166].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 166 }, - [167] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[167].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 167 }, - [168] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[168].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 168 }, - [169] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[169].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 169 }, - [170] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[170].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 170 }, - [171] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[171].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 171 }, - [172] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[172].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 172 }, - [173] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[173].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 173 }, - [174] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[174].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 174 }, - [175] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[175].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 175 }, - [176] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[176].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 176 }, - [177] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[177].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 177 }, - [178] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[178].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 178 }, - [179] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[179].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 179 }, - [180] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[180].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 180 }, - [181] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[181].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 181 }, - [182] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[182].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 182 }, - [183] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[183].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 183 }, - [184] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[184].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 184 }, - [185] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[185].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 185 }, - [186] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[186].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 186 }, - [187] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[187].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 187 }, - [188] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[188].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 188 }, - [189] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[189].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 189 }, - [190] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[190].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 190 }, - [191] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[191].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 191 }, - [192] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[192].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 192 }, - [193] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[193].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 193 }, - [194] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[194].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 194 }, - [195] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[195].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 195 }, - [196] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[196].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 196 }, - [197] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[197].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 197 }, - [198] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[198].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 198 }, - [199] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[199].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 199 }, - [200] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[200].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 200 }, - [201] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[201].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 201 }, - [202] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[202].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 202 }, - [203] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[203].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 203 }, - [204] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[204].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 204 }, - [205] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[205].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 205 }, - [206] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[206].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 206 }, - [207] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[207].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 207 }, - [208] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[208].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 208 }, - [209] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[209].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 209 }, - [210] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[210].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 210 }, - [211] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[211].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 211 }, - [212] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[212].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 212 }, - [213] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[213].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 213 }, - [214] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[214].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 214 }, - [215] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[215].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 215 }, - [216] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[216].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 216 }, - [217] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[217].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 217 }, - [218] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[218].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 218 }, - [219] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[219].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 219 }, - [220] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[220].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 220 }, - [221] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[221].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 221 }, - [222] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[222].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 222 }, - [223] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[223].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 223 }, - [224] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[224].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 224 }, - [225] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[225].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 225 }, - [226] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[226].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 226 }, - [227] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[227].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 227 }, - [228] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[228].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 228 }, - [229] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[229].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 229 }, - [230] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[230].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 230 }, - [231] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[231].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 231 }, - [232] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[232].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 232 }, - [233] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[233].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 233 }, - [234] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[234].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 234 }, - [235] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[235].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 235 }, - [236] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[236].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 236 }, - [237] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[237].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 237 }, - [238] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[238].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 238 }, - [239] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[239].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 239 }, - [240] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[240].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 240 }, - [241] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[241].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 241 }, - [242] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[242].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 242 }, - [243] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[243].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 243 }, - [244] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[244].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 244 }, - [245] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[245].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 245 }, - [246] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[246].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 246 }, - [247] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[247].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 247 }, - [248] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[248].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 248 }, - [249] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[249].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 249 }, - [250] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[250].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 250 }, - [251] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[251].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 251 }, - [252] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[252].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 252 }, - [253] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[253].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 253 }, - [254] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[254].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 254 }, - [255] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[255].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 255 }, - [256] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[256].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 256 }, - [257] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[257].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 257 }, - [258] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[258].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 258 }, - [259] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[259].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 259 }, - [260] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[260].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 260 }, - [261] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[261].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 261 }, - [262] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[262].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 262 }, - [263] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[263].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 263 }, - [264] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[264].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 264 }, - [265] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[265].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 265 }, - [266] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[266].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 266 }, - [267] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[267].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 267 }, - [268] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[268].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 268 }, - [269] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[269].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 269 }, - [270] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[270].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 270 }, - [271] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[271].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 271 }, - [272] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[272].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 272 }, - [273] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[273].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 273 }, - [274] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[274].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 274 }, - [275] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[275].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 275 }, - [276] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[276].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 276 }, - [277] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[277].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 277 }, - [278] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[278].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 278 }, - [279] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[279].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 279 }, - [280] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[280].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 280 }, - [281] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[281].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 281 }, - [282] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[282].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 282 }, - [283] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[283].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 283 }, - [284] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[284].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 284 }, - [285] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[285].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 285 }, - [286] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[286].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 286 }, - [287] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[287].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 287 }, - [288] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[288].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 288 }, - [289] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[289].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 289 }, - [290] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[290].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 290 }, - [291] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[291].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 291 }, - [292] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[292].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 292 }, - [293] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[293].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 293 }, - [294] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[294].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 294 }, - [295] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[295].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 295 }, - [296] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[296].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 296 }, - [297] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[297].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 297 }, - [298] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[298].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 298 }, - [299] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[299].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 299 }, - [300] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[300].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 300 }, - [301] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[301].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 301 }, - [302] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[302].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 302 }, - [303] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[303].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 303 }, - [304] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[304].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 304 }, - [305] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[305].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 305 }, - [306] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[306].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 306 }, - [307] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[307].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 307 }, - [308] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[308].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 308 }, - [309] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[309].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 309 }, - [310] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[310].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 310 }, - [311] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[311].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 311 }, - [312] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[312].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 312 }, - [313] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[313].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 313 }, - [314] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[314].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 314 }, - [315] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[315].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 315 }, - [316] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[316].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 316 }, - [317] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[317].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 317 }, - [318] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[318].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 318 }, - [319] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[319].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 319 }, - [320] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[320].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 320 }, - [321] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[321].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 321 }, - [322] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[322].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 322 }, - [323] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[323].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 323 }, - [324] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[324].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 324 }, - [325] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[325].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 325 }, - [326] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[326].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 326 }, - [327] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[327].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 327 }, - [328] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[328].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 328 }, - [329] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[329].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 329 }, - [330] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[330].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 330 }, - [331] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[331].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 331 }, - [332] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[332].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 332 }, - [333] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[333].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 333 }, - [334] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[334].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 334 }, - [335] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[335].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 335 }, - [336] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[336].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 336 }, - [337] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[337].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 337 }, - [338] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[338].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 338 }, - [339] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[339].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 339 }, - [340] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[340].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 340 }, - [341] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[341].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 341 }, - [342] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[342].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 342 }, - [343] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[343].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 343 }, - [344] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[344].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 344 }, - [345] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[345].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 345 }, - [346] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[346].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 346 }, - [347] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[347].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 347 }, - [348] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[348].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 348 }, - [349] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[349].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 349 }, - [350] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[350].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 350 }, - [351] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[351].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 351 }, - [352] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[352].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 352 }, - [353] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[353].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 353 }, - [354] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[354].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 354 }, - [355] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[355].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 355 }, - [356] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[356].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 356 }, - [357] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[357].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 357 }, - [358] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[358].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 358 }, - [359] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[359].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 359 }, - [360] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[360].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 360 }, - [361] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[361].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 361 }, - [362] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[362].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 362 }, - [363] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[363].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 363 }, - [364] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[364].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 364 }, - [365] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[365].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 365 }, - [366] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[366].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 366 }, - [367] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[367].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 367 }, - [368] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[368].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 368 }, - [369] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[369].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 369 }, - [370] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[370].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 370 }, - [371] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[371].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 371 }, - [372] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[372].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 372 }, - [373] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[373].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 373 }, - [374] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[374].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 374 }, - [375] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[375].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 375 }, - [376] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[376].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 376 }, - [377] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[377].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 377 }, - [378] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[378].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 378 }, - [379] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[379].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 379 }, - [380] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[380].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 380 }, - [381] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[381].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 381 }, - [382] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[382].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 382 }, - [383] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[383].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 383 }, - [384] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[384].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 384 }, - [385] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[385].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 385 }, - [386] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[386].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 386 }, - [387] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[387].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 387 }, - [388] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[388].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 388 }, - [389] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[389].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 389 }, - [390] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[390].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 390 }, - [391] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[391].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 391 }, - [392] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[392].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 392 }, - [393] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[393].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 393 }, - [394] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[394].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 394 }, - [395] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[395].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 395 }, - [396] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[396].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 396 }, - [397] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[397].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 397 }, - [398] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[398].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 398 }, - [399] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[399].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 399 }, - [400] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[400].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 400 }, - [401] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[401].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 401 }, - [402] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[402].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 402 }, - [403] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[403].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 403 }, - [404] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[404].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 404 }, - [405] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[405].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 405 }, - [406] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[406].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 406 }, - [407] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[407].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 407 }, - [408] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[408].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 408 }, - [409] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[409].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 409 }, - [410] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[410].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 410 }, - [411] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[411].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 411 }, - [412] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[412].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 412 }, - [413] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[413].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 413 }, - [414] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[414].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 414 }, - [415] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[415].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 415 }, - [416] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[416].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 416 }, - [417] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[417].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 417 }, - [418] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[418].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 418 }, - [419] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[419].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 419 }, - [420] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[420].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 420 }, - [421] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[421].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 421 }, - [422] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[422].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 422 }, - [423] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[423].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 423 }, - [424] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[424].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 424 }, - [425] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[425].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 425 }, - [426] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[426].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 426 }, - [427] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[427].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 427 }, - [428] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[428].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 428 }, - [429] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[429].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 429 }, - [430] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[430].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 430 }, - [431] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[431].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 431 }, - [432] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[432].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 432 }, - [433] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[433].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 433 }, - [434] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[434].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 434 }, - [435] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[435].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 435 }, - [436] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[436].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 436 }, - [437] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[437].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 437 }, - [438] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[438].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 438 }, - [439] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[439].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 439 }, - [440] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[440].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 440 }, - [441] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[441].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 441 }, - [442] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[442].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 442 }, - [443] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[443].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 443 }, - [444] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[444].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 444 }, - [445] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[445].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 445 }, - [446] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[446].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 446 }, - [447] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[447].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 447 }, - [448] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[448].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 448 }, - [449] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[449].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 449 }, - [450] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[450].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 450 }, - [451] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[451].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 451 }, - [452] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[452].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 452 }, - [453] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[453].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 453 }, - [454] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[454].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 454 }, - [455] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[455].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 455 }, - [456] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[456].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 456 }, - [457] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[457].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 457 }, - [458] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[458].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 458 }, - [459] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[459].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 459 }, - [460] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[460].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 460 }, - [461] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[461].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 461 }, - [462] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[462].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 462 }, - [463] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[463].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 463 }, - [464] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[464].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 464 }, - [465] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[465].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 465 }, - [466] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[466].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 466 }, - [467] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[467].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 467 }, - [468] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[468].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 468 }, - [469] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[469].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 469 }, - [470] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[470].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 470 }, - [471] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[471].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 471 }, - [472] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[472].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 472 }, - [473] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[473].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 473 }, - [474] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[474].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 474 }, - [475] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[475].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 475 }, - [476] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[476].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 476 }, - [477] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[477].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 477 }, - [478] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[478].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 478 }, - [479] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[479].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 479 }, - [480] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[480].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 480 }, - [481] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[481].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 481 }, - [482] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[482].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 482 }, - [483] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[483].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 483 }, - [484] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[484].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 484 }, - [485] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[485].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 485 }, - [486] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[486].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 486 }, - [487] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[487].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 487 }, - [488] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[488].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 488 }, - [489] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[489].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 489 }, - [490] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[490].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 490 }, - [491] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[491].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 491 }, - [492] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[492].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 492 }, - [493] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[493].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 493 }, - [494] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[494].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 494 }, - [495] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[495].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 495 }, - [496] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[496].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 496 }, - [497] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[497].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 497 }, - [498] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[498].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 498 }, - [499] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[499].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 499 }, - [500] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[500].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 500 }, - [501] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[501].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 501 }, - [502] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[502].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 502 }, - [503] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[503].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 503 }, - [504] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[504].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 504 }, - [505] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[505].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 505 }, - [506] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[506].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 506 }, - [507] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[507].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 507 }, - [508] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[508].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 508 }, - [509] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[509].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 509 }, - [510] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[510].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 510 }, - [511] = { .base = { PyVarObject_HEAD_INIT(&_ColdExit_Type, 0) .vm_data = { 0 }, .trace = &COLD_EXITS[511].uop }, .uop.opcode = _COLD_EXIT, .uop.oparg = 511 }, -}; - From 941a14c2878506d0bc245ad5bb0478daae7683ea Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 9 Feb 2024 02:22:02 +0000 Subject: [PATCH 18/40] Tidy tier 2 code a bit --- Python/ceval.c | 26 +++++++++----------------- Python/optimizer.c | 4 ++-- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 7e927b4c8b7dee..7831948ce1abfd 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -739,16 +739,16 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto resume_with_error; } - /* State shared between Tier 1 and Tier 2 interpreter */ - _PyExecutorObject *current_executor = NULL; - /* Local "register" variables. * These are cached values from the frame and code object. */ - _Py_CODEUNIT *next_instr; PyObject **stack_pointer; - const _PyUOpInstruction *next_uop = NULL; +#ifndef Py_JIT + /* State shared between Tier 1 and Tier 2 interpreter */ + _PyExecutorObject *current_executor = NULL; + const _PyUOpInstruction *next_uop = NULL; +#endif start_frame: if (_Py_EnterRecursivePy(tstate)) { @@ -962,18 +962,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int enter_tier_two: #ifdef _Py_JIT - - ; // ;) - jit_func jitted = current_executor->jit_code; - next_instr = jitted(frame, stack_pointer, tstate); - frame = tstate->current_frame; - Py_DECREF(current_executor); - if (next_instr == NULL) { - goto resume_with_error; - } - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - + assert(0); #else #undef LOAD_IP @@ -1094,6 +1083,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // Jump here from EXIT_IF() side_exit: +#ifdef _Py_JIT + assert(0); +#endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); uint16_t exit_index = next_uop[-1].exit_index; diff --git a/Python/optimizer.c b/Python/optimizer.c index a1fb86342ce9e6..a51356a62b5f01 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -915,7 +915,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende #ifdef _Py_JIT executor->jit_code = NULL; executor->code_size = 0; - if (_PyJIT_Compile(executor, executor->trace, Py_SIZE(executor))) { + if (_PyJIT_Compile(executor, executor->trace, length+1)) { Py_DECREF(executor); return NULL; } @@ -941,7 +941,7 @@ make_cold_exit_executor(int oparg) #ifdef _Py_JIT executor->jit_code = NULL; executor->code_size = 0; - if (_PyJIT_Compile(executor, executor->trace, Py_SIZE(executor))) { + if (_PyJIT_Compile(executor, executor->trace, 1)) { Py_DECREF(executor); return NULL; } From cfd3285fd7278f93f277b2e8363fa396e96d84ae Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 9 Feb 2024 08:13:06 +0000 Subject: [PATCH 19/40] Add Brandt's fixes --- Include/cpython/optimizer.h | 1 + Python/ceval_macros.h | 14 ++++++++++++++ Python/jit.c | 11 ++++++++--- Python/optimizer.c | 2 +- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index f9506c0fded3f8..9a0778914f5acb 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -51,6 +51,7 @@ typedef struct _PyExecutorObject { _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ uint32_t exit_count; uint32_t code_size; + size_t jit_size; void *jit_code; _PyExitData exits[1]; } _PyExecutorObject; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 58a778dd728ed1..b6e31bb3f5de76 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -387,12 +387,26 @@ stack_pointer = _PyFrame_GetStackPointer(frame); /* Tier-switching macros. */ +#ifdef _Py_JIT +#define GOTO_TIER_TWO(EXECUTOR) \ + do { \ + jit_func jitted = (EXECUTOR)->jit_code; \ + next_instr = jitted(frame, stack_pointer, tstate); \ + frame = tstate->current_frame; \ + if (next_instr == NULL) { \ + goto resume_with_error; \ + } \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + DISPATCH(); \ + } while (0) +#else #define GOTO_TIER_TWO(EXECUTOR) \ do { \ next_uop = (EXECUTOR)->trace; \ assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); \ goto enter_tier_two; \ } while (0) +#endif #define GOTO_TIER_ONE(TARGET) \ next_instr = target; \ diff --git a/Python/jit.c b/Python/jit.c index c210ecb12e3c93..9efd4a7e093eab 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -323,6 +323,11 @@ _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t len // Loop again to emit the code: char *code = memory; char *data = memory + code_size; + char *top = code; + if (trace[0].opcode == _START_EXECUTOR) { + // Don't want to execute this more than once: + top += stencil_groups[_START_EXECUTOR].code.body_size; + } for (size_t i = 0; i < length; i++) { _PyUOpInstruction *instruction = &trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; @@ -335,7 +340,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t len patches[HoleValue_OPARG] = instruction->oparg; patches[HoleValue_OPERAND] = instruction->operand; patches[HoleValue_TARGET] = instruction->target; - patches[HoleValue_TOP] = (uint64_t)memory; + patches[HoleValue_TOP] = (uint64_t)top; patches[HoleValue_ZERO] = 0; emit(group, patches); code += group->code.body_size; @@ -348,7 +353,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t len return -1; } executor->jit_code = memory; - executor->code_size = code_size + data_size; + executor->jit_size = code_size + data_size; return 0; } @@ -359,7 +364,7 @@ _PyJIT_Free(_PyExecutorObject *executor) size_t size = executor->code_size; if (memory) { executor->jit_code = NULL; - executor->code_size = 0; + executor->jit_size = 0; if (jit_free(memory, size)) { PyErr_WriteUnraisable(NULL); } diff --git a/Python/optimizer.c b/Python/optimizer.c index a51356a62b5f01..edbe0053ffdb0d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -914,7 +914,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende #endif #ifdef _Py_JIT executor->jit_code = NULL; - executor->code_size = 0; + executor->jit_size = 0; if (_PyJIT_Compile(executor, executor->trace, length+1)) { Py_DECREF(executor); return NULL; From 10254951d9b3816495d3012b08de119880e2f90a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 9 Feb 2024 08:30:16 +0000 Subject: [PATCH 20/40] Free the correct amount of memory --- Python/jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/jit.c b/Python/jit.c index 9efd4a7e093eab..43930347a60a24 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -361,7 +361,7 @@ void _PyJIT_Free(_PyExecutorObject *executor) { char *memory = (char *)executor->jit_code; - size_t size = executor->code_size; + size_t size = executor->jit_size; if (memory) { executor->jit_code = NULL; executor->jit_size = 0; From e6ca3fe24bdebc188efd9703aa2860058e563cc7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 9 Feb 2024 10:23:03 +0000 Subject: [PATCH 21/40] Remove unreachable code --- Python/ceval.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index c671bfdf0c2ae8..d81c6afb5e5586 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1083,9 +1083,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // Jump here from EXIT_IF() side_exit: -#ifdef _Py_JIT - assert(0); -#endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); uint16_t exit_index = next_uop[-1].exit_index; From 518143ee509b44bcc24c2e3c718eb2a17a52b62f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 9 Feb 2024 15:24:37 +0000 Subject: [PATCH 22/40] Clear executors attached to exits when clearing executors --- Lib/test/test_frame.py | 1 + Python/optimizer.c | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index baed03d92b9e56..0510a3cd9a2599 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -328,6 +328,7 @@ def f(): next(g) # Move all objects to the oldest generation, and tell the GC to run # on the *very next* allocation: + sys._clear_internal_caches() gc.collect() gc.set_threshold(1, 0, 0) # Okay, so here's the nightmare scenario: diff --git a/Python/optimizer.c b/Python/optimizer.c index f7098c37b4cc34..39c5600de4b919 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1182,9 +1182,6 @@ link_executor(_PyExecutorObject *executor) static void unlink_executor(_PyExecutorObject *executor) { - if (!executor->vm_data.valid) { - return; - } _PyExecutorLinkListNode *links = &executor->vm_data.links; _PyExecutorObject *next = links->next; _PyExecutorObject *prev = links->previous; @@ -1218,11 +1215,19 @@ _Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_s void _Py_ExecutorClear(_PyExecutorObject *executor) { + if (!executor->vm_data.valid) { + return; + } unlink_executor(executor); PyCodeObject *code = executor->vm_data.code; if (code == NULL) { return; } + for (uint32_t i = 0; i < executor->exit_count; i++) { + Py_DECREF(executor->exits[i].executor); + executor->exits[i].executor = COLD_EXITS[i]; + executor->exits[i].temperature = INT16_MIN; + } _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; assert(instruction->op.code == ENTER_EXECUTOR); int index = instruction->op.arg; From 19b6b84096ae4ebb57a347e96e5737237adfafdd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 9 Feb 2024 18:15:03 +0000 Subject: [PATCH 23/40] Keep c-analyzer happy --- Python/optimizer.c | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 09980350b8173b..c9609b27e24b95 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -132,7 +132,7 @@ static _PyOptimizerObject _PyOptimizer_Default = { }; static uint32_t -shift_and_offset_threshold(uint16_t threshold) +shift_and_offset_threshold(uint32_t threshold) { return (threshold << OPTIMIZER_BITS_IN_COUNTER) + (1 << 15); } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 116c75eb1a1b8f..77713e81b9ed1d 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -385,6 +385,8 @@ Python/optimizer.c - _PyOptimizer_Default - Python/optimizer.c - _ColdExit_Type - Python/optimizer.c - COLD_EXITS - Python/optimizer.c - Py_FatalErrorExecutor - +Python/optimizer.c - EMPTY - +Python/optimizer.c - cold_exits_initialized - ##----------------------- ## test code From f393ba59a74617630637aea14e82985acf1d0eec Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Feb 2024 11:30:49 +0000 Subject: [PATCH 24/40] Use threshold for side exits --- Include/internal/pycore_interp.h | 2 +- Python/bytecodes.c | 5 +++-- Python/executor_cases.c.h | 5 +++-- Python/optimizer.c | 1 + 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index bb3097fc554bec..661aba1bf4764c 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -241,8 +241,8 @@ struct _is { _PyExecutorObject *executor_list_head; /* These values are shifted and offset to speed up check in JUMP_BACKWARD */ uint32_t optimizer_resume_threshold; - uint32_t optimizer_side_threshold; uint32_t optimizer_backedge_threshold; + uint16_t optimizer_side_threshold; uint32_t next_func_version; _rare_events rare_events; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b15920dd269be8..03f0719dc95363 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4112,7 +4112,7 @@ dummy_func( assert(exit->executor->trace[0].opcode == _COLD_EXIT); PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - if (exit->temperature < tstate->interp->optimizer_side_threshold) { + if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { Py_DECREF(previous); tstate->previous_executor = NULL; GOTO_TIER_ONE(target); @@ -4124,7 +4124,8 @@ dummy_func( } else { int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { - exit->temperature = (int16_t)(-1 * tstate->interp->optimizer_side_threshold); + int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; + exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; Py_DECREF(previous); tstate->previous_executor = NULL; ERROR_IF(optimized < 0, error); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 182dc648b07f1c..465eee81c411e0 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3468,7 +3468,7 @@ assert(exit->executor->trace[0].opcode == _COLD_EXIT); PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; - if (exit->temperature < tstate->interp->optimizer_side_threshold) { + if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { Py_DECREF(previous); tstate->previous_executor = NULL; GOTO_TIER_ONE(target); @@ -3480,7 +3480,8 @@ } else { int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); if (optimized <= 0) { - exit->temperature = (int16_t)(-1 * tstate->interp->optimizer_side_threshold); + int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; + exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; Py_DECREF(previous); tstate->previous_executor = NULL; if (optimized < 0) goto error_tier_two; diff --git a/Python/optimizer.c b/Python/optimizer.c index 45e3415b5789da..f0ebc5e2575cfe 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1082,6 +1082,7 @@ PyUnstable_Optimizer_NewCounter(void) } opt->base.optimize = counter_optimize; opt->base.resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; + opt->base.side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD; opt->base.backedge_threshold = 0; opt->count = 0; return (PyObject *)opt; From bd66b011c97c281abad1fa9784626005df99b29c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Feb 2024 12:12:52 +0000 Subject: [PATCH 25/40] Statically allocate cold exits --- Python/optimizer.c | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index f0ebc5e2575cfe..27586a753077a6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -156,17 +156,23 @@ PyUnstable_GetOptimizer(void) static _PyExecutorObject * make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies); -static _PyExecutorObject * -make_cold_exit_executor(int oparg); +static void +init_cold_exit_executor(_PyExecutorObject *executor, int oparg); static int cold_exits_initialized = 0; -static const _PyExecutorObject *COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 }; +static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 }; static const _PyBloomFilter EMPTY = { 0 }; _PyOptimizerObject * _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) { + if (cold_exits_initialized == 0) { + cold_exits_initialized = 1; + for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { + init_cold_exit_executor(&COLD_EXITS[i], i); + } + } if (optimizer == NULL) { optimizer = &_PyOptimizer_Default; } @@ -199,12 +205,6 @@ _PyOptimizer_Optimize( _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **executor_ptr) { - if (cold_exits_initialized == 0) { - cold_exits_initialized = 1; - for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { - COLD_EXITS[i] = make_cold_exit_executor(i); - } - } PyCodeObject *code = (PyCodeObject *)frame->f_executable; assert(PyCode_Check(code)); PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -841,7 +841,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende } /* Initialize exits */ for (int i = 0; i < exit_count; i++) { - executor->exits[i].executor = COLD_EXITS[i]; + executor->exits[i].executor = &COLD_EXITS[i]; executor->exits[i].temperature = 0; } int next_exit = exit_count-1; @@ -905,30 +905,29 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende return executor; } -static _PyExecutorObject * -make_cold_exit_executor(int oparg) +static void +init_cold_exit_executor(_PyExecutorObject *executor, int oparg) { - _PyExecutorObject *executor = allocate_executor(0, 1); - if (executor == NULL) { - return NULL; - } + _Py_SetImmortal(executor); + Py_SET_TYPE(executor, &_PyUOpExecutor_Type); + executor->trace = (_PyUOpInstruction *)executor->exits; + executor->code_size = 1; + executor->exit_count = 0; _PyUOpInstruction *inst = (_PyUOpInstruction *)&executor->trace[0]; inst->opcode = _COLD_EXIT; inst->oparg = oparg; executor->vm_data.valid = true; for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { - executor->vm_data.bloom.bits[i] = 0; + assert(executor->vm_data.bloom.bits[i] == 0); } #ifdef _Py_JIT executor->jit_code = NULL; - executor->code_size = 0; + executor->jit_size = 0; if (_PyJIT_Compile(executor, executor->trace, 1)) { Py_DECREF(executor); return NULL; } #endif - _Py_SetImmortal(executor); - return executor; } static int @@ -1249,7 +1248,7 @@ _Py_ExecutorClear(_PyExecutorObject *executor) } for (uint32_t i = 0; i < executor->exit_count; i++) { Py_DECREF(executor->exits[i].executor); - executor->exits[i].executor = COLD_EXITS[i]; + executor->exits[i].executor = &COLD_EXITS[i]; executor->exits[i].temperature = INT16_MIN; } _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; From fe7548404f4a1c3c8b511d3a9c647c5950146fcf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Feb 2024 12:33:16 +0000 Subject: [PATCH 26/40] Handle errors in JIT compile --- Include/cpython/optimizer.h | 2 +- Modules/_testinternalcapi.c | 4 +++- Python/optimizer.c | 30 +++++++++++++++++++----------- Python/pylifecycle.c | 4 +++- Python/pystate.c | 1 + 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index d59d0dc3186c55..7b161a16156afd 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -92,7 +92,7 @@ PyAPI_FUNC(int) PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *in _PyOptimizerObject *_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject* optimizer); -PyAPI_FUNC(void) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); +PyAPI_FUNC(int) PyUnstable_SetOptimizer(_PyOptimizerObject* optimizer); PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 3834f00009cea4..f0d1a8aaed5ca2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -977,7 +977,9 @@ set_optimizer(PyObject *self, PyObject *opt) if (opt == Py_None) { opt = NULL; } - PyUnstable_SetOptimizer((_PyOptimizerObject*)opt); + if (PyUnstable_SetOptimizer((_PyOptimizerObject*)opt)) { + return NULL; + } Py_RETURN_NONE; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 27586a753077a6..4b8ebefaf3d397 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -156,7 +156,7 @@ PyUnstable_GetOptimizer(void) static _PyExecutorObject * make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies); -static void +static int init_cold_exit_executor(_PyExecutorObject *executor, int oparg); static int cold_exits_initialized = 0; @@ -167,16 +167,23 @@ static const _PyBloomFilter EMPTY = { 0 }; _PyOptimizerObject * _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) { - if (cold_exits_initialized == 0) { + + if (optimizer == NULL) { + optimizer = &_PyOptimizer_Default; + } + else if (cold_exits_initialized == 0) { cold_exits_initialized = 1; for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { - init_cold_exit_executor(&COLD_EXITS[i], i); + if (init_cold_exit_executor(&COLD_EXITS[i], i)) { + PyErr_NoMemory(); + return NULL; + } } } - if (optimizer == NULL) { - optimizer = &_PyOptimizer_Default; - } _PyOptimizerObject *old = interp->optimizer; + if (old == NULL) { + old = &_PyOptimizer_Default; + } Py_INCREF(optimizer); interp->optimizer = optimizer; interp->optimizer_backedge_threshold = shift_and_offset_threshold(optimizer->backedge_threshold); @@ -189,12 +196,13 @@ _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) return old; } -void +int PyUnstable_SetOptimizer(_PyOptimizerObject *optimizer) { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyOptimizerObject *old = _Py_SetOptimizer(interp, optimizer); - Py_DECREF(old); + Py_XDECREF(old); + return old == NULL ? -1 : 0; } /* Returns 1 if optimized, 0 if not optimized, and -1 for an error. @@ -905,7 +913,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende return executor; } -static void +static int init_cold_exit_executor(_PyExecutorObject *executor, int oparg) { _Py_SetImmortal(executor); @@ -924,10 +932,10 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) executor->jit_code = NULL; executor->jit_size = 0; if (_PyJIT_Compile(executor, executor->trace, 1)) { - Py_DECREF(executor); - return NULL; + return -1; } #endif + return 0; } static int diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 7e4c07bb657d19..c2f3d58d2ccf4b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1262,7 +1262,9 @@ init_interp_main(PyThreadState *tstate) if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); } - PyUnstable_SetOptimizer((_PyOptimizerObject *)opt); + if (PyUnstable_SetOptimizer((_PyOptimizerObject *)opt)) { + return _PyStatus_ERR("can't initialize optimizer"); + } Py_DECREF(opt); } } diff --git a/Python/pystate.c b/Python/pystate.c index 2180281275a442..7b5a08ea5ddcc1 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -779,6 +779,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); + assert(old != NULL); Py_DECREF(old); /* It is possible that any of the objects below have a finalizer From de93130eb560b28a3fe9f6418b8dfde59d02bd68 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Feb 2024 14:33:19 +0000 Subject: [PATCH 27/40] Fix possible leak --- Tools/jit/template.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 2b66e21b6fa0d6..cae6684f637b55 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -40,7 +40,6 @@ #undef GOTO_TIER_TWO #define GOTO_TIER_TWO(EXECUTOR) \ - tstate->previous_executor = (PyObject *)current_executor; \ __attribute__((musttail)) \ return ((jit_func)((EXECUTOR)->jit_code))(frame, stack_pointer, tstate); From 77a67403705e28deb849bda48d94a93ed12be70d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Feb 2024 15:57:04 +0000 Subject: [PATCH 28/40] Fix refleak transfering from JIT to tier 1 --- Include/internal/pycore_jit.h | 2 +- Python/bytecodes.c | 10 +++++----- Python/ceval_macros.h | 4 ++++ Python/executor_cases.c.h | 10 +++++----- Python/jit.c | 2 +- Tools/jit/template.c | 2 ++ 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 0b71eb6f758ac6..17bd23f0752be2 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -13,7 +13,7 @@ extern "C" { typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate); -int _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length); +int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); #endif // _Py_JIT diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 03f0719dc95363..7649558f3b10ef 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4113,8 +4113,6 @@ dummy_func( PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { - Py_DECREF(previous); - tstate->previous_executor = NULL; GOTO_TIER_ONE(target); } _PyExecutorObject *executor; @@ -4126,9 +4124,11 @@ dummy_func( if (optimized <= 0) { int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; - Py_DECREF(previous); - tstate->previous_executor = NULL; - ERROR_IF(optimized < 0, error); + if (optimized < 0) { + Py_DECREF(previous); + tstate->previous_executor = Py_None; + ERROR_IF(1, error); + } GOTO_TIER_ONE(target); } } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b6e31bb3f5de76..f55a4ddf947793 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -392,6 +392,8 @@ stack_pointer = _PyFrame_GetStackPointer(frame); do { \ jit_func jitted = (EXECUTOR)->jit_code; \ next_instr = jitted(frame, stack_pointer, tstate); \ + Py_DECREF(tstate->previous_executor); \ + tstate->previous_executor = NULL; \ frame = tstate->current_frame; \ if (next_instr == NULL) { \ goto resume_with_error; \ @@ -409,6 +411,8 @@ do { \ #endif #define GOTO_TIER_ONE(TARGET) \ + Py_DECREF(tstate->previous_executor); \ + tstate->previous_executor = NULL; \ next_instr = target; \ DISPATCH(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 465eee81c411e0..b89cce74a7e746 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3469,8 +3469,6 @@ PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { - Py_DECREF(previous); - tstate->previous_executor = NULL; GOTO_TIER_ONE(target); } _PyExecutorObject *executor; @@ -3482,9 +3480,11 @@ if (optimized <= 0) { int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp; - Py_DECREF(previous); - tstate->previous_executor = NULL; - if (optimized < 0) goto error_tier_two; + if (optimized < 0) { + Py_DECREF(previous); + tstate->previous_executor = Py_None; + if (1) goto error_tier_two; + } GOTO_TIER_ONE(target); } } diff --git a/Python/jit.c b/Python/jit.c index 43930347a60a24..a23b56d2471d28 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -300,7 +300,7 @@ emit(const StencilGroup *group, uint64_t patches[]) // Compiles executor in-place. Don't forget to call _PyJIT_Free later! int -_PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length) +_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length) { // Loop once to find the total compiled size: size_t code_size = 0; diff --git a/Tools/jit/template.c b/Tools/jit/template.c index cae6684f637b55..6e9d8dd64a7d35 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -102,9 +102,11 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * STACK_SHRINK(1); error_tier_two: _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->previous_executor = (PyObject *)current_executor; return NULL; deoptimize: _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->previous_executor = (PyObject *)current_executor; return _PyCode_CODE(_PyFrame_GetCode(frame)) + _target; side_exit: { From 0a61d29d93ed613e61a6d697e23927ed6fbb748f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Feb 2024 17:44:05 +0000 Subject: [PATCH 29/40] Check that only one of EXIT_IF and DEOPT_IF is present --- Lib/test/test_generated_cases.py | 11 +++++++++++ Tools/cases_generator/analyzer.py | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index a7ad6c7320b4ee..0d2ccd558d436d 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -794,6 +794,17 @@ def test_annotated_op(self): self.run_cases_test(input, output) + def test_deopt_and_exit(self): + input = """ + pure op(OP, (arg1 -- out)) { + DEOPT_IF(1); + EXIT_IF(1); + } + """ + output = "" + with self.assertRaises(Exception): + self.run_cases_test(input, output) + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: super().setUp() diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index f88032a2f7a514..2a1b28b1605f00 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -453,6 +453,10 @@ def compute_properties(op: parser.InstDef) -> Properties: ) deopts_if = variable_used(op, "DEOPT_IF") exits_if = variable_used(op, "EXIT_IF") + if deopts_if and exits_if: + tkn = op.tokens[0] + raise lexer.make_syntax_error("Op cannot contain both EXIT_IF and DEOPT_IF", + tkn.filename, tkn.line, tkn.column, op.name) infallible = is_infallible(op) passthrough = stack_effect_only_peeks(op) and infallible return Properties( From b3e306df7e04f053ff59a26d8b2cc98807552ceb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Feb 2024 17:54:10 +0000 Subject: [PATCH 30/40] Address review comments --- Python/bytecodes.c | 5 +++-- Python/ceval.c | 4 ++-- Python/executor_cases.c.h | 2 -- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7649558f3b10ef..d64e9fc6dd7500 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4103,13 +4103,14 @@ dummy_func( exe->count++; } + /* Only used for handling cold side exits, should never appear in + * a normal trace or as part of an instruction. + */ op(_COLD_EXIT, (--)) { TIER_TWO_ONLY _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; - assert(previous->trace[0].opcode != _COLD_EXIT); _PyExitData *exit = &previous->exits[oparg]; exit->temperature++; - assert(exit->executor->trace[0].opcode == _COLD_EXIT); PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { diff --git a/Python/ceval.c b/Python/ceval.c index d81c6afb5e5586..752ee88c026f7b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1088,8 +1088,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uint16_t exit_index = next_uop[-1].exit_index; _PyExitData *exit = ¤t_executor->exits[exit_index]; DPRINTF(2, "SIDE EXIT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", exit %u, temp %d, target %d -> %s]\n", - uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit_index, exit->temperature, exit->target, - _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); + uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit_index, exit->temperature, + exit->target, _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); Py_INCREF(exit->executor); tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_TWO(exit->executor); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b89cce74a7e746..a62acf6dc9aac1 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3462,10 +3462,8 @@ oparg = CURRENT_OPARG(); TIER_TWO_ONLY _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; - assert(previous->trace[0].opcode != _COLD_EXIT); _PyExitData *exit = &previous->exits[oparg]; exit->temperature++; - assert(exit->executor->trace[0].opcode == _COLD_EXIT); PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { From 8f3aa33363000d30eeeaa08e51987e0564379a00 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 14 Feb 2024 18:13:33 +0000 Subject: [PATCH 31/40] Make exit_index 32 bits to avoid endianness issues in JIT --- Include/cpython/optimizer.h | 2 +- Python/ceval.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 7b161a16156afd..851ad54e09a801 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -35,7 +35,7 @@ typedef struct { uint16_t oparg; union { uint32_t target; - uint16_t exit_index; + uint32_t exit_index; }; uint64_t operand; // A cache entry } _PyUOpInstruction; diff --git a/Python/ceval.c b/Python/ceval.c index 752ee88c026f7b..0bc651bc8b64e4 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1085,7 +1085,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int side_exit: OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); - uint16_t exit_index = next_uop[-1].exit_index; + uint32_t exit_index = next_uop[-1].exit_index; + assert(exit_index < current_executor->exit_count); _PyExitData *exit = ¤t_executor->exits[exit_index]; DPRINTF(2, "SIDE EXIT: [UOp %d (%s), oparg %d, operand %" PRIu64 ", exit %u, temp %d, target %d -> %s]\n", uopcode, _PyUOpName(uopcode), next_uop[-1].oparg, next_uop[-1].operand, exit_index, exit->temperature, From 7c84967579f82c48446081cd1db085d9f3ad2804 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 15 Feb 2024 07:28:16 +0000 Subject: [PATCH 32/40] Run black --- Tools/cases_generator/analyzer.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 2a1b28b1605f00..bcffd75269ac0b 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -272,9 +272,7 @@ def override_error( def convert_stack_item(item: parser.StackEffect) -> StackItem: - return StackItem( - item.name, item.type, item.cond, (item.size or "1") - ) + return StackItem(item.name, item.type, item.cond, (item.size or "1")) def analyze_stack(op: parser.InstDef) -> StackEffect: @@ -455,8 +453,13 @@ def compute_properties(op: parser.InstDef) -> Properties: exits_if = variable_used(op, "EXIT_IF") if deopts_if and exits_if: tkn = op.tokens[0] - raise lexer.make_syntax_error("Op cannot contain both EXIT_IF and DEOPT_IF", - tkn.filename, tkn.line, tkn.column, op.name) + raise lexer.make_syntax_error( + "Op cannot contain both EXIT_IF and DEOPT_IF", + tkn.filename, + tkn.line, + tkn.column, + op.name, + ) infallible = is_infallible(op) passthrough = stack_effect_only_peeks(op) and infallible return Properties( From 8ee6710077e528f246c23c262fa6b2a309d42cee Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 15 Feb 2024 07:31:20 +0000 Subject: [PATCH 33/40] Address code review --- Include/internal/pycore_interp.h | 4 +++- Python/optimizer.c | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 8a4d33dcd18486..5fd9f4f50013a2 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -235,9 +235,11 @@ struct _is { struct callable_cache callable_cache; _PyOptimizerObject *optimizer; _PyExecutorObject *executor_list_head; - /* These values are shifted and offset to speed up check in JUMP_BACKWARD */ + + /* These two values are shifted and offset to speed up check in JUMP_BACKWARD */ uint32_t optimizer_resume_threshold; uint32_t optimizer_backedge_threshold; + uint16_t optimizer_side_threshold; uint32_t next_func_version; diff --git a/Python/optimizer.c b/Python/optimizer.c index 4b8ebefaf3d397..4ddcf469e047a3 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -128,8 +128,8 @@ static _PyOptimizerObject _PyOptimizer_Default = { PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type) .optimize = never_optimize, .resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, - .side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, .backedge_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, + .side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD, }; static uint32_t @@ -167,7 +167,6 @@ static const _PyBloomFilter EMPTY = { 0 }; _PyOptimizerObject * _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) { - if (optimizer == NULL) { optimizer = &_PyOptimizer_Default; } @@ -187,8 +186,8 @@ _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) Py_INCREF(optimizer); interp->optimizer = optimizer; interp->optimizer_backedge_threshold = shift_and_offset_threshold(optimizer->backedge_threshold); - interp->optimizer_side_threshold = optimizer->side_threshold; interp->optimizer_resume_threshold = shift_and_offset_threshold(optimizer->resume_threshold); + interp->optimizer_side_threshold = optimizer->side_threshold; if (optimizer == &_PyOptimizer_Default) { assert(interp->optimizer_backedge_threshold > (1 << 16)); assert(interp->optimizer_resume_threshold > (1 << 16)); From f37d7fc2bd6f90b4d402136fbfc931a3029949f6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 15 Feb 2024 07:57:00 +0000 Subject: [PATCH 34/40] Update comment --- Python/ceval.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index 0bc651bc8b64e4..adccf8fc00f69c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -745,7 +745,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject **stack_pointer; #ifndef _Py_JIT - /* State shared between Tier 1 and Tier 2 interpreter */ + /* Tier 2 interpreter state */ _PyExecutorObject *current_executor = NULL; const _PyUOpInstruction *next_uop = NULL; #endif From 1f8967d4affac173e771fb031cd35b1b42da05f3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 15 Feb 2024 08:35:31 +0000 Subject: [PATCH 35/40] Address review comments --- .gitattributes | 3 ++- Python/tier2_engine.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitattributes b/.gitattributes index 07d877027b09f6..159cd83ff7407d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -77,6 +77,7 @@ Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated Include/internal/pycore_uop_ids.h generated +Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated Include/opcode_ids.h generated Include/token.h generated @@ -94,7 +95,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Python/tier2_redundancy_eliminator_bytecodes.c.h generated +Python/tier2_redundancy_eliminator_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/Python/tier2_engine.md b/Python/tier2_engine.md index f9a06f77d1e495..80ec1e4bba0353 100644 --- a/Python/tier2_engine.md +++ b/Python/tier2_engine.md @@ -25,7 +25,7 @@ which is equivalent but more efficient. A superblock is a representation of the code we want to execute, but it is not in executable form. -The executable form is know as an executor. +The executable form is known as an executor. Executors are semantically equivalent to the superblock they are created from, but are in a form that can be efficiently executable. @@ -89,7 +89,7 @@ executing it. We want to minimize the amount of data that is passed from one executor to the next. In the JIT, this reduces the number of arguments in the tailcall, freeing up registers for other uses. -It is less important in the interpreter, but follwing the same +It is less important in the interpreter, but following the same design as the JIT simplifies debugging and is good for performance. Provided that we incref the new executor before executing it, we From 8e4c6013e9c78ef2706ae718a78f47a27bdb4a75 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 15 Feb 2024 09:08:35 +0000 Subject: [PATCH 36/40] Fix compiler warning --- Python/optimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 4ddcf469e047a3..b4afa1768dd5bd 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -874,7 +874,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende } /* Set the oparg to be the destination offset, * so that we can set the oparg of earlier jumps correctly. */ - buffer[i].oparg = dest - executor->trace; + buffer[i].oparg = (uint16_t)(dest - executor->trace); dest--; } assert(next_exit == -1); From 4eb2cfc2a5038c59d191629d992fa4f3a88a6c9a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 15 Feb 2024 11:57:39 +0000 Subject: [PATCH 37/40] Address review comments --- Include/cpython/optimizer.h | 6 ------ Lib/test/test_frame.py | 2 +- Python/bytecodes.c | 6 ++++-- Python/ceval_macros.h | 30 ++++++++++++++++-------------- Python/executor_cases.c.h | 6 ++++-- Python/jit.c | 4 ++-- Python/optimizer.c | 5 ++--- Python/tier2_engine.md | 13 +++++++------ Tools/jit/template.c | 19 +++++++++---------- 9 files changed, 45 insertions(+), 46 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 851ad54e09a801..fe54d1ddfe6129 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -57,12 +57,6 @@ typedef struct _PyExecutorObject { _PyExitData exits[1]; } _PyExecutorObject; -typedef struct _cold_exit { - _PyExecutorObject base; - _PyUOpInstruction uop; -} _PyColdExitObject; - - typedef struct _PyOptimizerObject _PyOptimizerObject; /* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 0510a3cd9a2599..f0b7efe719abe1 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -328,9 +328,9 @@ def f(): next(g) # Move all objects to the oldest generation, and tell the GC to run # on the *very next* allocation: - sys._clear_internal_caches() gc.collect() gc.set_threshold(1, 0, 0) + sys._clear_internal_caches() # Okay, so here's the nightmare scenario: # - We're tracing the resumption of a generator, which creates a new # frame object. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d64e9fc6dd7500..504554640910c9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4035,7 +4035,7 @@ dummy_func( } op(_JUMP_TO_TOP, (--)) { -#ifndef Py_JIT +#ifndef _Py_JIT next_uop = ¤t_executor->trace[1]; #endif CHECK_EVAL_BREAKER(); @@ -4133,6 +4133,8 @@ dummy_func( GOTO_TIER_ONE(target); } } + /* We need two references. One to store in exit->executor and + * one to keep the executor alive when executing. */ Py_INCREF(executor); exit->executor = executor; GOTO_TIER_TWO(executor); @@ -4142,7 +4144,7 @@ dummy_func( TIER_TWO_ONLY Py_DECREF(tstate->previous_executor); tstate->previous_executor = NULL; -#ifndef Py_JIT +#ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f55a4ddf947793..df6e95ea8a4ddb 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -388,19 +388,19 @@ stack_pointer = _PyFrame_GetStackPointer(frame); /* Tier-switching macros. */ #ifdef _Py_JIT -#define GOTO_TIER_TWO(EXECUTOR) \ - do { \ - jit_func jitted = (EXECUTOR)->jit_code; \ - next_instr = jitted(frame, stack_pointer, tstate); \ - Py_DECREF(tstate->previous_executor); \ - tstate->previous_executor = NULL; \ - frame = tstate->current_frame; \ - if (next_instr == NULL) { \ - goto resume_with_error; \ - } \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - DISPATCH(); \ - } while (0) +#define GOTO_TIER_TWO(EXECUTOR) \ +do { \ + jit_func jitted = (EXECUTOR)->jit_code; \ + next_instr = jitted(frame, stack_pointer, tstate); \ + Py_DECREF(tstate->previous_executor); \ + tstate->previous_executor = NULL; \ + frame = tstate->current_frame; \ + if (next_instr == NULL) { \ + goto resume_with_error; \ + } \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + DISPATCH(); \ +} while (0) #else #define GOTO_TIER_TWO(EXECUTOR) \ do { \ @@ -411,10 +411,12 @@ do { \ #endif #define GOTO_TIER_ONE(TARGET) \ +do { \ Py_DECREF(tstate->previous_executor); \ tstate->previous_executor = NULL; \ next_instr = target; \ - DISPATCH(); + DISPATCH(); \ +while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a62acf6dc9aac1..7fdbf35dc9c63e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3352,7 +3352,7 @@ } case _JUMP_TO_TOP: { - #ifndef Py_JIT + #ifndef _Py_JIT next_uop = ¤t_executor->trace[1]; #endif CHECK_EVAL_BREAKER(); @@ -3486,6 +3486,8 @@ GOTO_TIER_ONE(target); } } + /* We need two references. One to store in exit->executor and + * one to keep the executor alive when executing. */ Py_INCREF(executor); exit->executor = executor; GOTO_TIER_TWO(executor); @@ -3497,7 +3499,7 @@ TIER_TWO_ONLY Py_DECREF(tstate->previous_executor); tstate->previous_executor = NULL; - #ifndef Py_JIT + #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif break; diff --git a/Python/jit.c b/Python/jit.c index a23b56d2471d28..839414bd810677 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -306,7 +306,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size size_t code_size = 0; size_t data_size = 0; for (size_t i = 0; i < length; i++) { - _PyUOpInstruction *instruction = &trace[i]; + _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; code_size += group->code.body_size; data_size += group->data.body_size; @@ -329,7 +329,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size top += stencil_groups[_START_EXECUTOR].code.body_size; } for (size_t i = 0; i < length; i++) { - _PyUOpInstruction *instruction = &trace[i]; + _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; // Think of patches as a dictionary mapping HoleValue to uint64_t: uint64_t patches[] = GET_PATCHES(); diff --git a/Python/optimizer.c b/Python/optimizer.c index b4afa1768dd5bd..1f31f69dea354b 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -174,7 +174,6 @@ _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) cold_exits_initialized = 1; for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { if (init_cold_exit_executor(&COLD_EXITS[i], i)) { - PyErr_NoMemory(); return NULL; } } @@ -1005,7 +1004,7 @@ PyUnstable_Optimizer_NewUOpOptimizer(void) static void counter_dealloc(_PyExecutorObject *self) { - /* The optimizer is the operand of the first uop. */ + /* The optimizer is the operand of the second uop. */ PyObject *opt = (PyObject *)self->trace[1].operand; Py_DECREF(opt); uop_dealloc(self); @@ -1043,7 +1042,7 @@ counter_optimize( return 0; } _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; - _PyUOpInstruction buffer[4] = { + _PyUOpInstruction buffer[3] = { { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) } diff --git a/Python/tier2_engine.md b/Python/tier2_engine.md index 80ec1e4bba0353..acf0ce975ade32 100644 --- a/Python/tier2_engine.md +++ b/Python/tier2_engine.md @@ -95,8 +95,11 @@ design as the JIT simplifies debugging and is good for performance. Provided that we incref the new executor before executing it, we can jump directly to the code of the executor, without needing to pass a reference to that executor object. -However, we do need to pass a reference to the previous executor, +However, we do need a reference to the previous executor, so that it can be decref'd and for handling of cold exits. +To avoid messing up the JIT's register allocation, we pass a +reference to the previous executor in the thread state's +`previous_executor` field. #### The interpreter @@ -112,7 +115,7 @@ points to the currently live executor. When transfering from executor We also make the first instruction in `B` do the following: 1. Set `current_executor` to point to `B` -2. Decrement the reference count of `A` +2. Decrement the reference count of `A` (`A` is referenced by `tstate->previous_executor`) The net effect of the above is to safely decrement the refcount of `A`, increment the refcount of `B` and set `current_executor` to point to `B`. @@ -122,8 +125,7 @@ increment the refcount of `B` and set `current_executor` to point to `B`. Transfering control from one executor to another is done via tailcalls. The compiled executor should do the same, except that there is no local -variable `current_executor`, so that the old executor, `A`, must be passed -as an additional parameter when tailcalling. +variable `current_executor`. ### Tier 1 to tier 2 @@ -131,9 +133,8 @@ Since the executor doesn't know if the previous code was tier 1 or tier 2, we need to make a transfer from tier 1 to tier 2 look like a tier 2 to tier 2 transfer to the executor. -To do that, we create a single, immortal executor that cannot be executed. We can then perform a tier 1 to tier 2 transfer by setting `current_executor` -to this singleton, and then performing a tier 2 to tier 2 transfer as above. +to `None`, and then performing a tier 2 to tier 2 transfer as above. ### Tier 2 to tier 1 diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 6e9d8dd64a7d35..d79c6efb8f6de4 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -40,13 +40,17 @@ #undef GOTO_TIER_TWO #define GOTO_TIER_TWO(EXECUTOR) \ +do { \ __attribute__((musttail)) \ - return ((jit_func)((EXECUTOR)->jit_code))(frame, stack_pointer, tstate); + return ((jit_func)((EXECUTOR)->jit_code))(frame, stack_pointer, tstate); \ +} while (0) #undef GOTO_TIER_ONE #define GOTO_TIER_ONE(TARGET) \ +do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; + return TARGET; \ +} while (0) #undef LOAD_IP #define LOAD_IP(UNUSED) \ @@ -62,8 +66,6 @@ __attribute__((musttail)) \ return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); -#define Py_JIT 1 - _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) { @@ -101,19 +103,16 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * pop_1_error_tier_two: STACK_SHRINK(1); error_tier_two: - _PyFrame_SetStackPointer(frame, stack_pointer); tstate->previous_executor = (PyObject *)current_executor; - return NULL; + GOTO_TIER_ONE(NULL); deoptimize: - _PyFrame_SetStackPointer(frame, stack_pointer); tstate->previous_executor = (PyObject *)current_executor; - return _PyCode_CODE(_PyFrame_GetCode(frame)) + _target; + GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); side_exit: { _PyExitData *exit = ¤t_executor->exits[_target]; Py_INCREF(exit->executor); tstate->previous_executor = (PyObject *)current_executor; - __attribute__((musttail)) - return ((jit_func)exit->executor->jit_code)(frame, stack_pointer, tstate); + GOTO_TIER_TWO(exit->executor); } } From ebe804f7f9fed3835b44dbcc316a9417b24ed882 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 15 Feb 2024 12:42:49 +0000 Subject: [PATCH 38/40] Add missing brace --- Python/ceval_macros.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index df6e95ea8a4ddb..188dac71524133 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -416,7 +416,7 @@ do { \ tstate->previous_executor = NULL; \ next_instr = target; \ DISPATCH(); \ -while (0) +} while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) From c38d4e8838eecebbe8b1a0b992ee291b1fc45b37 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 15 Feb 2024 16:59:25 +0000 Subject: [PATCH 39/40] Address review comments --- Modules/_testinternalcapi.c | 2 +- Python/optimizer.c | 4 ++-- Python/tier2_engine.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f0d1a8aaed5ca2..bcc431a27001f2 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -977,7 +977,7 @@ set_optimizer(PyObject *self, PyObject *opt) if (opt == Py_None) { opt = NULL; } - if (PyUnstable_SetOptimizer((_PyOptimizerObject*)opt)) { + if (PyUnstable_SetOptimizer((_PyOptimizerObject*)opt) < 0) { return NULL; } Py_RETURN_NONE; diff --git a/Python/optimizer.c b/Python/optimizer.c index 1f31f69dea354b..acc1d545d2b8ad 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -162,7 +162,7 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg); static int cold_exits_initialized = 0; static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 }; -static const _PyBloomFilter EMPTY = { 0 }; +static const _PyBloomFilter EMPTY_FILTER = { 0 }; _PyOptimizerObject * _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) @@ -1047,7 +1047,7 @@ counter_optimize( { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) } }; - _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY); + _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY_FILTER); if (executor == NULL) { return -1; } diff --git a/Python/tier2_engine.md b/Python/tier2_engine.md index acf0ce975ade32..df9f6c124509bd 100644 --- a/Python/tier2_engine.md +++ b/Python/tier2_engine.md @@ -10,7 +10,7 @@ Since each executor must exit, we also track the "hotness" of those exits and attach new executors to those exits. As the program executes, and the hot parts of the program get optimized, -a graph of executors will form. +a graph of executors forms. ## Superblocks and Executors @@ -50,7 +50,7 @@ different instruction set, and does not adapt. The JIT compiler converts superblocks into machine code executors. These have identical behavior to interpreted executors, except that -they consume a bit more memory and are a lot faster. +they consume more memory for the generated machine code and are a lot faster. ## Transfering control From 830eb4eb10ec99bf85716d2778a677687ee3c4c0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 15 Feb 2024 17:08:56 +0000 Subject: [PATCH 40/40] Keep c-analyzer quiet --- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 77713e81b9ed1d..94be3375849597 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -385,7 +385,7 @@ Python/optimizer.c - _PyOptimizer_Default - Python/optimizer.c - _ColdExit_Type - Python/optimizer.c - COLD_EXITS - Python/optimizer.c - Py_FatalErrorExecutor - -Python/optimizer.c - EMPTY - +Python/optimizer.c - EMPTY_FILTER - Python/optimizer.c - cold_exits_initialized - ##-----------------------